Lines Matching +full:peer +full:- +full:hub
37 #include <linux/pci-p2pdma.h>
38 #include <linux/apple-gmux.h>
89 #include <asm/intel-family.h>
104 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
155 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0)
193 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0; in amdgpu_ip_member_of_hwini()
201 adev->init_lvl = &amdgpu_init_minimal_xgmi; in amdgpu_set_init_level()
204 adev->init_lvl = &amdgpu_init_recovery; in amdgpu_set_init_level()
209 adev->init_lvl = &amdgpu_init_default; in amdgpu_set_init_level()
245 ret = sysfs_create_file(&adev->dev->kobj, in amdgpu_device_attr_sysfs_init()
254 sysfs_remove_file(&adev->dev->kobj, in amdgpu_device_attr_sysfs_fini()
289 return -EINVAL; in amdgpu_sysfs_reg_state_get()
305 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_init()
314 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); in amdgpu_reg_state_sysfs_fini()
327 * - "cem" - PCIE CEM card
328 * - "oam" - Open Compute Accelerator Module
329 * - "unknown" - Not known
342 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) in amdgpu_device_get_board_info()
343 pkg_type = adev->smuio.funcs->get_pkg_type(adev); in amdgpu_device_get_board_info()
374 if (adev->flags & AMD_IS_APU) in amdgpu_board_attrs_is_visible()
377 return attr->mode; in amdgpu_board_attrs_is_visible()
388 * This is a read-only file that lists all available UMA allocation
409 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; in carveout_options_show()
413 if (!uma_info || !uma_info->num_entries) in carveout_options_show()
414 return -ENODEV; in carveout_options_show()
416 for (int i = 0; i < uma_info->num_entries; i++) { in carveout_options_show()
417 memory_carved = uma_info->entries[i].memory_carved_mb; in carveout_options_show()
421 uma_info->entries[i].name, in carveout_options_show()
426 uma_info->entries[i].name, in carveout_options_show()
453 return sysfs_emit(buf, "%u\n", adev->uma_info.uma_option_index); in carveout_show()
462 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; in carveout_store()
472 if (val >= uma_info->num_entries) in carveout_store()
473 return -EINVAL; in carveout_store()
475 val = array_index_nospec(val, uma_info->num_entries); in carveout_store()
476 opt = &uma_info->entries[val]; in carveout_store()
478 if (!(opt->flags & AMDGPU_UMA_FLAG_AUTO) && in carveout_store()
479 !(opt->flags & AMDGPU_UMA_FLAG_CUSTOM)) { in carveout_store()
481 return -EINVAL; in carveout_store()
484 flags = opt->flags; in carveout_store()
487 guard(mutex)(&uma_info->update_lock); in carveout_store()
493 uma_info->uma_option_index = val; in carveout_store()
514 if (!(adev->flags & AMD_IS_APU)) in amdgpu_uma_sysfs_init()
520 rc = amdgpu_atomfirmware_get_uma_carveout_info(adev, &adev->uma_info); in amdgpu_uma_sysfs_init()
527 mutex_init(&adev->uma_info.update_lock); in amdgpu_uma_sysfs_init()
529 rc = devm_device_add_group(adev->dev, &amdgpu_uma_attr_group); in amdgpu_uma_sysfs_init()
538 mutex_destroy(&adev->uma_info.update_lock); in amdgpu_uma_sysfs_init()
545 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; in amdgpu_uma_sysfs_fini()
550 mutex_destroy(&uma_info->update_lock); in amdgpu_uma_sysfs_fini()
551 uma_info->num_entries = 0; in amdgpu_uma_sysfs_fini()
557 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
566 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) in amdgpu_device_supports_px()
572 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
584 if (adev->has_pr3 || in amdgpu_device_supports_boco()
585 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) in amdgpu_device_supports_boco()
591 * amdgpu_device_supports_baco - Does the device support BACO
609 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; in amdgpu_device_detect_runtime_pm_mode()
615 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; in amdgpu_device_detect_runtime_pm_mode()
616 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
618 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
619 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); in amdgpu_device_detect_runtime_pm_mode()
624 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
625 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
628 case -1: in amdgpu_device_detect_runtime_pm_mode()
629 case -2: in amdgpu_device_detect_runtime_pm_mode()
632 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; in amdgpu_device_detect_runtime_pm_mode()
633 dev_info(adev->dev, "Using ATPX for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
636 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; in amdgpu_device_detect_runtime_pm_mode()
637 dev_info(adev->dev, "Using BOCO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
642 switch (adev->asic_type) { in amdgpu_device_detect_runtime_pm_mode()
649 if (!adev->gmc.noretry && !amdgpu_passthrough(adev)) in amdgpu_device_detect_runtime_pm_mode()
650 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
655 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; in amdgpu_device_detect_runtime_pm_mode()
659 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { in amdgpu_device_detect_runtime_pm_mode()
661 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; in amdgpu_device_detect_runtime_pm_mode()
662 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
664 dev_info(adev->dev, "Using BACO for runtime pm\n"); in amdgpu_device_detect_runtime_pm_mode()
670 dev_info(adev->dev, "runtime pm is manually disabled\n"); in amdgpu_device_detect_runtime_pm_mode()
677 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) in amdgpu_device_detect_runtime_pm_mode()
678 dev_info(adev->dev, "Runtime PM not available\n"); in amdgpu_device_detect_runtime_pm_mode()
681 * amdgpu_device_supports_smart_shift - Is the device dGPU with
700 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
706 * @write: true - write to vram, otherwise - read from vram
722 spin_lock_irqsave(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
737 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); in amdgpu_device_mm_access()
742 * amdgpu_device_aper_access - access vram by vram aperture
748 * @write: true - write to vram, otherwise - read from vram
760 if (!adev->mman.aper_base_kaddr) in amdgpu_device_aper_access()
763 last = min(pos + size, adev->gmc.visible_vram_size); in amdgpu_device_aper_access()
765 addr = adev->mman.aper_base_kaddr + pos; in amdgpu_device_aper_access()
766 count = last - pos; in amdgpu_device_aper_access()
793 * amdgpu_device_vram_access - read/write a buffer in vram
799 * @write: true - write to vram, otherwise - read from vram
808 size -= count; in amdgpu_device_vram_access()
824 if (adev->no_hw_access) in amdgpu_device_skip_hw_access()
840 if (down_read_trylock(&adev->reset_domain->sem)) in amdgpu_device_skip_hw_access()
841 up_read(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
843 lockdep_assert_held(&adev->reset_domain->sem); in amdgpu_device_skip_hw_access()
850 * amdgpu_device_get_rev_id - query device rev_id
858 return adev->nbio.funcs->get_rev_id(adev); in amdgpu_device_get_rev_id()
863 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) in amdgpu_device_get_vbios_flags()
866 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev)) in amdgpu_device_get_vbios_flags()
873 * amdgpu_device_asic_init - Wrapper for atom asic_init
894 if (optional && !adev->bios) in amdgpu_device_asic_init()
900 if (optional && !adev->bios) in amdgpu_device_asic_init()
903 return amdgpu_atom_asic_init(adev->mode_info.atom_context); in amdgpu_device_asic_init()
910 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
922 &adev->mem_scratch.robj, in amdgpu_device_mem_scratch_init()
923 &adev->mem_scratch.gpu_addr, in amdgpu_device_mem_scratch_init()
924 (void **)&adev->mem_scratch.ptr); in amdgpu_device_mem_scratch_init()
928 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
936 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); in amdgpu_device_mem_scratch_fini()
940 * amdgpu_device_program_register_sequence - program an array of registers.
969 if (adev->family >= AMDGPU_FAMILY_AI) in amdgpu_device_program_register_sequence()
979 * amdgpu_device_pci_config_reset - reset the GPU
988 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); in amdgpu_device_pci_config_reset()
992 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1000 return pci_reset_function(adev->pdev); in amdgpu_device_pci_reset()
1010 * amdgpu_device_wb_fini - Disable Writeback and free memory
1019 if (adev->wb.wb_obj) { in amdgpu_device_wb_fini()
1020 amdgpu_bo_free_kernel(&adev->wb.wb_obj, in amdgpu_device_wb_fini()
1021 &adev->wb.gpu_addr, in amdgpu_device_wb_fini()
1022 (void **)&adev->wb.wb); in amdgpu_device_wb_fini()
1023 adev->wb.wb_obj = NULL; in amdgpu_device_wb_fini()
1028 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1034 * Returns 0 on success or an -error on failure.
1040 if (adev->wb.wb_obj == NULL) { in amdgpu_device_wb_init()
1044 &adev->wb.wb_obj, &adev->wb.gpu_addr, in amdgpu_device_wb_init()
1045 (void **)&adev->wb.wb); in amdgpu_device_wb_init()
1047 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); in amdgpu_device_wb_init()
1051 adev->wb.num_wb = AMDGPU_MAX_WB; in amdgpu_device_wb_init()
1052 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); in amdgpu_device_wb_init()
1055 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); in amdgpu_device_wb_init()
1062 * amdgpu_device_wb_get - Allocate a wb entry
1068 * Returns 0 on success or -EINVAL on failure.
1074 spin_lock_irqsave(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1075 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); in amdgpu_device_wb_get()
1076 if (offset < adev->wb.num_wb) { in amdgpu_device_wb_get()
1077 __set_bit(offset, adev->wb.used); in amdgpu_device_wb_get()
1078 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1082 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_get()
1083 return -EINVAL; in amdgpu_device_wb_get()
1088 * amdgpu_device_wb_free - Free a wb entry
1100 spin_lock_irqsave(&adev->wb.lock, flags); in amdgpu_device_wb_free()
1101 if (wb < adev->wb.num_wb) in amdgpu_device_wb_free()
1102 __clear_bit(wb, adev->wb.used); in amdgpu_device_wb_free()
1103 spin_unlock_irqrestore(&adev->wb.lock, flags); in amdgpu_device_wb_free()
1107 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1113 * driver loading by returning -ENODEV.
1117 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); in amdgpu_device_resize_fb_bar()
1136 adev->pdev->vendor == PCI_VENDOR_ID_ATI && in amdgpu_device_resize_fb_bar()
1137 adev->pdev->device == 0x731f && in amdgpu_device_resize_fb_bar()
1138 adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL) in amdgpu_device_resize_fb_bar()
1142 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) in amdgpu_device_resize_fb_bar()
1144 adev->dev, in amdgpu_device_resize_fb_bar()
1148 if (adev->gmc.real_vram_size && in amdgpu_device_resize_fb_bar()
1149 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) in amdgpu_device_resize_fb_bar()
1153 root = adev->pdev->bus; in amdgpu_device_resize_fb_bar()
1154 while (root->parent) in amdgpu_device_resize_fb_bar()
1155 root = root->parent; in amdgpu_device_resize_fb_bar()
1158 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && in amdgpu_device_resize_fb_bar()
1159 res->start > 0x100000000ull) in amdgpu_device_resize_fb_bar()
1163 /* Trying to resize is pointless without a root hub window above 4GB */ in amdgpu_device_resize_fb_bar()
1168 max_size = pci_rebar_get_max_size(adev->pdev, 0); in amdgpu_device_resize_fb_bar()
1174 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); in amdgpu_device_resize_fb_bar()
1175 pci_write_config_word(adev->pdev, PCI_COMMAND, in amdgpu_device_resize_fb_bar()
1181 r = pci_resize_resource(adev->pdev, 0, rbar_size, in amdgpu_device_resize_fb_bar()
1182 (adev->asic_type >= CHIP_BONAIRE) ? 1 << 5 in amdgpu_device_resize_fb_bar()
1184 if (r == -ENOSPC) in amdgpu_device_resize_fb_bar()
1185 dev_info(adev->dev, in amdgpu_device_resize_fb_bar()
1187 else if (r && r != -ENOTSUPP) in amdgpu_device_resize_fb_bar()
1188 dev_err(adev->dev, "Problem resizing BAR0 (%d).", r); in amdgpu_device_resize_fb_bar()
1194 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) in amdgpu_device_resize_fb_bar()
1195 return -ENODEV; in amdgpu_device_resize_fb_bar()
1197 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); in amdgpu_device_resize_fb_bar()
1206 * amdgpu_device_need_post - check if the hw need post or not
1224 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios) in amdgpu_device_need_post()
1228 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot in amdgpu_device_need_post()
1233 if (adev->asic_type == CHIP_FIJI) { in amdgpu_device_need_post()
1237 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); in amdgpu_device_need_post()
1242 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); in amdgpu_device_need_post()
1243 release_firmware(adev->pm.fw); in amdgpu_device_need_post()
1250 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) in amdgpu_device_need_post()
1253 if (adev->has_hw_reset) { in amdgpu_device_need_post()
1254 adev->has_hw_reset = false; in amdgpu_device_need_post()
1259 if (adev->asic_type >= CHIP_BONAIRE) in amdgpu_device_need_post()
1281 case -1: in amdgpu_device_seamless_boot_supported()
1288 dev_err(adev->dev, "Invalid value for amdgpu.seamless: %d\n", in amdgpu_device_seamless_boot_supported()
1293 if (!(adev->flags & AMD_IS_APU)) in amdgpu_device_seamless_boot_supported()
1296 if (adev->mman.keep_stolen_vga_memory) in amdgpu_device_seamless_boot_supported()
1307 …gn/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-…
1308 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1316 if (dev_is_removable(adev->dev)) in amdgpu_device_pcie_dynamic_switching_supported()
1319 if (c->x86_vendor == X86_VENDOR_INTEL) in amdgpu_device_pcie_dynamic_switching_supported()
1328 * It's unclear if this is a platform-specific or GPU-specific issue. in amdgpu_device_aspm_support_quirk()
1331 if (adev->family == AMDGPU_FAMILY_SI) in amdgpu_device_aspm_support_quirk()
1337 if (c->x86_vendor == X86_VENDOR_INTEL) { in amdgpu_device_aspm_support_quirk()
1338 switch (c->x86_model) { in amdgpu_device_aspm_support_quirk()
1359 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1371 case -1: in amdgpu_device_should_use_aspm()
1380 if (adev->flags & AMD_IS_APU) in amdgpu_device_should_use_aspm()
1384 return pcie_aspm_enabled(adev->pdev); in amdgpu_device_should_use_aspm()
1389 * amdgpu_device_vga_set_decode - enable/disable vga decode
1411 * amdgpu_device_check_block_size - validate the vm block size
1426 if (amdgpu_vm_block_size == -1) in amdgpu_device_check_block_size()
1430 dev_warn(adev->dev, "VM page table size (%d) too small\n", in amdgpu_device_check_block_size()
1432 amdgpu_vm_block_size = -1; in amdgpu_device_check_block_size()
1437 * amdgpu_device_check_vm_size - validate the vm size
1447 if (amdgpu_vm_size == -1) in amdgpu_device_check_vm_size()
1451 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", in amdgpu_device_check_vm_size()
1453 amdgpu_vm_size = -1; in amdgpu_device_check_vm_size()
1469 dev_warn(adev->dev, "Not 64-bit OS, feature not supported\n"); in amdgpu_device_check_smu_prv_buffer_size()
1484 dev_warn(adev->dev, "Smu memory pool size not supported\n"); in amdgpu_device_check_smu_prv_buffer_size()
1487 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; in amdgpu_device_check_smu_prv_buffer_size()
1492 dev_warn(adev->dev, "No enough system memory\n"); in amdgpu_device_check_smu_prv_buffer_size()
1494 adev->pm.smu_prv_buffer_size = 0; in amdgpu_device_check_smu_prv_buffer_size()
1499 if (!(adev->flags & AMD_IS_APU) || in amdgpu_device_init_apu_flags()
1500 adev->asic_type < CHIP_RAVEN) in amdgpu_device_init_apu_flags()
1503 switch (adev->asic_type) { in amdgpu_device_init_apu_flags()
1505 if (adev->pdev->device == 0x15dd) in amdgpu_device_init_apu_flags()
1506 adev->apu_flags |= AMD_APU_IS_RAVEN; in amdgpu_device_init_apu_flags()
1507 if (adev->pdev->device == 0x15d8) in amdgpu_device_init_apu_flags()
1508 adev->apu_flags |= AMD_APU_IS_PICASSO; in amdgpu_device_init_apu_flags()
1511 if ((adev->pdev->device == 0x1636) || in amdgpu_device_init_apu_flags()
1512 (adev->pdev->device == 0x164c)) in amdgpu_device_init_apu_flags()
1513 adev->apu_flags |= AMD_APU_IS_RENOIR; in amdgpu_device_init_apu_flags()
1515 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; in amdgpu_device_init_apu_flags()
1518 adev->apu_flags |= AMD_APU_IS_VANGOGH; in amdgpu_device_init_apu_flags()
1523 if ((adev->pdev->device == 0x13FE) || in amdgpu_device_init_apu_flags()
1524 (adev->pdev->device == 0x143F)) in amdgpu_device_init_apu_flags()
1525 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; in amdgpu_device_init_apu_flags()
1535 * amdgpu_device_check_arguments - validate module params
1547 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", in amdgpu_device_check_arguments()
1551 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1556 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { in amdgpu_device_check_arguments()
1558 dev_warn(adev->dev, "gart size (%d) too small\n", in amdgpu_device_check_arguments()
1560 amdgpu_gart_size = -1; in amdgpu_device_check_arguments()
1563 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { in amdgpu_device_check_arguments()
1565 dev_warn(adev->dev, "gtt size (%d) too small\n", in amdgpu_device_check_arguments()
1567 amdgpu_gtt_size = -1; in amdgpu_device_check_arguments()
1571 if (amdgpu_vm_fragment_size != -1 && in amdgpu_device_check_arguments()
1573 dev_warn(adev->dev, "valid range is between 4 and 9\n"); in amdgpu_device_check_arguments()
1574 amdgpu_vm_fragment_size = -1; in amdgpu_device_check_arguments()
1578 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", in amdgpu_device_check_arguments()
1582 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", in amdgpu_device_check_arguments()
1587 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { in amdgpu_device_check_arguments()
1588 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); in amdgpu_device_check_arguments()
1589 amdgpu_reset_method = -1; in amdgpu_device_check_arguments()
1598 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); in amdgpu_device_check_arguments()
1602 case -1: in amdgpu_device_check_arguments()
1606 adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE; in amdgpu_device_check_arguments()
1610 adev->enforce_isolation[i] = in amdgpu_device_check_arguments()
1615 adev->enforce_isolation[i] = in amdgpu_device_check_arguments()
1620 adev->enforce_isolation[i] = in amdgpu_device_check_arguments()
1630 * amdgpu_switcheroo_set_state - set switcheroo state
1651 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
1657 dev_warn(&pdev->dev, "pci_enable_device failed (%d)\n", in amdgpu_switcheroo_set_state()
1661 dev->switch_power_state = DRM_SWITCH_POWER_ON; in amdgpu_switcheroo_set_state()
1663 dev_info(&pdev->dev, "switched off\n"); in amdgpu_switcheroo_set_state()
1664 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; in amdgpu_switcheroo_set_state()
1671 dev->switch_power_state = DRM_SWITCH_POWER_OFF; in amdgpu_switcheroo_set_state()
1676 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1693 return atomic_read(&dev->open_count) == 0; in amdgpu_switcheroo_can_switch()
1703 * amdgpu_device_enable_virtual_display - enable virtual display feature
1716 adev->enable_virtual_display = false; in amdgpu_device_enable_virtual_display()
1719 const char *pci_address_name = pci_name(adev->pdev); in amdgpu_device_enable_virtual_display()
1729 int res = -1; in amdgpu_device_enable_virtual_display()
1731 adev->enable_virtual_display = true; in amdgpu_device_enable_virtual_display()
1742 adev->mode_info.num_crtc = num_crtc; in amdgpu_device_enable_virtual_display()
1744 adev->mode_info.num_crtc = 1; in amdgpu_device_enable_virtual_display()
1751 adev->dev, in amdgpu_device_enable_virtual_display()
1754 adev->enable_virtual_display, adev->mode_info.num_crtc); in amdgpu_device_enable_virtual_display()
1762 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { in amdgpu_device_set_sriov_virtual_display()
1763 adev->mode_info.num_crtc = 1; in amdgpu_device_set_sriov_virtual_display()
1764 adev->enable_virtual_display = true; in amdgpu_device_set_sriov_virtual_display()
1765 dev_info(adev->dev, "virtual_display:%d, num_crtc:%d\n", in amdgpu_device_set_sriov_virtual_display()
1766 adev->enable_virtual_display, in amdgpu_device_set_sriov_virtual_display()
1767 adev->mode_info.num_crtc); in amdgpu_device_set_sriov_virtual_display()
1772 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1779 * Returns 0 on success, -EINVAL on failure.
1787 adev->firmware.gpu_info_fw = NULL; in amdgpu_device_parse_gpu_info_fw()
1789 switch (adev->asic_type) { in amdgpu_device_parse_gpu_info_fw()
1799 if (adev->apu_flags & AMD_APU_IS_RAVEN2) in amdgpu_device_parse_gpu_info_fw()
1801 else if (adev->apu_flags & AMD_APU_IS_PICASSO) in amdgpu_device_parse_gpu_info_fw()
1810 if (adev->discovery.bin) in amdgpu_device_parse_gpu_info_fw()
1815 if (adev->discovery.bin) in amdgpu_device_parse_gpu_info_fw()
1821 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, in amdgpu_device_parse_gpu_info_fw()
1825 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1831 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; in amdgpu_device_parse_gpu_info_fw()
1832 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); in amdgpu_device_parse_gpu_info_fw()
1834 switch (hdr->version_major) { in amdgpu_device_parse_gpu_info_fw()
1838 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1839 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
1844 if (adev->asic_type == CHIP_NAVI12) in amdgpu_device_parse_gpu_info_fw()
1847 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); in amdgpu_device_parse_gpu_info_fw()
1848 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); in amdgpu_device_parse_gpu_info_fw()
1849 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); in amdgpu_device_parse_gpu_info_fw()
1850 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); in amdgpu_device_parse_gpu_info_fw()
1851 adev->gfx.config.max_texture_channel_caches = in amdgpu_device_parse_gpu_info_fw()
1852 le32_to_cpu(gpu_info_fw->gc_num_tccs); in amdgpu_device_parse_gpu_info_fw()
1853 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); in amdgpu_device_parse_gpu_info_fw()
1854 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); in amdgpu_device_parse_gpu_info_fw()
1855 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); in amdgpu_device_parse_gpu_info_fw()
1856 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); in amdgpu_device_parse_gpu_info_fw()
1857 adev->gfx.config.double_offchip_lds_buf = in amdgpu_device_parse_gpu_info_fw()
1858 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); in amdgpu_device_parse_gpu_info_fw()
1859 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); in amdgpu_device_parse_gpu_info_fw()
1860 adev->gfx.cu_info.max_waves_per_simd = in amdgpu_device_parse_gpu_info_fw()
1861 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); in amdgpu_device_parse_gpu_info_fw()
1862 adev->gfx.cu_info.max_scratch_slots_per_cu = in amdgpu_device_parse_gpu_info_fw()
1863 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); in amdgpu_device_parse_gpu_info_fw()
1864 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); in amdgpu_device_parse_gpu_info_fw()
1865 if (hdr->version_minor >= 1) { in amdgpu_device_parse_gpu_info_fw()
1867 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1868 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
1869 adev->gfx.config.num_sc_per_sh = in amdgpu_device_parse_gpu_info_fw()
1870 le32_to_cpu(gpu_info_fw->num_sc_per_sh); in amdgpu_device_parse_gpu_info_fw()
1871 adev->gfx.config.num_packer_per_sc = in amdgpu_device_parse_gpu_info_fw()
1872 le32_to_cpu(gpu_info_fw->num_packer_per_sc); in amdgpu_device_parse_gpu_info_fw()
1880 if (hdr->version_minor == 2) { in amdgpu_device_parse_gpu_info_fw()
1882 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + in amdgpu_device_parse_gpu_info_fw()
1883 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in amdgpu_device_parse_gpu_info_fw()
1884 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; in amdgpu_device_parse_gpu_info_fw()
1889 dev_err(adev->dev, in amdgpu_device_parse_gpu_info_fw()
1890 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); in amdgpu_device_parse_gpu_info_fw()
1891 err = -EINVAL; in amdgpu_device_parse_gpu_info_fw()
1901 adev->uid_info = kzalloc_obj(struct amdgpu_uid); in amdgpu_uid_init()
1902 if (!adev->uid_info) { in amdgpu_uid_init()
1903 dev_warn(adev->dev, "Failed to allocate memory for UID\n"); in amdgpu_uid_init()
1906 adev->uid_info->adev = adev; in amdgpu_uid_init()
1912 kfree(adev->uid_info); in amdgpu_uid_fini()
1913 adev->uid_info = NULL; in amdgpu_uid_fini()
1917 * amdgpu_device_ip_early_init - run early init for hardware IPs
1946 switch (adev->asic_type) { in amdgpu_device_ip_early_init()
1953 adev->family = AMDGPU_FAMILY_SI; in amdgpu_device_ip_early_init()
1965 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
1966 adev->family = AMDGPU_FAMILY_KV; in amdgpu_device_ip_early_init()
1968 adev->family = AMDGPU_FAMILY_CI; in amdgpu_device_ip_early_init()
1984 if (adev->flags & AMD_IS_APU) in amdgpu_device_ip_early_init()
1985 adev->family = AMDGPU_FAMILY_CZ; in amdgpu_device_ip_early_init()
1987 adev->family = AMDGPU_FAMILY_VI; in amdgpu_device_ip_early_init()
1996 adev->num_ip_blocks = 0; in amdgpu_device_ip_early_init()
2005 dev_err(adev->dev, "Unsupported A0 hardware\n"); in amdgpu_device_ip_early_init()
2006 return -ENODEV; /* device unsupported - no device error */ in amdgpu_device_ip_early_init()
2012 ((adev->flags & AMD_IS_APU) == 0) && in amdgpu_device_ip_early_init()
2013 !dev_is_removable(&adev->pdev->dev)) in amdgpu_device_ip_early_init()
2014 adev->flags |= AMD_IS_PX; in amdgpu_device_ip_early_init()
2016 if (!(adev->flags & AMD_IS_APU)) { in amdgpu_device_ip_early_init()
2017 parent = pcie_find_root_port(adev->pdev); in amdgpu_device_ip_early_init()
2018 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; in amdgpu_device_ip_early_init()
2021 adev->pm.pp_feature = amdgpu_pp_feature_mask; in amdgpu_device_ip_early_init()
2023 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; in amdgpu_device_ip_early_init()
2024 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) in amdgpu_device_ip_early_init()
2025 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; in amdgpu_device_ip_early_init()
2027 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; in amdgpu_device_ip_early_init()
2029 adev->virt.is_xgmi_node_migrate_enabled = false; in amdgpu_device_ip_early_init()
2031 adev->virt.is_xgmi_node_migrate_enabled = in amdgpu_device_ip_early_init()
2036 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_early_init()
2037 ip_block = &adev->ip_blocks[i]; in amdgpu_device_ip_early_init()
2040 dev_warn(adev->dev, "disabled ip block: %d <%s>\n", i, in amdgpu_device_ip_early_init()
2041 adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_early_init()
2042 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2043 } else if (ip_block->version->funcs->early_init) { in amdgpu_device_ip_early_init()
2044 r = ip_block->version->funcs->early_init(ip_block); in amdgpu_device_ip_early_init()
2045 if (r == -ENOENT) { in amdgpu_device_ip_early_init()
2046 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_early_init()
2048 dev_err(adev->dev, in amdgpu_device_ip_early_init()
2050 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_ip_early_init()
2054 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2057 adev->ip_blocks[i].status.valid = true; in amdgpu_device_ip_early_init()
2060 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_early_init()
2072 return -EINVAL; in amdgpu_device_ip_early_init()
2074 if (optional && !adev->bios) in amdgpu_device_ip_early_init()
2076 adev->dev, in amdgpu_device_ip_early_init()
2079 if (adev->bios) { in amdgpu_device_ip_early_init()
2082 dev_err(adev->dev, in amdgpu_device_ip_early_init()
2100 return -ENODEV; in amdgpu_device_ip_early_init()
2102 if (adev->gmc.xgmi.supported) in amdgpu_device_ip_early_init()
2108 if (ip_block->status.valid != false) in amdgpu_device_ip_early_init()
2111 adev->cg_flags &= amdgpu_cg_mask; in amdgpu_device_ip_early_init()
2112 adev->pg_flags &= amdgpu_pg_mask; in amdgpu_device_ip_early_init()
2121 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase1()
2122 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase1()
2124 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase1()
2127 adev, adev->ip_blocks[i].version->type)) in amdgpu_device_ip_hw_init_phase1()
2129 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_hw_init_phase1()
2130 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || in amdgpu_device_ip_hw_init_phase1()
2131 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { in amdgpu_device_ip_hw_init_phase1()
2132 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_hw_init_phase1()
2134 dev_err(adev->dev, in amdgpu_device_ip_hw_init_phase1()
2136 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_ip_hw_init_phase1()
2140 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase1()
2151 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_hw_init_phase2()
2152 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_hw_init_phase2()
2154 if (adev->ip_blocks[i].status.hw) in amdgpu_device_ip_hw_init_phase2()
2157 adev, adev->ip_blocks[i].version->type)) in amdgpu_device_ip_hw_init_phase2()
2159 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_hw_init_phase2()
2161 dev_err(adev->dev, in amdgpu_device_ip_hw_init_phase2()
2163 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_hw_init_phase2()
2166 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_hw_init_phase2()
2178 if (adev->asic_type >= CHIP_VEGA10) { in amdgpu_device_fw_loading()
2179 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_fw_loading()
2180 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_fw_loading()
2187 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_fw_loading()
2191 if (adev->ip_blocks[i].status.hw == true) in amdgpu_device_fw_loading()
2194 if (amdgpu_in_reset(adev) || adev->in_suspend) { in amdgpu_device_fw_loading()
2195 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); in amdgpu_device_fw_loading()
2199 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_fw_loading()
2201 dev_err(adev->dev, in amdgpu_device_fw_loading()
2203 adev->ip_blocks[i] in amdgpu_device_fw_loading()
2204 .version->funcs->name, in amdgpu_device_fw_loading()
2208 adev->ip_blocks[i].status.hw = true; in amdgpu_device_fw_loading()
2214 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) in amdgpu_device_fw_loading()
2225 .timeout_wq = adev->reset_domain->wq, in amdgpu_device_init_schedulers()
2226 .dev = adev->dev, in amdgpu_device_init_schedulers()
2232 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_init_schedulers()
2235 if (!ring || ring->no_scheduler) in amdgpu_device_init_schedulers()
2238 switch (ring->funcs->type) { in amdgpu_device_init_schedulers()
2240 timeout = adev->gfx_timeout; in amdgpu_device_init_schedulers()
2243 timeout = adev->compute_timeout; in amdgpu_device_init_schedulers()
2246 timeout = adev->sdma_timeout; in amdgpu_device_init_schedulers()
2249 timeout = adev->video_timeout; in amdgpu_device_init_schedulers()
2254 args.credit_limit = ring->num_hw_submission; in amdgpu_device_init_schedulers()
2255 args.score = ring->sched_score; in amdgpu_device_init_schedulers()
2256 args.name = ring->name; in amdgpu_device_init_schedulers()
2258 r = drm_sched_init(&ring->sched, &args); in amdgpu_device_init_schedulers()
2260 dev_err(adev->dev, in amdgpu_device_init_schedulers()
2262 ring->name); in amdgpu_device_init_schedulers()
2267 dev_err(adev->dev, in amdgpu_device_init_schedulers()
2269 ring->name); in amdgpu_device_init_schedulers()
2274 dev_err(adev->dev, in amdgpu_device_init_schedulers()
2276 ring->name); in amdgpu_device_init_schedulers()
2281 if (adev->xcp_mgr) in amdgpu_device_init_schedulers()
2289 * amdgpu_device_ip_init - run init for hardware IPs
2308 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_init()
2309 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_init()
2311 if (adev->ip_blocks[i].version->funcs->sw_init) { in amdgpu_device_ip_init()
2312 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_init()
2314 dev_err(adev->dev, in amdgpu_device_ip_init()
2316 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_ip_init()
2321 adev->ip_blocks[i].status.sw = true; in amdgpu_device_ip_init()
2324 adev, adev->ip_blocks[i].version->type)) in amdgpu_device_ip_init()
2327 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { in amdgpu_device_ip_init()
2329 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_init()
2331 dev_err(adev->dev, "hw_init %d failed %d\n", i, in amdgpu_device_ip_init()
2335 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2336 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_init()
2344 dev_err(adev->dev, in amdgpu_device_ip_init()
2349 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_init()
2351 dev_err(adev->dev, "hw_init %d failed %d\n", i, in amdgpu_device_ip_init()
2357 dev_err(adev->dev, in amdgpu_device_ip_init()
2361 adev->ip_blocks[i].status.hw = true; in amdgpu_device_ip_init()
2364 if (adev->gfx.mcbp) { in amdgpu_device_ip_init()
2365 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, in amdgpu_device_ip_init()
2370 dev_err(adev->dev, in amdgpu_device_ip_init()
2378 dev_err(adev->dev, "allocate seq64 failed %d\n", in amdgpu_device_ip_init()
2390 dev_err(adev->dev, "IB initialization failed (%d).\n", r); in amdgpu_device_ip_init()
2426 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI); in amdgpu_device_ip_init()
2434 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_init()
2440 r = -ENOENT; in amdgpu_device_ip_init()
2444 if (!hive->reset_domain || in amdgpu_device_ip_init()
2445 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { in amdgpu_device_ip_init()
2446 r = -ENOENT; in amdgpu_device_ip_init()
2452 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_ip_init()
2453 adev->reset_domain = hive->reset_domain; in amdgpu_device_ip_init()
2466 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { in amdgpu_device_ip_init()
2480 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2490 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); in amdgpu_device_fill_reset_magic()
2494 * amdgpu_device_check_vram_lost - check if vram is valid
2505 if (memcmp(adev->gart.ptr, adev->reset_magic, in amdgpu_device_check_vram_lost()
2528 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2548 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_cg_state()
2549 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_cg_state()
2550 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_cg_state()
2552 if (!adev->ip_blocks[i].version) in amdgpu_device_set_cg_state()
2555 if (adev->in_s0ix && in amdgpu_device_set_cg_state()
2556 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_cg_state()
2557 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_cg_state()
2560 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_cg_state()
2561 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_cg_state()
2562 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_cg_state()
2563 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_cg_state()
2564 adev->ip_blocks[i].version->funcs->set_clockgating_state) { in amdgpu_device_set_cg_state()
2566 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i], in amdgpu_device_set_cg_state()
2569 dev_err(adev->dev, in amdgpu_device_set_cg_state()
2571 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_set_cg_state()
2589 for (j = 0; j < adev->num_ip_blocks; j++) { in amdgpu_device_set_pg_state()
2590 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; in amdgpu_device_set_pg_state()
2591 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_set_pg_state()
2593 if (!adev->ip_blocks[i].version) in amdgpu_device_set_pg_state()
2596 if (adev->in_s0ix && in amdgpu_device_set_pg_state()
2597 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_set_pg_state()
2598 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) in amdgpu_device_set_pg_state()
2601 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && in amdgpu_device_set_pg_state()
2602 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && in amdgpu_device_set_pg_state()
2603 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && in amdgpu_device_set_pg_state()
2604 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && in amdgpu_device_set_pg_state()
2605 adev->ip_blocks[i].version->funcs->set_powergating_state) { in amdgpu_device_set_pg_state()
2607 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i], in amdgpu_device_set_pg_state()
2610 dev_err(adev->dev, in amdgpu_device_set_pg_state()
2612 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_set_pg_state()
2639 adev = gpu_ins->adev; in amdgpu_device_enable_mgpu_fan_boost()
2640 if (!(adev->flags & AMD_IS_APU || amdgpu_sriov_multi_vf_mode(adev)) && in amdgpu_device_enable_mgpu_fan_boost()
2641 !gpu_ins->mgpu_fan_enabled) { in amdgpu_device_enable_mgpu_fan_boost()
2646 gpu_ins->mgpu_fan_enabled = 1; in amdgpu_device_enable_mgpu_fan_boost()
2657 * amdgpu_device_ip_late_init - run late init for hardware IPs
2673 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_late_init()
2674 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_late_init()
2676 if (adev->ip_blocks[i].version->funcs->late_init) { in amdgpu_device_ip_late_init()
2677 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]); in amdgpu_device_ip_late_init()
2679 dev_err(adev->dev, in amdgpu_device_ip_late_init()
2681 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_ip_late_init()
2686 adev->ip_blocks[i].status.late_initialized = true; in amdgpu_device_ip_late_init()
2691 dev_err(adev->dev, "amdgpu_ras_late_init failed %d", r); in amdgpu_device_ip_late_init()
2705 dev_err(adev->dev, "enable mgpu fan boost failed (%d).\n", r); in amdgpu_device_ip_late_init()
2709 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || in amdgpu_device_ip_late_init()
2710 adev->asic_type == CHIP_ALDEBARAN)) in amdgpu_device_ip_late_init()
2713 if (adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_device_ip_late_init()
2717 * Reset device p-state to low as this was booted with high. in amdgpu_device_ip_late_init()
2729 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_ip_late_init()
2732 if (gpu_instance->adev->flags & AMD_IS_APU) in amdgpu_device_ip_late_init()
2735 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, in amdgpu_device_ip_late_init()
2738 dev_err(adev->dev, in amdgpu_device_ip_late_init()
2754 struct amdgpu_device *adev = ip_block->adev; in amdgpu_ip_block_hw_fini()
2757 if (!ip_block->version->funcs->hw_fini) { in amdgpu_ip_block_hw_fini()
2758 dev_err(adev->dev, "hw_fini of IP block <%s> not defined\n", in amdgpu_ip_block_hw_fini()
2759 ip_block->version->funcs->name); in amdgpu_ip_block_hw_fini()
2761 r = ip_block->version->funcs->hw_fini(ip_block); in amdgpu_ip_block_hw_fini()
2764 dev_dbg(adev->dev, in amdgpu_ip_block_hw_fini()
2766 ip_block->version->funcs->name, r); in amdgpu_ip_block_hw_fini()
2770 ip_block->status.hw = false; in amdgpu_ip_block_hw_fini()
2774 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2787 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_smu_fini_early()
2788 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_smu_fini_early()
2790 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_smu_fini_early()
2791 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); in amdgpu_device_smu_fini_early()
2801 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_fini_early()
2802 if (!adev->ip_blocks[i].version) in amdgpu_device_ip_fini_early()
2804 if (!adev->ip_blocks[i].version->funcs->early_fini) in amdgpu_device_ip_fini_early()
2807 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]); in amdgpu_device_ip_fini_early()
2809 dev_dbg(adev->dev, in amdgpu_device_ip_fini_early()
2811 adev->ip_blocks[i].version->funcs->name, r); in amdgpu_device_ip_fini_early()
2822 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini_early()
2823 if (!adev->ip_blocks[i].status.hw) in amdgpu_device_ip_fini_early()
2826 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); in amdgpu_device_ip_fini_early()
2831 dev_err(adev->dev, in amdgpu_device_ip_fini_early()
2846 if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu && in amdgpu_device_ip_fini_early()
2850 dev_err(adev->dev, "asic reset on %s failed\n", __func__); in amdgpu_device_ip_fini_early()
2857 * amdgpu_device_ip_fini - run fini for hardware IPs
2873 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) in amdgpu_device_ip_fini()
2876 if (adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_ip_fini()
2881 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
2882 if (!adev->ip_blocks[i].status.sw) in amdgpu_device_ip_fini()
2885 if (!adev->ip_blocks[i].version) in amdgpu_device_ip_fini()
2887 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { in amdgpu_device_ip_fini()
2889 amdgpu_free_static_csa(&adev->virt.csa_obj); in amdgpu_device_ip_fini()
2896 if (adev->ip_blocks[i].version->funcs->sw_fini) { in amdgpu_device_ip_fini()
2897 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]); in amdgpu_device_ip_fini()
2900 dev_dbg(adev->dev, in amdgpu_device_ip_fini()
2902 adev->ip_blocks[i].version->funcs->name, in amdgpu_device_ip_fini()
2906 adev->ip_blocks[i].status.sw = false; in amdgpu_device_ip_fini()
2907 adev->ip_blocks[i].status.valid = false; in amdgpu_device_ip_fini()
2910 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_fini()
2911 if (!adev->ip_blocks[i].status.late_initialized) in amdgpu_device_ip_fini()
2913 if (!adev->ip_blocks[i].version) in amdgpu_device_ip_fini()
2915 if (adev->ip_blocks[i].version->funcs->late_fini) in amdgpu_device_ip_fini()
2916 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); in amdgpu_device_ip_fini()
2917 adev->ip_blocks[i].status.late_initialized = false; in amdgpu_device_ip_fini()
2927 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2939 dev_err(adev->dev, "ib ring test failed (%d).\n", r); in amdgpu_device_delayed_init_work_handler()
2947 WARN_ON_ONCE(adev->gfx.gfx_off_state); in amdgpu_device_delay_enable_gfx_off()
2948 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); in amdgpu_device_delay_enable_gfx_off()
2951 adev->gfx.gfx_off_state = true; in amdgpu_device_delay_enable_gfx_off()
2955 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2978 dev_warn(adev->dev, "Failed to disallow df cstate"); in amdgpu_device_ip_suspend_phase1()
2980 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase1()
2981 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase1()
2985 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase1()
2988 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); in amdgpu_device_ip_suspend_phase1()
2997 dev_err(adev->dev, in amdgpu_device_ip_suspend_phase1()
3010 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3024 if (adev->in_s0ix) in amdgpu_device_ip_suspend_phase2()
3027 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { in amdgpu_device_ip_suspend_phase2()
3028 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_suspend_phase2()
3031 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) in amdgpu_device_ip_suspend_phase2()
3035 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_suspend_phase2()
3036 adev->ip_blocks[i].status.hw = false; in amdgpu_device_ip_suspend_phase2()
3042 adev, adev->ip_blocks[i].version->type)) in amdgpu_device_ip_suspend_phase2()
3048 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3049 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX && in amdgpu_device_ip_suspend_phase2()
3051 cancel_delayed_work_sync(&adev->gfx.idle_work); in amdgpu_device_ip_suspend_phase2()
3057 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3058 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || in amdgpu_device_ip_suspend_phase2()
3059 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || in amdgpu_device_ip_suspend_phase2()
3060 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) in amdgpu_device_ip_suspend_phase2()
3064 if (adev->in_s0ix && in amdgpu_device_ip_suspend_phase2()
3067 (adev->ip_blocks[i].version->type == in amdgpu_device_ip_suspend_phase2()
3071 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. in amdgpu_device_ip_suspend_phase2()
3072 * These are in TMR, hence are expected to be reused by PSP-TOS to reload in amdgpu_device_ip_suspend_phase2()
3074 * from here based on PMFW -> PSP message during re-init sequence. in amdgpu_device_ip_suspend_phase2()
3079 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && in amdgpu_device_ip_suspend_phase2()
3080 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_suspend_phase2()
3083 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); in amdgpu_device_ip_suspend_phase2()
3089 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_suspend_phase2()
3090 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); in amdgpu_device_ip_suspend_phase2()
3092 dev_err(adev->dev, in amdgpu_device_ip_suspend_phase2()
3094 adev->mp1_state, r); in amdgpu_device_ip_suspend_phase2()
3106 dev_err(adev->dev, in amdgpu_device_ip_suspend_phase2()
3114 dev_err(adev->dev, in amdgpu_device_ip_suspend_phase2()
3122 dev_err(adev->dev, in amdgpu_device_ip_suspend_phase2()
3132 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3175 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_reinit_early_sriov()
3179 block = &adev->ip_blocks[i]; in amdgpu_device_ip_reinit_early_sriov()
3180 block->status.hw = false; in amdgpu_device_ip_reinit_early_sriov()
3184 if (block->version->type != ip_order[j] || in amdgpu_device_ip_reinit_early_sriov()
3185 !block->status.valid) in amdgpu_device_ip_reinit_early_sriov()
3188 r = block->version->funcs->hw_init(&adev->ip_blocks[i]); in amdgpu_device_ip_reinit_early_sriov()
3190 dev_err(adev->dev, "RE-INIT-early: %s failed\n", in amdgpu_device_ip_reinit_early_sriov()
3191 block->version->funcs->name); in amdgpu_device_ip_reinit_early_sriov()
3194 block->status.hw = true; in amdgpu_device_ip_reinit_early_sriov()
3224 if (block->status.valid && !block->status.hw) { in amdgpu_device_ip_reinit_late_sriov()
3225 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) { in amdgpu_device_ip_reinit_late_sriov()
3228 r = block->version->funcs->hw_init(block); in amdgpu_device_ip_reinit_late_sriov()
3232 dev_err(adev->dev, "RE-INIT-late: %s failed\n", in amdgpu_device_ip_reinit_late_sriov()
3233 block->version->funcs->name); in amdgpu_device_ip_reinit_late_sriov()
3236 block->status.hw = true; in amdgpu_device_ip_reinit_late_sriov()
3244 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3259 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase1()
3260 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase1()
3262 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase1()
3263 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase1()
3264 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase1()
3265 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { in amdgpu_device_ip_resume_phase1()
3267 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); in amdgpu_device_ip_resume_phase1()
3277 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3293 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase2()
3294 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase2()
3296 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || in amdgpu_device_ip_resume_phase2()
3297 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || in amdgpu_device_ip_resume_phase2()
3298 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || in amdgpu_device_ip_resume_phase2()
3299 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE || in amdgpu_device_ip_resume_phase2()
3300 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) in amdgpu_device_ip_resume_phase2()
3302 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); in amdgpu_device_ip_resume_phase2()
3311 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3327 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_resume_phase3()
3328 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) in amdgpu_device_ip_resume_phase3()
3330 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { in amdgpu_device_ip_resume_phase3()
3331 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); in amdgpu_device_ip_resume_phase3()
3341 * amdgpu_device_ip_resume - run resume for hardware IPs
3379 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3383 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3388 if (adev->is_atom_fw) { in amdgpu_device_detect_sriov_bios()
3390 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3393 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; in amdgpu_device_detect_sriov_bios()
3396 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) in amdgpu_device_detect_sriov_bios()
3402 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3432 &pdev->dev, in amdgpu_device_asic_has_dc_support()
3440 * amdgpu_device_has_dc_support - check if dc is supported
3448 if (adev->enable_virtual_display || in amdgpu_device_has_dc_support()
3449 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_dc_support()
3452 return amdgpu_device_asic_has_dc_support(adev->pdev, adev->asic_type); in amdgpu_device_has_dc_support()
3473 task_barrier_enter(&hive->tb); in amdgpu_device_xgmi_reset_func()
3474 adev->asic_reset_res = amdgpu_device_baco_enter(adev); in amdgpu_device_xgmi_reset_func()
3476 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3479 task_barrier_exit(&hive->tb); in amdgpu_device_xgmi_reset_func()
3480 adev->asic_reset_res = amdgpu_device_baco_exit(adev); in amdgpu_device_xgmi_reset_func()
3482 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3488 task_barrier_full(&hive->tb); in amdgpu_device_xgmi_reset_func()
3489 adev->asic_reset_res = amdgpu_asic_reset(adev); in amdgpu_device_xgmi_reset_func()
3493 if (adev->asic_reset_res) in amdgpu_device_xgmi_reset_func()
3494 dev_warn(adev->dev, in amdgpu_device_xgmi_reset_func()
3496 adev->asic_reset_res, adev_to_drm(adev)->unique); in amdgpu_device_xgmi_reset_func()
3510 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = in amdgpu_device_get_job_timeout_settings()
3511 adev->video_timeout = msecs_to_jiffies(2000); in amdgpu_device_get_job_timeout_settings()
3519 * remains intact for multi-GPU systems where this function is called in amdgpu_device_get_job_timeout_settings()
3535 dev_warn(adev->dev, "lockup timeout disabled"); in amdgpu_device_get_job_timeout_settings()
3543 adev->gfx_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3546 adev->compute_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3549 adev->sdma_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3552 adev->video_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3561 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = in amdgpu_device_get_job_timeout_settings()
3562 adev->video_timeout = timeout; in amdgpu_device_get_job_timeout_settings()
3568 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3578 domain = iommu_get_domain_for_dev(adev->dev); in amdgpu_device_check_iommu_direct_map()
3579 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) in amdgpu_device_check_iommu_direct_map()
3580 adev->ram_is_direct_mapped = true; in amdgpu_device_check_iommu_direct_map()
3585 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
3595 domain = iommu_get_domain_for_dev(adev->dev); in amdgpu_device_check_iommu_remap()
3596 if (domain && (domain->type == IOMMU_DOMAIN_DMA || in amdgpu_device_check_iommu_remap()
3597 domain->type == IOMMU_DOMAIN_DMA_FQ)) in amdgpu_device_check_iommu_remap()
3607 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3609 adev->gfx.mcbp = false; in amdgpu_device_set_mcbp()
3612 adev->gfx.mcbp = true; in amdgpu_device_set_mcbp()
3614 if (adev->gfx.mcbp) in amdgpu_device_set_mcbp()
3615 dev_info(adev->dev, "MCBP is enabled\n"); in amdgpu_device_set_mcbp()
3624 drm_err(&adev->ddev, in amdgpu_device_sys_interface_init()
3629 dev_err(adev->dev, "registering pm sysfs failed (%d).\n", r); in amdgpu_device_sys_interface_init()
3633 adev->ucode_sysfs_en = false; in amdgpu_device_sys_interface_init()
3634 dev_err(adev->dev, "Creating firmware sysfs failed (%d).\n", r); in amdgpu_device_sys_interface_init()
3636 adev->ucode_sysfs_en = true; in amdgpu_device_sys_interface_init()
3640 dev_err(adev->dev, "Could not create amdgpu device attr\n"); in amdgpu_device_sys_interface_init()
3642 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); in amdgpu_device_sys_interface_init()
3644 dev_err(adev->dev, in amdgpu_device_sys_interface_init()
3657 if (adev->pm.sysfs_initialized) in amdgpu_device_sys_interface_fini()
3659 if (adev->ucode_sysfs_en) in amdgpu_device_sys_interface_fini()
3670 * amdgpu_device_init - initialize the driver
3682 struct pci_dev *pdev = adev->pdev; in amdgpu_device_init()
3688 adev->shutdown = false; in amdgpu_device_init()
3689 adev->flags = flags; in amdgpu_device_init()
3692 adev->asic_type = amdgpu_force_asic_type; in amdgpu_device_init()
3694 adev->asic_type = flags & AMD_ASIC_MASK; in amdgpu_device_init()
3696 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; in amdgpu_device_init()
3698 adev->usec_timeout *= 10; in amdgpu_device_init()
3699 adev->gmc.gart_size = 512 * 1024 * 1024; in amdgpu_device_init()
3700 adev->accel_working = false; in amdgpu_device_init()
3701 adev->num_rings = 0; in amdgpu_device_init()
3702 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); in amdgpu_device_init()
3703 adev->mman.buffer_funcs = NULL; in amdgpu_device_init()
3704 adev->mman.buffer_funcs_ring = NULL; in amdgpu_device_init()
3705 adev->vm_manager.vm_pte_funcs = NULL; in amdgpu_device_init()
3706 adev->vm_manager.vm_pte_num_scheds = 0; in amdgpu_device_init()
3707 adev->gmc.gmc_funcs = NULL; in amdgpu_device_init()
3708 adev->harvest_ip_mask = 0x0; in amdgpu_device_init()
3709 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); in amdgpu_device_init()
3710 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in amdgpu_device_init()
3715 adev->dev, in amdgpu_device_init()
3717 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, in amdgpu_device_init()
3718 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); in amdgpu_device_init()
3723 mutex_init(&adev->firmware.mutex); in amdgpu_device_init()
3724 mutex_init(&adev->pm.mutex); in amdgpu_device_init()
3725 mutex_init(&adev->gfx.gpu_clock_mutex); in amdgpu_device_init()
3726 mutex_init(&adev->srbm_mutex); in amdgpu_device_init()
3727 mutex_init(&adev->gfx.pipe_reserve_mutex); in amdgpu_device_init()
3728 mutex_init(&adev->gfx.gfx_off_mutex); in amdgpu_device_init()
3729 mutex_init(&adev->gfx.partition_mutex); in amdgpu_device_init()
3730 mutex_init(&adev->grbm_idx_mutex); in amdgpu_device_init()
3731 mutex_init(&adev->mn_lock); in amdgpu_device_init()
3732 mutex_init(&adev->virt.vf_errors.lock); in amdgpu_device_init()
3733 hash_init(adev->mn_hash); in amdgpu_device_init()
3734 mutex_init(&adev->psp.mutex); in amdgpu_device_init()
3735 mutex_init(&adev->notifier_lock); in amdgpu_device_init()
3736 mutex_init(&adev->pm.stable_pstate_ctx_lock); in amdgpu_device_init()
3737 mutex_init(&adev->benchmark_mutex); in amdgpu_device_init()
3738 mutex_init(&adev->gfx.reset_sem_mutex); in amdgpu_device_init()
3740 mutex_init(&adev->enforce_isolation_mutex); in amdgpu_device_init()
3742 adev->isolation[i].spearhead = dma_fence_get_stub(); in amdgpu_device_init()
3743 amdgpu_sync_create(&adev->isolation[i].active); in amdgpu_device_init()
3744 amdgpu_sync_create(&adev->isolation[i].prev); in amdgpu_device_init()
3746 mutex_init(&adev->gfx.userq_sch_mutex); in amdgpu_device_init()
3747 mutex_init(&adev->gfx.workload_profile_mutex); in amdgpu_device_init()
3748 mutex_init(&adev->vcn.workload_profile_mutex); in amdgpu_device_init()
3756 spin_lock_init(&adev->mmio_idx_lock); in amdgpu_device_init()
3757 spin_lock_init(&adev->mm_stats.lock); in amdgpu_device_init()
3758 spin_lock_init(&adev->virt.rlcg_reg_lock); in amdgpu_device_init()
3759 spin_lock_init(&adev->wb.lock); in amdgpu_device_init()
3761 INIT_LIST_HEAD(&adev->reset_list); in amdgpu_device_init()
3763 INIT_LIST_HEAD(&adev->ras_list); in amdgpu_device_init()
3765 INIT_LIST_HEAD(&adev->pm.od_kobj_list); in amdgpu_device_init()
3767 xa_init_flags(&adev->userq_doorbell_xa, XA_FLAGS_LOCK_IRQ); in amdgpu_device_init()
3769 INIT_DELAYED_WORK(&adev->delayed_init_work, in amdgpu_device_init()
3771 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, in amdgpu_device_init()
3783 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, in amdgpu_device_init()
3785 adev->gfx.enforce_isolation[i].adev = adev; in amdgpu_device_init()
3786 adev->gfx.enforce_isolation[i].xcp_id = i; in amdgpu_device_init()
3789 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); in amdgpu_device_init()
3793 adev->gfx.gfx_off_req_count = 1; in amdgpu_device_init()
3794 adev->gfx.gfx_off_residency = 0; in amdgpu_device_init()
3795 adev->gfx.gfx_off_entrycount = 0; in amdgpu_device_init()
3796 adev->pm.ac_power = power_supply_is_system_supplied() > 0; in amdgpu_device_init()
3798 atomic_set(&adev->throttling_logging_enabled, 1); in amdgpu_device_init()
3801 * to avoid log flooding. "-1" is subtracted since the thermal in amdgpu_device_init()
3806 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); in amdgpu_device_init()
3808 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); in amdgpu_device_init()
3812 if (adev->asic_type >= CHIP_BONAIRE) { in amdgpu_device_init()
3813 adev->rmmio_base = pci_resource_start(adev->pdev, 5); in amdgpu_device_init()
3814 adev->rmmio_size = pci_resource_len(adev->pdev, 5); in amdgpu_device_init()
3816 adev->rmmio_base = pci_resource_start(adev->pdev, 2); in amdgpu_device_init()
3817 adev->rmmio_size = pci_resource_len(adev->pdev, 2); in amdgpu_device_init()
3821 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); in amdgpu_device_init()
3823 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); in amdgpu_device_init()
3824 if (!adev->rmmio) in amdgpu_device_init()
3825 return -ENOMEM; in amdgpu_device_init()
3827 dev_info(adev->dev, "register mmio base: 0x%08X\n", in amdgpu_device_init()
3828 (uint32_t)adev->rmmio_base); in amdgpu_device_init()
3829 dev_info(adev->dev, "register mmio size: %u\n", in amdgpu_device_init()
3830 (unsigned int)adev->rmmio_size); in amdgpu_device_init()
3837 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); in amdgpu_device_init()
3838 if (!adev->reset_domain) in amdgpu_device_init()
3839 return -ENOMEM; in amdgpu_device_init()
3848 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); in amdgpu_device_init()
3866 * No need to remove conflicting FBs for non-display class devices. in amdgpu_device_init()
3869 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA || in amdgpu_device_init()
3870 (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) { in amdgpu_device_init()
3872 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name); in amdgpu_device_init()
3885 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; in amdgpu_device_init()
3889 if (adev->gmc.xgmi.supported) { in amdgpu_device_init()
3890 if (adev->gfxhub.funcs && in amdgpu_device_init()
3891 adev->gfxhub.funcs->get_xgmi_info) { in amdgpu_device_init()
3892 r = adev->gfxhub.funcs->get_xgmi_info(adev); in amdgpu_device_init()
3898 if (adev->gmc.xgmi.connected_to_cpu) { in amdgpu_device_init()
3899 if (adev->mmhub.funcs && in amdgpu_device_init()
3900 adev->mmhub.funcs->get_xgmi_info) { in amdgpu_device_init()
3901 r = adev->mmhub.funcs->get_xgmi_info(adev); in amdgpu_device_init()
3909 if (adev->virt.fw_reserve.p_pf2vf) in amdgpu_device_init()
3910 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) in amdgpu_device_init()
3911 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == in amdgpu_device_init()
3916 } else if ((adev->flags & AMD_IS_APU && in amdgpu_device_init()
3918 (adev->gmc.xgmi.connected_to_cpu && in amdgpu_device_init()
3920 adev->have_atomics_support = true; in amdgpu_device_init()
3922 adev->have_atomics_support = in amdgpu_device_init()
3923 !pci_enable_atomic_ops_to_root(adev->pdev, in amdgpu_device_init()
3928 if (!adev->have_atomics_support) in amdgpu_device_init()
3929 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); in amdgpu_device_init()
3943 if (adev->bios) in amdgpu_device_init()
3950 if (adev->gmc.xgmi.num_physical_nodes) { in amdgpu_device_init()
3951 dev_info(adev->dev, "Pending hive reset.\n"); in amdgpu_device_init()
3965 dev_err(adev->dev, "asic reset on init failed\n"); in amdgpu_device_init()
3972 if (!adev->bios) { in amdgpu_device_init()
3973 dev_err(adev->dev, "no vBIOS found\n"); in amdgpu_device_init()
3974 r = -EINVAL; in amdgpu_device_init()
3977 dev_info(adev->dev, "GPU posting now...\n"); in amdgpu_device_init()
3980 dev_err(adev->dev, "gpu post error!\n"); in amdgpu_device_init()
3985 if (adev->bios) { in amdgpu_device_init()
3986 if (adev->is_atom_fw) { in amdgpu_device_init()
3990 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); in amdgpu_device_init()
3998 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); in amdgpu_device_init()
4011 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); in amdgpu_device_init()
4021 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); in amdgpu_device_init()
4028 dev_info(adev->dev, in amdgpu_device_init()
4030 adev->gfx.config.max_shader_engines, in amdgpu_device_init()
4031 adev->gfx.config.max_sh_per_se, in amdgpu_device_init()
4032 adev->gfx.config.max_cu_per_sh, in amdgpu_device_init()
4033 adev->gfx.cu_info.number); in amdgpu_device_init()
4035 adev->accel_working = true; in amdgpu_device_init()
4045 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); in amdgpu_device_init()
4057 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { in amdgpu_device_init()
4060 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); in amdgpu_device_init()
4066 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work, in amdgpu_device_init()
4072 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_init()
4076 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { in amdgpu_device_init()
4081 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) in amdgpu_device_init()
4094 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); in amdgpu_device_init()
4097 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_device_init()
4104 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_init()
4105 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); in amdgpu_device_init()
4109 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_init()
4111 vga_switcheroo_register_client(adev->pdev, in amdgpu_device_init()
4115 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); in amdgpu_device_init()
4119 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; in amdgpu_device_init()
4120 r = register_pm_notifier(&adev->pm_nb); in amdgpu_device_init()
4135 dev_err(adev->dev, "VF exclusive mode timeout\n"); in amdgpu_device_init()
4137 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; in amdgpu_device_init()
4138 adev->virt.ops = NULL; in amdgpu_device_init()
4139 r = -EAGAIN; in amdgpu_device_init()
4153 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); in amdgpu_device_unmap_mmio()
4155 /* Unmap all mapped bars - Doorbell, registers and VRAM */ in amdgpu_device_unmap_mmio()
4158 iounmap(adev->rmmio); in amdgpu_device_unmap_mmio()
4159 adev->rmmio = NULL; in amdgpu_device_unmap_mmio()
4160 if (adev->mman.aper_base_kaddr) in amdgpu_device_unmap_mmio()
4161 iounmap(adev->mman.aper_base_kaddr); in amdgpu_device_unmap_mmio()
4162 adev->mman.aper_base_kaddr = NULL; in amdgpu_device_unmap_mmio()
4165 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { in amdgpu_device_unmap_mmio()
4166 arch_phys_wc_del(adev->gmc.vram_mtrr); in amdgpu_device_unmap_mmio()
4167 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); in amdgpu_device_unmap_mmio()
4172 * amdgpu_device_fini_hw - tear down the driver
4181 dev_info(adev->dev, "finishing device.\n"); in amdgpu_device_fini_hw()
4182 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_fini_hw()
4184 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4185 drain_workqueue(adev->mman.bdev.wq); in amdgpu_device_fini_hw()
4186 adev->shutdown = true; in amdgpu_device_fini_hw()
4188 unregister_pm_notifier(&adev->pm_nb); in amdgpu_device_fini_hw()
4203 if (adev->mode_info.mode_config_initialized) { in amdgpu_device_fini_hw()
4223 if (pci_dev_is_disconnected(adev->pdev)) in amdgpu_device_fini_hw()
4231 if (adev->mman.initialized) in amdgpu_device_fini_hw()
4232 ttm_device_clear_dma_mappings(&adev->mman.bdev); in amdgpu_device_fini_hw()
4236 if (pci_dev_is_disconnected(adev->pdev)) in amdgpu_device_fini_hw()
4248 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); in amdgpu_device_fini_sw()
4249 adev->accel_working = false; in amdgpu_device_fini_sw()
4250 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); in amdgpu_device_fini_sw()
4252 dma_fence_put(adev->isolation[i].spearhead); in amdgpu_device_fini_sw()
4253 amdgpu_sync_free(&adev->isolation[i].active); in amdgpu_device_fini_sw()
4254 amdgpu_sync_free(&adev->isolation[i].prev); in amdgpu_device_fini_sw()
4262 if (adev->bios) { in amdgpu_device_fini_sw()
4268 kfree(adev->fru_info); in amdgpu_device_fini_sw()
4269 adev->fru_info = NULL; in amdgpu_device_fini_sw()
4271 kfree(adev->xcp_mgr); in amdgpu_device_fini_sw()
4272 adev->xcp_mgr = NULL; in amdgpu_device_fini_sw()
4276 if (px || (!dev_is_removable(&adev->pdev->dev) && in amdgpu_device_fini_sw()
4278 vga_switcheroo_unregister_client(adev->pdev); in amdgpu_device_fini_sw()
4281 vga_switcheroo_fini_domain_pm_ops(adev->dev); in amdgpu_device_fini_sw()
4283 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) in amdgpu_device_fini_sw()
4284 vga_client_unregister(adev->pdev); in amdgpu_device_fini_sw()
4288 iounmap(adev->rmmio); in amdgpu_device_fini_sw()
4289 adev->rmmio = NULL; in amdgpu_device_fini_sw()
4295 if (adev->discovery.bin) in amdgpu_device_fini_sw()
4298 amdgpu_reset_put_reset_domain(adev->reset_domain); in amdgpu_device_fini_sw()
4299 adev->reset_domain = NULL; in amdgpu_device_fini_sw()
4301 kfree(adev->pci_state); in amdgpu_device_fini_sw()
4302 kfree(adev->pcie_reset_ctx.swds_pcistate); in amdgpu_device_fini_sw()
4303 kfree(adev->pcie_reset_ctx.swus_pcistate); in amdgpu_device_fini_sw()
4307 * amdgpu_device_evict_resources - evict device resources
4320 if (!adev->in_s4 && (adev->flags & AMD_IS_APU)) in amdgpu_device_evict_resources()
4329 dev_warn(adev->dev, "evicting device resources failed\n"); in amdgpu_device_evict_resources()
4333 if (adev->in_s4) { in amdgpu_device_evict_resources()
4334 ret = ttm_device_prepare_hibernation(&adev->mman.bdev); in amdgpu_device_evict_resources()
4336 dev_err(adev->dev, "prepare hibernation failed, %d\n", ret); in amdgpu_device_evict_resources()
4345 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events
4361 adev->in_s4 = true; in amdgpu_device_pm_notifier()
4364 adev->in_s4 = false; in amdgpu_device_pm_notifier()
4372 * amdgpu_device_prepare - prepare for device suspend
4385 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_prepare()
4393 flush_delayed_work(&adev->gfx.gfx_off_delay_work); in amdgpu_device_prepare()
4395 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_prepare()
4396 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_prepare()
4398 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) in amdgpu_device_prepare()
4400 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]); in amdgpu_device_prepare()
4409 * amdgpu_device_complete - complete power state transition
4421 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_complete()
4422 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_complete()
4424 if (!adev->ip_blocks[i].version->funcs->complete) in amdgpu_device_complete()
4426 adev->ip_blocks[i].version->funcs->complete(&adev->ip_blocks[i]); in amdgpu_device_complete()
4431 * amdgpu_device_suspend - initiate device suspend
4434 * @notify_clients: notify in-kernel DRM clients
4445 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_suspend()
4448 adev->in_suspend = true; in amdgpu_device_suspend()
4451 if (!adev->in_runpm) in amdgpu_device_suspend()
4466 cancel_delayed_work_sync(&adev->delayed_init_work); in amdgpu_device_suspend()
4474 amdgpu_amdkfd_suspend(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); in amdgpu_device_suspend()
4503 dev_warn(adev->dev, "failed to re-initialize user queues: %d\n", rec); in amdgpu_device_suspend()
4506 rec = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); in amdgpu_device_suspend()
4508 dev_warn(adev->dev, "failed to re-initialize kfd: %d\n", rec); in amdgpu_device_suspend()
4516 dev_warn(adev->dev, "failed to re-initialize IPs phase1: %d\n", rec); in amdgpu_device_suspend()
4523 dev_warn(adev->dev, "failed to re-update smart shift: %d\n", rec); in amdgpu_device_suspend()
4536 dev_warn(adev->dev, "failed to reinitialize sriov: %d\n", rec); in amdgpu_device_suspend()
4541 adev->in_suspend = adev->in_s0ix = adev->in_s3 = false; in amdgpu_device_suspend()
4549 unsigned int prev_physical_node_id = adev->gmc.xgmi.physical_node_id; in amdgpu_virt_resume()
4558 r = adev->gfxhub.funcs->get_xgmi_info(adev); in amdgpu_virt_resume()
4562 dev_info(adev->dev, "xgmi node, old id %d, new id %d\n", in amdgpu_virt_resume()
4563 prev_physical_node_id, adev->gmc.xgmi.physical_node_id); in amdgpu_virt_resume()
4565 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); in amdgpu_virt_resume()
4566 adev->vm_manager.vram_base_offset += in amdgpu_virt_resume()
4567 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; in amdgpu_virt_resume()
4573 * amdgpu_device_resume - initiate device resume
4576 * @notify_clients: notify in-kernel DRM clients
4599 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) in amdgpu_device_resume()
4602 if (adev->in_s0ix) in amdgpu_device_resume()
4609 dev_err(adev->dev, "amdgpu asic init failed\n"); in amdgpu_device_resume()
4615 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); in amdgpu_device_resume()
4619 r = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); in amdgpu_device_resume()
4631 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work, in amdgpu_device_resume()
4638 if (!r && !adev->in_runpm) in amdgpu_device_resume()
4646 flush_delayed_work(&adev->delayed_init_work); in amdgpu_device_resume()
4653 if (adev->mode_info.num_crtc) { in amdgpu_device_resume()
4664 dev->dev->power.disable_depth++; in amdgpu_device_resume()
4666 if (!adev->dc_enabled) in amdgpu_device_resume()
4671 dev->dev->power.disable_depth--; in amdgpu_device_resume()
4676 adev->in_suspend = false; in amdgpu_device_resume()
4679 dev_warn(adev->dev, "smart shift update failed\n"); in amdgpu_device_resume()
4685 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4705 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_check_soft_reset()
4706 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_check_soft_reset()
4708 if (adev->ip_blocks[i].version->funcs->check_soft_reset) in amdgpu_device_ip_check_soft_reset()
4709 adev->ip_blocks[i].status.hang = in amdgpu_device_ip_check_soft_reset()
4710 adev->ip_blocks[i].version->funcs->check_soft_reset( in amdgpu_device_ip_check_soft_reset()
4711 &adev->ip_blocks[i]); in amdgpu_device_ip_check_soft_reset()
4712 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_check_soft_reset()
4713 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); in amdgpu_device_ip_check_soft_reset()
4721 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4735 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_pre_soft_reset()
4736 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_pre_soft_reset()
4738 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_pre_soft_reset()
4739 adev->ip_blocks[i].version->funcs->pre_soft_reset) { in amdgpu_device_ip_pre_soft_reset()
4740 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]); in amdgpu_device_ip_pre_soft_reset()
4750 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4765 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_need_full_reset()
4766 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_need_full_reset()
4768 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || in amdgpu_device_ip_need_full_reset()
4769 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || in amdgpu_device_ip_need_full_reset()
4770 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || in amdgpu_device_ip_need_full_reset()
4771 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || in amdgpu_device_ip_need_full_reset()
4772 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { in amdgpu_device_ip_need_full_reset()
4773 if (adev->ip_blocks[i].status.hang) { in amdgpu_device_ip_need_full_reset()
4774 dev_info(adev->dev, "Some block need full reset!\n"); in amdgpu_device_ip_need_full_reset()
4783 * amdgpu_device_ip_soft_reset - do a soft reset
4797 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_soft_reset()
4798 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_soft_reset()
4800 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_soft_reset()
4801 adev->ip_blocks[i].version->funcs->soft_reset) { in amdgpu_device_ip_soft_reset()
4802 r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]); in amdgpu_device_ip_soft_reset()
4812 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4826 for (i = 0; i < adev->num_ip_blocks; i++) { in amdgpu_device_ip_post_soft_reset()
4827 if (!adev->ip_blocks[i].status.valid) in amdgpu_device_ip_post_soft_reset()
4829 if (adev->ip_blocks[i].status.hang && in amdgpu_device_ip_post_soft_reset()
4830 adev->ip_blocks[i].version->funcs->post_soft_reset) in amdgpu_device_ip_post_soft_reset()
4831 r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]); in amdgpu_device_ip_post_soft_reset()
4840 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4854 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { in amdgpu_device_reset_sriov()
4858 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); in amdgpu_device_reset_sriov()
4890 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_reset_sriov()
4901 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) in amdgpu_device_reset_sriov()
4905 * bare-metal does. in amdgpu_device_reset_sriov()
4924 * amdgpu_device_has_job_running - check if there is any unfinished job
4938 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_has_job_running()
4950 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4970 if (amdgpu_gpu_recovery == -1) { in amdgpu_device_should_recover_gpu()
4971 switch (adev->asic_type) { in amdgpu_device_should_recover_gpu()
4996 dev_info(adev->dev, "GPU recovery disabled.\n"); in amdgpu_device_should_recover_gpu()
5005 if (adev->bios) in amdgpu_device_mode1_reset()
5008 dev_info(adev->dev, "GPU mode1 reset\n"); in amdgpu_device_mode1_reset()
5011 * values are used in other cases like restore after mode-2 reset. in amdgpu_device_mode1_reset()
5013 amdgpu_device_cache_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5016 pci_clear_master(adev->pdev); in amdgpu_device_mode1_reset()
5019 dev_info(adev->dev, "GPU smu mode1 reset\n"); in amdgpu_device_mode1_reset()
5022 dev_info(adev->dev, "GPU psp mode1 reset\n"); in amdgpu_device_mode1_reset()
5030 adev->no_hw_access = false; in amdgpu_device_mode1_reset()
5035 amdgpu_device_load_pci_state(adev->pdev); in amdgpu_device_mode1_reset()
5041 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_device_mode1_reset()
5042 u32 memsize = adev->nbio.funcs->get_memsize(adev); in amdgpu_device_mode1_reset()
5049 if (i >= adev->usec_timeout) { in amdgpu_device_mode1_reset()
5050 ret = -ETIMEDOUT; in amdgpu_device_mode1_reset()
5054 if (adev->bios) in amdgpu_device_mode1_reset()
5060 dev_err(adev->dev, "GPU mode1 reset failed\n"); in amdgpu_device_mode1_reset()
5068 dev_info(adev->dev, "GPU link reset\n"); in amdgpu_device_link_reset()
5083 dev_err(adev->dev, "GPU link reset failed\n"); in amdgpu_device_link_reset()
5092 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; in amdgpu_device_pre_asic_reset()
5094 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5096 if (reset_context->reset_req_dev == adev) in amdgpu_device_pre_asic_reset()
5097 job = reset_context->job; in amdgpu_device_pre_asic_reset()
5106 struct amdgpu_ring *ring = adev->rings[i]; in amdgpu_device_pre_asic_reset()
5117 if (job && job->vm) in amdgpu_device_pre_asic_reset()
5118 drm_sched_increase_karma(&job->base); in amdgpu_device_pre_asic_reset()
5122 if (r == -EOPNOTSUPP) in amdgpu_device_pre_asic_reset()
5139 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); in amdgpu_device_pre_asic_reset()
5144 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { in amdgpu_device_pre_asic_reset()
5145 dev_info(tmp_adev->dev, "Dumping IP State\n"); in amdgpu_device_pre_asic_reset()
5147 for (i = 0; i < tmp_adev->num_ip_blocks; i++) in amdgpu_device_pre_asic_reset()
5148 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) in amdgpu_device_pre_asic_reset()
5149 tmp_adev->ip_blocks[i].version->funcs in amdgpu_device_pre_asic_reset()
5150 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]); in amdgpu_device_pre_asic_reset()
5151 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); in amdgpu_device_pre_asic_reset()
5157 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_pre_asic_reset()
5160 &reset_context->flags); in amdgpu_device_pre_asic_reset()
5173 device_list_handle = reset_context->reset_device_list; in amdgpu_device_reinit_after_reset()
5176 return -EINVAL; in amdgpu_device_reinit_after_reset()
5178 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_device_reinit_after_reset()
5184 if (reset_context->method == AMD_RESET_METHOD_ON_INIT) in amdgpu_device_reinit_after_reset()
5198 dev_warn(tmp_adev->dev, "asic atom init failed!"); in amdgpu_device_reinit_after_reset()
5200 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); in amdgpu_device_reinit_after_reset()
5208 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) in amdgpu_device_reinit_after_reset()
5209 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); in amdgpu_device_reinit_after_reset()
5213 tmp_adev->dev, in amdgpu_device_reinit_after_reset()
5223 tmp_adev->xcp_mgr); in amdgpu_device_reinit_after_reset()
5246 if (!reset_context->hive && in amdgpu_device_reinit_after_reset()
5247 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_reinit_after_reset()
5274 r = -EINVAL; in amdgpu_device_reinit_after_reset()
5279 if (reset_context->hive && in amdgpu_device_reinit_after_reset()
5280 tmp_adev->gmc.xgmi.num_physical_nodes > 1) in amdgpu_device_reinit_after_reset()
5282 reset_context->hive, tmp_adev); in amdgpu_device_reinit_after_reset()
5294 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); in amdgpu_device_reinit_after_reset()
5295 r = -EAGAIN; in amdgpu_device_reinit_after_reset()
5301 tmp_adev->asic_reset_res = r; in amdgpu_device_reinit_after_reset()
5319 reset_context->reset_device_list = device_list_handle; in amdgpu_do_asic_reset()
5322 if (r == -EOPNOTSUPP) in amdgpu_do_asic_reset()
5329 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5330 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5339 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5341 &tmp_adev->xgmi_reset_work)) in amdgpu_do_asic_reset()
5342 r = -EALREADY; in amdgpu_do_asic_reset()
5347 dev_err(tmp_adev->dev, in amdgpu_do_asic_reset()
5349 r, adev_to_drm(tmp_adev)->unique); in amdgpu_do_asic_reset()
5358 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { in amdgpu_do_asic_reset()
5359 flush_work(&tmp_adev->xgmi_reset_work); in amdgpu_do_asic_reset()
5360 r = tmp_adev->asic_reset_res; in amdgpu_do_asic_reset()
5378 if (r == -EAGAIN) in amdgpu_do_asic_reset()
5379 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5381 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); in amdgpu_do_asic_reset()
5393 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; in amdgpu_device_set_mp1_state()
5396 adev->mp1_state = PP_MP1_STATE_RESET; in amdgpu_device_set_mp1_state()
5399 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_set_mp1_state()
5407 adev->mp1_state = PP_MP1_STATE_NONE; in amdgpu_device_unset_mp1_state()
5414 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_resume_display_audio()
5415 adev->pdev->bus->number, 1); in amdgpu_device_resume_display_audio()
5417 pm_runtime_enable(&(p->dev)); in amdgpu_device_resume_display_audio()
5418 pm_runtime_resume(&(p->dev)); in amdgpu_device_resume_display_audio()
5437 return -EINVAL; in amdgpu_device_suspend_display_audio()
5439 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), in amdgpu_device_suspend_display_audio()
5440 adev->pdev->bus->number, 1); in amdgpu_device_suspend_display_audio()
5442 return -ENODEV; in amdgpu_device_suspend_display_audio()
5444 expires = pm_runtime_autosuspend_expiration(&(p->dev)); in amdgpu_device_suspend_display_audio()
5454 while (!pm_runtime_status_suspended(&(p->dev))) { in amdgpu_device_suspend_display_audio()
5455 if (!pm_runtime_suspend(&(p->dev))) in amdgpu_device_suspend_display_audio()
5459 dev_warn(adev->dev, "failed to suspend display audio\n"); in amdgpu_device_suspend_display_audio()
5462 return -ETIMEDOUT; in amdgpu_device_suspend_display_audio()
5466 pm_runtime_disable(&(p->dev)); in amdgpu_device_suspend_display_audio()
5478 cancel_work(&adev->reset_work); in amdgpu_device_stop_pending_resets()
5482 if (adev->kfd.dev) in amdgpu_device_stop_pending_resets()
5483 cancel_work(&adev->kfd.reset_work); in amdgpu_device_stop_pending_resets()
5486 cancel_work(&adev->virt.flr_work); in amdgpu_device_stop_pending_resets()
5488 if (con && adev->ras_enabled) in amdgpu_device_stop_pending_resets()
5489 cancel_work(&con->recovery_work); in amdgpu_device_stop_pending_resets()
5516 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { in amdgpu_device_recovery_prepare()
5517 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { in amdgpu_device_recovery_prepare()
5518 list_add_tail(&tmp_adev->reset_list, device_list); in amdgpu_device_recovery_prepare()
5519 if (adev->shutdown) in amdgpu_device_recovery_prepare()
5520 tmp_adev->shutdown = true; in amdgpu_device_recovery_prepare()
5522 if (!list_is_first(&adev->reset_list, device_list)) in amdgpu_device_recovery_prepare()
5523 list_rotate_to_front(&adev->reset_list, device_list); in amdgpu_device_recovery_prepare()
5525 list_add_tail(&adev->reset_list, device_list); in amdgpu_device_recovery_prepare()
5538 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_recovery_get_reset_lock()
5550 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); in amdgpu_device_recovery_put_reset_lock()
5578 tmp_adev->pcie_reset_ctx.audio_suspended = true; in amdgpu_device_halt_activities()
5582 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); in amdgpu_device_halt_activities()
5602 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_halt_activities()
5607 drm_sched_wqueue_stop(&ring->sched); in amdgpu_device_halt_activities()
5610 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); in amdgpu_device_halt_activities()
5612 atomic_inc(&tmp_adev->gpu_reset_counter); in amdgpu_device_halt_activities()
5629 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", in amdgpu_device_asic_reset()
5630 r, adev_to_drm(tmp_adev)->unique); in amdgpu_device_asic_reset()
5631 tmp_adev->asic_reset_res = r; in amdgpu_device_asic_reset()
5641 return -ENODEV; in amdgpu_device_asic_reset()
5644 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); in amdgpu_device_asic_reset()
5646 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); in amdgpu_device_asic_reset()
5650 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { in amdgpu_device_asic_reset()
5655 adev->asic_reset_res = r; in amdgpu_device_asic_reset()
5658 if (r && r == -EAGAIN) in amdgpu_device_asic_reset()
5686 struct amdgpu_ring *ring = tmp_adev->rings[i]; in amdgpu_device_sched_resume()
5691 drm_sched_wqueue_start(&ring->sched); in amdgpu_device_sched_resume()
5697 if (tmp_adev->asic_reset_res) { in amdgpu_device_sched_resume()
5702 if (reset_context->src != AMDGPU_RESET_SRC_RAS || in amdgpu_device_sched_resume()
5705 tmp_adev->dev, in amdgpu_device_sched_resume()
5708 &tmp_adev->gpu_reset_counter), in amdgpu_device_sched_resume()
5709 tmp_adev->asic_reset_res); in amdgpu_device_sched_resume()
5712 tmp_adev->asic_reset_res); in amdgpu_device_sched_resume()
5714 r = tmp_adev->asic_reset_res; in amdgpu_device_sched_resume()
5715 tmp_adev->asic_reset_res = 0; in amdgpu_device_sched_resume()
5717 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", in amdgpu_device_sched_resume()
5718 atomic_read(&tmp_adev->gpu_reset_counter)); in amdgpu_device_sched_resume()
5721 dev_warn(tmp_adev->dev, in amdgpu_device_sched_resume()
5743 if (!adev->kfd.init_complete) in amdgpu_device_gpu_resume()
5746 if (tmp_adev->pcie_reset_ctx.audio_suspended) in amdgpu_device_gpu_resume()
5758 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5765 * Attempt to do soft-reset or full-reset and reinitialize Asic
5779 int pasid = job ? job->pasid : -EINVAL; in amdgpu_device_gpu_recover()
5787 reset_context->src != AMDGPU_RESET_SRC_RAS) { in amdgpu_device_gpu_recover()
5788 dev_dbg(adev->dev, in amdgpu_device_gpu_recover()
5790 reset_context->src); in amdgpu_device_gpu_recover()
5804 amdgpu_ras_get_context(adev)->reboot) { in amdgpu_device_gpu_recover()
5805 dev_warn(adev->dev, "Emergency reboot."); in amdgpu_device_gpu_recover()
5811 dev_info(adev->dev, "GPU %s begin!. Source: %d\n", in amdgpu_device_gpu_recover()
5813 reset_context->src); in amdgpu_device_gpu_recover()
5818 mutex_lock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5820 reset_context->job = job; in amdgpu_device_gpu_recover()
5821 reset_context->hive = hive; in amdgpu_device_gpu_recover()
5841 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); in amdgpu_device_gpu_recover()
5852 * job->base holds a reference to parent fence in amdgpu_device_gpu_recover()
5854 if (job && (dma_fence_get_status(&job->hw_fence->base) > 0)) { in amdgpu_device_gpu_recover()
5856 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); in amdgpu_device_gpu_recover()
5874 mutex_unlock(&hive->hive_lock); in amdgpu_device_gpu_recover()
5879 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); in amdgpu_device_gpu_recover()
5881 atomic_set(&adev->reset_domain->reset_res, r); in amdgpu_device_gpu_recover()
5894 ti ? &ti->task : NULL); in amdgpu_device_gpu_recover()
5903 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5917 struct pci_dev *parent = adev->pdev; in amdgpu_device_partner_bandwidth()
5928 if (parent->vendor == PCI_VENDOR_ID_ATI) in amdgpu_device_partner_bandwidth()
5936 pcie_bandwidth_available(adev->pdev, NULL, speed, width); in amdgpu_device_partner_bandwidth()
5941 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU
5954 struct pci_dev *parent = adev->pdev; in amdgpu_device_gpu_bandwidth()
5960 if (parent && parent->vendor == PCI_VENDOR_ID_ATI) { in amdgpu_device_gpu_bandwidth()
5965 if (parent->vendor == PCI_VENDOR_ID_ATI) { in amdgpu_device_gpu_bandwidth()
5973 *speed = pcie_get_speed_cap(adev->pdev); in amdgpu_device_gpu_bandwidth()
5974 *width = pcie_get_width_cap(adev->pdev); in amdgpu_device_gpu_bandwidth()
5979 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5993 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; in amdgpu_device_get_pcie_info()
5996 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; in amdgpu_device_get_pcie_info()
5999 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { in amdgpu_device_get_pcie_info()
6000 if (adev->pm.pcie_gen_mask == 0) in amdgpu_device_get_pcie_info()
6001 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; in amdgpu_device_get_pcie_info()
6002 if (adev->pm.pcie_mlw_mask == 0) in amdgpu_device_get_pcie_info()
6003 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
6007 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) in amdgpu_device_get_pcie_info()
6014 if (adev->pm.pcie_gen_mask == 0) { in amdgpu_device_get_pcie_info()
6017 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6022 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6028 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6033 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6037 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6040 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
6044 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6048 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6054 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6059 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6063 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | in amdgpu_device_get_pcie_info()
6066 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; in amdgpu_device_get_pcie_info()
6070 if (adev->pm.pcie_mlw_mask == 0) { in amdgpu_device_get_pcie_info()
6073 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
6077 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
6086 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
6094 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
6101 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
6107 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
6112 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
6116 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()
6124 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; in amdgpu_device_get_pcie_info()
6128 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | in amdgpu_device_get_pcie_info()
6137 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | in amdgpu_device_get_pcie_info()
6145 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | in amdgpu_device_get_pcie_info()
6152 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | in amdgpu_device_get_pcie_info()
6158 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | in amdgpu_device_get_pcie_info()
6163 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | in amdgpu_device_get_pcie_info()
6167 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; in amdgpu_device_get_pcie_info()
6177 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6180 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6191 !adev->gmc.xgmi.connected_to_cpu && in amdgpu_device_is_peer_accessible()
6192 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); in amdgpu_device_is_peer_accessible()
6194 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n", in amdgpu_device_is_peer_accessible()
6195 pci_name(peer_adev->pdev)); in amdgpu_device_is_peer_accessible()
6197 bool is_large_bar = adev->gmc.visible_vram_size && in amdgpu_device_is_peer_accessible()
6198 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; in amdgpu_device_is_peer_accessible()
6202 uint64_t address_mask = peer_adev->dev->dma_mask ? in amdgpu_device_is_peer_accessible()
6203 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); in amdgpu_device_is_peer_accessible()
6205 adev->gmc.aper_base + adev->gmc.aper_size - 1; in amdgpu_device_is_peer_accessible()
6207 p2p_addressable = !(adev->gmc.aper_base & address_mask || in amdgpu_device_is_peer_accessible()
6221 return -ENOTSUPP; in amdgpu_device_baco_enter()
6223 if (ras && adev->ras_enabled && in amdgpu_device_baco_enter()
6224 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_enter()
6225 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); in amdgpu_device_baco_enter()
6236 return -ENOTSUPP; in amdgpu_device_baco_exit()
6242 if (ras && adev->ras_enabled && in amdgpu_device_baco_exit()
6243 adev->nbio.funcs->enable_doorbell_interrupt) in amdgpu_device_baco_exit()
6244 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); in amdgpu_device_baco_exit()
6246 if (amdgpu_passthrough(adev) && adev->nbio.funcs && in amdgpu_device_baco_exit()
6247 adev->nbio.funcs->clear_doorbell_interrupt) in amdgpu_device_baco_exit()
6248 adev->nbio.funcs->clear_doorbell_interrupt(adev); in amdgpu_device_baco_exit()
6254 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6271 dev_info(adev->dev, "PCI error: detected callback!!\n"); in amdgpu_pci_error_detected()
6273 adev->pci_channel_state = state; in amdgpu_pci_error_detected()
6277 dev_info(adev->dev, "pci_channel_io_normal: state(%d)!!\n", state); in amdgpu_pci_error_detected()
6281 dev_info(adev->dev, "pci_channel_io_frozen: state(%d)!!\n", state); in amdgpu_pci_error_detected()
6287 dev_warn(adev->dev, in amdgpu_pci_error_detected()
6292 * Non-hive devices should be able to recover after in amdgpu_pci_error_detected()
6297 mutex_lock(&hive->hive_lock); in amdgpu_pci_error_detected()
6310 mutex_unlock(&hive->hive_lock); in amdgpu_pci_error_detected()
6314 dev_info(adev->dev, "pci_channel_io_perm_failure: state(%d)!!\n", state); in amdgpu_pci_error_detected()
6322 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6330 dev_info(adev->dev, "PCI error: mmio enabled callback!!\n"); in amdgpu_pci_mmio_enabled()
6332 /* TODO - dump whatever for debugging purposes */ in amdgpu_pci_mmio_enabled()
6343 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6363 dev_info(adev->dev, "PCI error: slot reset callback!!\n"); in amdgpu_pci_slot_reset()
6369 mutex_lock(&hive->hive_lock); in amdgpu_pci_slot_reset()
6370 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) in amdgpu_pci_slot_reset()
6371 list_add_tail(&tmp_adev->reset_list, &device_list); in amdgpu_pci_slot_reset()
6373 list_add_tail(&adev->reset_list, &device_list); in amdgpu_pci_slot_reset()
6376 if (adev->pcie_reset_ctx.swus) in amdgpu_pci_slot_reset()
6377 link_dev = adev->pcie_reset_ctx.swus; in amdgpu_pci_slot_reset()
6379 link_dev = adev->pdev; in amdgpu_pci_slot_reset()
6385 timeout -= 10; in amdgpu_pci_slot_reset()
6390 r = -ETIME; in amdgpu_pci_slot_reset()
6399 for (i = 0; i < adev->usec_timeout; i++) { in amdgpu_pci_slot_reset()
6407 r = -ETIME; in amdgpu_pci_slot_reset()
6418 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) in amdgpu_pci_slot_reset()
6419 tmp_adev->pcie_reset_ctx.in_link_reset = true; in amdgpu_pci_slot_reset()
6421 adev->pcie_reset_ctx.in_link_reset = true; in amdgpu_pci_slot_reset()
6428 if (amdgpu_device_cache_pci_state(adev->pdev)) in amdgpu_pci_slot_reset()
6429 pci_restore_state(adev->pdev); in amdgpu_pci_slot_reset()
6430 dev_info(adev->dev, "PCIe error recovery succeeded\n"); in amdgpu_pci_slot_reset()
6432 dev_err(adev->dev, "PCIe error recovery failed, err:%d\n", r); in amdgpu_pci_slot_reset()
6441 mutex_unlock(&hive->hive_lock); in amdgpu_pci_slot_reset()
6449 * amdgpu_pci_resume() - resume normal ops after PCI reset
6463 dev_info(adev->dev, "PCI error: resume callback!!\n"); in amdgpu_pci_resume()
6466 if (adev->pci_channel_state != pci_channel_io_frozen) in amdgpu_pci_resume()
6473 mutex_lock(&hive->hive_lock); in amdgpu_pci_resume()
6474 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { in amdgpu_pci_resume()
6475 tmp_adev->pcie_reset_ctx.in_link_reset = false; in amdgpu_pci_resume()
6476 list_add_tail(&tmp_adev->reset_list, &device_list); in amdgpu_pci_resume()
6479 adev->pcie_reset_ctx.in_link_reset = false; in amdgpu_pci_resume()
6480 list_add_tail(&adev->reset_list, &device_list); in amdgpu_pci_resume()
6487 mutex_unlock(&hive->hive_lock); in amdgpu_pci_resume()
6497 swds = pci_upstream_bridge(adev->pdev); in amdgpu_device_cache_switch_state()
6498 if (!swds || swds->vendor != PCI_VENDOR_ID_ATI || in amdgpu_device_cache_switch_state()
6503 (swus->vendor != PCI_VENDOR_ID_ATI && in amdgpu_device_cache_switch_state()
6504 swus->vendor != PCI_VENDOR_ID_AMD) || in amdgpu_device_cache_switch_state()
6509 if (adev->pcie_reset_ctx.swus) in amdgpu_device_cache_switch_state()
6515 adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(swds); in amdgpu_device_cache_switch_state()
6520 adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(swus); in amdgpu_device_cache_switch_state()
6522 adev->pcie_reset_ctx.swus = swus; in amdgpu_device_cache_switch_state()
6530 if (!adev->pcie_reset_ctx.swds_pcistate || in amdgpu_device_load_switch_state()
6531 !adev->pcie_reset_ctx.swus_pcistate) in amdgpu_device_load_switch_state()
6534 pdev = adev->pcie_reset_ctx.swus; in amdgpu_device_load_switch_state()
6535 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate); in amdgpu_device_load_switch_state()
6539 dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r); in amdgpu_device_load_switch_state()
6543 pdev = pci_upstream_bridge(adev->pdev); in amdgpu_device_load_switch_state()
6544 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate); in amdgpu_device_load_switch_state()
6548 dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r); in amdgpu_device_load_switch_state()
6562 kfree(adev->pci_state); in amdgpu_device_cache_pci_state()
6564 adev->pci_state = pci_store_saved_state(pdev); in amdgpu_device_cache_pci_state()
6566 if (!adev->pci_state) { in amdgpu_device_cache_pci_state()
6567 dev_err(adev->dev, "Failed to store PCI saved state"); in amdgpu_device_cache_pci_state()
6571 dev_warn(adev->dev, "Failed to save PCI state, err:%d\n", r); in amdgpu_device_cache_pci_state()
6586 if (!adev->pci_state) in amdgpu_device_load_pci_state()
6589 r = pci_load_saved_state(pdev, adev->pci_state); in amdgpu_device_load_pci_state()
6594 dev_warn(adev->dev, "Failed to load PCI state, err:%d\n", r); in amdgpu_device_load_pci_state()
6605 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_flush_hdp()
6608 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_flush_hdp()
6611 if (ring && ring->funcs->emit_hdp_flush) { in amdgpu_device_flush_hdp()
6628 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) in amdgpu_device_invalidate_hdp()
6631 if (adev->gmc.xgmi.connected_to_cpu) in amdgpu_device_invalidate_hdp()
6639 return atomic_read(&adev->reset_domain->in_gpu_reset); in amdgpu_in_reset()
6643 * amdgpu_device_halt() - bring hardware to some kind of halt state
6657 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6664 struct pci_dev *pdev = adev->pdev; in amdgpu_device_halt()
6677 adev->no_hw_access = true; in amdgpu_device_halt()
6686 * amdgpu_device_get_gang - return a reference to the current gang
6696 fence = dma_fence_get_rcu_safe(&adev->gang_submit); in amdgpu_device_get_gang()
6702 * amdgpu_device_switch_gang - switch to a new gang
6727 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, in amdgpu_device_switch_gang()
6740 * amdgpu_device_enforce_isolation - enforce HW isolation
6753 struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; in amdgpu_device_enforce_isolation()
6754 struct drm_sched_fence *f = job->base.s_fence; in amdgpu_device_enforce_isolation()
6763 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX && in amdgpu_device_enforce_isolation()
6764 ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) in amdgpu_device_enforce_isolation()
6772 owner = job->enforce_isolation ? f->owner : (void *)~0l; in amdgpu_device_enforce_isolation()
6774 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_device_enforce_isolation()
6781 if (&f->scheduled != isolation->spearhead && in amdgpu_device_enforce_isolation()
6782 !dma_fence_is_signaled(isolation->spearhead)) { in amdgpu_device_enforce_isolation()
6783 dep = isolation->spearhead; in amdgpu_device_enforce_isolation()
6787 if (isolation->owner != owner) { in amdgpu_device_enforce_isolation()
6794 if (!job->gang_submit) { in amdgpu_device_enforce_isolation()
6801 dma_fence_put(isolation->spearhead); in amdgpu_device_enforce_isolation()
6802 isolation->spearhead = dma_fence_get(&f->scheduled); in amdgpu_device_enforce_isolation()
6803 amdgpu_sync_move(&isolation->active, &isolation->prev); in amdgpu_device_enforce_isolation()
6804 trace_amdgpu_isolation(isolation->owner, owner); in amdgpu_device_enforce_isolation()
6805 isolation->owner = owner; in amdgpu_device_enforce_isolation()
6814 dep = amdgpu_sync_peek_fence(&isolation->prev, ring); in amdgpu_device_enforce_isolation()
6815 r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT); in amdgpu_device_enforce_isolation()
6817 dev_warn(adev->dev, "OOM tracking isolation\n"); in amdgpu_device_enforce_isolation()
6822 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_device_enforce_isolation()
6828 switch (adev->asic_type) { in amdgpu_device_has_display_hardware()
6861 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) in amdgpu_device_has_display_hardware()
6871 if (!ring || !ring->adev) in amdgpu_get_soft_full_reset_mask()
6874 if (amdgpu_device_should_recover_gpu(ring->adev)) in amdgpu_get_soft_full_reset_mask()
6877 if (unlikely(!ring->adev->debug_disable_soft_recovery) && in amdgpu_get_soft_full_reset_mask()
6878 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) in amdgpu_get_soft_full_reset_mask()
6919 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", in amdgpu_device_set_uid()
6925 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", in amdgpu_device_set_uid()
6930 if (uid_info->uid[type][inst] != 0) { in amdgpu_device_set_uid()
6932 uid_info->adev->dev, in amdgpu_device_set_uid()
6934 uid_info->uid[type][inst], type, inst); in amdgpu_device_set_uid()
6937 uid_info->uid[type][inst] = uid; in amdgpu_device_set_uid()
6947 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", in amdgpu_device_get_uid()
6953 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", in amdgpu_device_get_uid()
6958 return uid_info->uid[type][inst]; in amdgpu_device_get_uid()