xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c (revision 1b30456150e57a79e300b82eb2efac40c25a162e)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/aperture.h>
30 #include <linux/power_supply.h>
31 #include <linux/kthread.h>
32 #include <linux/module.h>
33 #include <linux/console.h>
34 #include <linux/slab.h>
35 #include <linux/iommu.h>
36 #include <linux/pci.h>
37 #include <linux/pci-p2pdma.h>
38 #include <linux/apple-gmux.h>
39 
40 #include <drm/drm_atomic_helper.h>
41 #include <drm/drm_client_event.h>
42 #include <drm/drm_crtc_helper.h>
43 #include <drm/drm_probe_helper.h>
44 #include <drm/amdgpu_drm.h>
45 #include <linux/device.h>
46 #include <linux/vgaarb.h>
47 #include <linux/vga_switcheroo.h>
48 #include <linux/efi.h>
49 #include "amdgpu.h"
50 #include "amdgpu_trace.h"
51 #include "amdgpu_i2c.h"
52 #include "atom.h"
53 #include "amdgpu_atombios.h"
54 #include "amdgpu_atomfirmware.h"
55 #include "amd_pcie.h"
56 #ifdef CONFIG_DRM_AMDGPU_SI
57 #include "si.h"
58 #endif
59 #ifdef CONFIG_DRM_AMDGPU_CIK
60 #include "cik.h"
61 #endif
62 #include "vi.h"
63 #include "soc15.h"
64 #include "nv.h"
65 #include "bif/bif_4_1_d.h"
66 #include <linux/firmware.h>
67 #include "amdgpu_vf_error.h"
68 
69 #include "amdgpu_amdkfd.h"
70 #include "amdgpu_pm.h"
71 
72 #include "amdgpu_xgmi.h"
73 #include "amdgpu_ras.h"
74 #include "amdgpu_pmu.h"
75 #include "amdgpu_fru_eeprom.h"
76 #include "amdgpu_reset.h"
77 #include "amdgpu_virt.h"
78 #include "amdgpu_dev_coredump.h"
79 
80 #include <linux/suspend.h>
81 #include <drm/task_barrier.h>
82 #include <linux/pm_runtime.h>
83 
84 #include <drm/drm_drv.h>
85 
86 #if IS_ENABLED(CONFIG_X86)
87 #include <asm/intel-family.h>
88 #endif
89 
90 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
91 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
92 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
93 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
94 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
95 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
96 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
97 
98 #define AMDGPU_RESUME_MS		2000
99 #define AMDGPU_MAX_RETRY_LIMIT		2
100 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
101 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
102 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
103 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
104 
105 #define AMDGPU_VBIOS_SKIP (1U << 0)
106 #define AMDGPU_VBIOS_OPTIONAL (1U << 1)
107 
108 static const struct drm_driver amdgpu_kms_driver;
109 
110 const char *amdgpu_asic_name[] = {
111 	"TAHITI",
112 	"PITCAIRN",
113 	"VERDE",
114 	"OLAND",
115 	"HAINAN",
116 	"BONAIRE",
117 	"KAVERI",
118 	"KABINI",
119 	"HAWAII",
120 	"MULLINS",
121 	"TOPAZ",
122 	"TONGA",
123 	"FIJI",
124 	"CARRIZO",
125 	"STONEY",
126 	"POLARIS10",
127 	"POLARIS11",
128 	"POLARIS12",
129 	"VEGAM",
130 	"VEGA10",
131 	"VEGA12",
132 	"VEGA20",
133 	"RAVEN",
134 	"ARCTURUS",
135 	"RENOIR",
136 	"ALDEBARAN",
137 	"NAVI10",
138 	"CYAN_SKILLFISH",
139 	"NAVI14",
140 	"NAVI12",
141 	"SIENNA_CICHLID",
142 	"NAVY_FLOUNDER",
143 	"VANGOGH",
144 	"DIMGREY_CAVEFISH",
145 	"BEIGE_GOBY",
146 	"YELLOW_CARP",
147 	"IP DISCOVERY",
148 	"LAST",
149 };
150 
151 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM  - 1, 0)
152 /*
153  * Default init level where all blocks are expected to be initialized. This is
154  * the level of initialization expected by default and also after a full reset
155  * of the device.
156  */
157 struct amdgpu_init_level amdgpu_init_default = {
158 	.level = AMDGPU_INIT_LEVEL_DEFAULT,
159 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
160 };
161 
162 struct amdgpu_init_level amdgpu_init_recovery = {
163 	.level = AMDGPU_INIT_LEVEL_RESET_RECOVERY,
164 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
165 };
166 
167 /*
168  * Minimal blocks needed to be initialized before a XGMI hive can be reset. This
169  * is used for cases like reset on initialization where the entire hive needs to
170  * be reset before first use.
171  */
172 struct amdgpu_init_level amdgpu_init_minimal_xgmi = {
173 	.level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI,
174 	.hwini_ip_block_mask =
175 		BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) |
176 		BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) |
177 		BIT(AMD_IP_BLOCK_TYPE_PSP)
178 };
179 
180 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,
181 					     enum amd_ip_block_type block)
182 {
183 	return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;
184 }
185 
186 void amdgpu_set_init_level(struct amdgpu_device *adev,
187 			   enum amdgpu_init_lvl_id lvl)
188 {
189 	switch (lvl) {
190 	case AMDGPU_INIT_LEVEL_MINIMAL_XGMI:
191 		adev->init_lvl = &amdgpu_init_minimal_xgmi;
192 		break;
193 	case AMDGPU_INIT_LEVEL_RESET_RECOVERY:
194 		adev->init_lvl = &amdgpu_init_recovery;
195 		break;
196 	case AMDGPU_INIT_LEVEL_DEFAULT:
197 		fallthrough;
198 	default:
199 		adev->init_lvl = &amdgpu_init_default;
200 		break;
201 	}
202 }
203 
204 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
205 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
206 				     void *data);
207 
208 /**
209  * DOC: pcie_replay_count
210  *
211  * The amdgpu driver provides a sysfs API for reporting the total number
212  * of PCIe replays (NAKs).
213  * The file pcie_replay_count is used for this and returns the total
214  * number of replays as a sum of the NAKs generated and NAKs received.
215  */
216 
217 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
218 		struct device_attribute *attr, char *buf)
219 {
220 	struct drm_device *ddev = dev_get_drvdata(dev);
221 	struct amdgpu_device *adev = drm_to_adev(ddev);
222 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
223 
224 	return sysfs_emit(buf, "%llu\n", cnt);
225 }
226 
227 static DEVICE_ATTR(pcie_replay_count, 0444,
228 		amdgpu_device_get_pcie_replay_count, NULL);
229 
230 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
231 					  struct bin_attribute *attr, char *buf,
232 					  loff_t ppos, size_t count)
233 {
234 	struct device *dev = kobj_to_dev(kobj);
235 	struct drm_device *ddev = dev_get_drvdata(dev);
236 	struct amdgpu_device *adev = drm_to_adev(ddev);
237 	ssize_t bytes_read;
238 
239 	switch (ppos) {
240 	case AMDGPU_SYS_REG_STATE_XGMI:
241 		bytes_read = amdgpu_asic_get_reg_state(
242 			adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
243 		break;
244 	case AMDGPU_SYS_REG_STATE_WAFL:
245 		bytes_read = amdgpu_asic_get_reg_state(
246 			adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
247 		break;
248 	case AMDGPU_SYS_REG_STATE_PCIE:
249 		bytes_read = amdgpu_asic_get_reg_state(
250 			adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
251 		break;
252 	case AMDGPU_SYS_REG_STATE_USR:
253 		bytes_read = amdgpu_asic_get_reg_state(
254 			adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
255 		break;
256 	case AMDGPU_SYS_REG_STATE_USR_1:
257 		bytes_read = amdgpu_asic_get_reg_state(
258 			adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
259 		break;
260 	default:
261 		return -EINVAL;
262 	}
263 
264 	return bytes_read;
265 }
266 
267 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
268 	 AMDGPU_SYS_REG_STATE_END);
269 
270 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
271 {
272 	int ret;
273 
274 	if (!amdgpu_asic_get_reg_state_supported(adev))
275 		return 0;
276 
277 	ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
278 
279 	return ret;
280 }
281 
282 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
283 {
284 	if (!amdgpu_asic_get_reg_state_supported(adev))
285 		return;
286 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
287 }
288 
289 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
290 {
291 	int r;
292 
293 	if (ip_block->version->funcs->suspend) {
294 		r = ip_block->version->funcs->suspend(ip_block);
295 		if (r) {
296 			dev_err(ip_block->adev->dev,
297 				"suspend of IP block <%s> failed %d\n",
298 				ip_block->version->funcs->name, r);
299 			return r;
300 		}
301 	}
302 
303 	ip_block->status.hw = false;
304 	return 0;
305 }
306 
307 int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block)
308 {
309 	int r;
310 
311 	if (ip_block->version->funcs->resume) {
312 		r = ip_block->version->funcs->resume(ip_block);
313 		if (r) {
314 			dev_err(ip_block->adev->dev,
315 				"resume of IP block <%s> failed %d\n",
316 				ip_block->version->funcs->name, r);
317 			return r;
318 		}
319 	}
320 
321 	ip_block->status.hw = true;
322 	return 0;
323 }
324 
325 /**
326  * DOC: board_info
327  *
328  * The amdgpu driver provides a sysfs API for giving board related information.
329  * It provides the form factor information in the format
330  *
331  *   type : form factor
332  *
333  * Possible form factor values
334  *
335  * - "cem"		- PCIE CEM card
336  * - "oam"		- Open Compute Accelerator Module
337  * - "unknown"	- Not known
338  *
339  */
340 
341 static ssize_t amdgpu_device_get_board_info(struct device *dev,
342 					    struct device_attribute *attr,
343 					    char *buf)
344 {
345 	struct drm_device *ddev = dev_get_drvdata(dev);
346 	struct amdgpu_device *adev = drm_to_adev(ddev);
347 	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
348 	const char *pkg;
349 
350 	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
351 		pkg_type = adev->smuio.funcs->get_pkg_type(adev);
352 
353 	switch (pkg_type) {
354 	case AMDGPU_PKG_TYPE_CEM:
355 		pkg = "cem";
356 		break;
357 	case AMDGPU_PKG_TYPE_OAM:
358 		pkg = "oam";
359 		break;
360 	default:
361 		pkg = "unknown";
362 		break;
363 	}
364 
365 	return sysfs_emit(buf, "%s : %s\n", "type", pkg);
366 }
367 
368 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
369 
370 static struct attribute *amdgpu_board_attrs[] = {
371 	&dev_attr_board_info.attr,
372 	NULL,
373 };
374 
375 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
376 					     struct attribute *attr, int n)
377 {
378 	struct device *dev = kobj_to_dev(kobj);
379 	struct drm_device *ddev = dev_get_drvdata(dev);
380 	struct amdgpu_device *adev = drm_to_adev(ddev);
381 
382 	if (adev->flags & AMD_IS_APU)
383 		return 0;
384 
385 	return attr->mode;
386 }
387 
388 static const struct attribute_group amdgpu_board_attrs_group = {
389 	.attrs = amdgpu_board_attrs,
390 	.is_visible = amdgpu_board_attrs_is_visible
391 };
392 
393 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
394 
395 
396 /**
397  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
398  *
399  * @dev: drm_device pointer
400  *
401  * Returns true if the device is a dGPU with ATPX power control,
402  * otherwise return false.
403  */
404 bool amdgpu_device_supports_px(struct drm_device *dev)
405 {
406 	struct amdgpu_device *adev = drm_to_adev(dev);
407 
408 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
409 		return true;
410 	return false;
411 }
412 
413 /**
414  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
415  *
416  * @dev: drm_device pointer
417  *
418  * Returns true if the device is a dGPU with ACPI power control,
419  * otherwise return false.
420  */
421 bool amdgpu_device_supports_boco(struct drm_device *dev)
422 {
423 	struct amdgpu_device *adev = drm_to_adev(dev);
424 
425 	if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
426 		return false;
427 
428 	if (adev->has_pr3 ||
429 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
430 		return true;
431 	return false;
432 }
433 
434 /**
435  * amdgpu_device_supports_baco - Does the device support BACO
436  *
437  * @dev: drm_device pointer
438  *
439  * Return:
440  * 1 if the device supports BACO;
441  * 3 if the device supports MACO (only works if BACO is supported)
442  * otherwise return 0.
443  */
444 int amdgpu_device_supports_baco(struct drm_device *dev)
445 {
446 	struct amdgpu_device *adev = drm_to_adev(dev);
447 
448 	return amdgpu_asic_supports_baco(adev);
449 }
450 
451 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
452 {
453 	struct drm_device *dev;
454 	int bamaco_support;
455 
456 	dev = adev_to_drm(adev);
457 
458 	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
459 	bamaco_support = amdgpu_device_supports_baco(dev);
460 
461 	switch (amdgpu_runtime_pm) {
462 	case 2:
463 		if (bamaco_support & MACO_SUPPORT) {
464 			adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
465 			dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
466 		} else if (bamaco_support == BACO_SUPPORT) {
467 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
468 			dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
469 		}
470 		break;
471 	case 1:
472 		if (bamaco_support & BACO_SUPPORT) {
473 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
474 			dev_info(adev->dev, "Forcing BACO for runtime pm\n");
475 		}
476 		break;
477 	case -1:
478 	case -2:
479 		if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */
480 			adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
481 			dev_info(adev->dev, "Using ATPX for runtime pm\n");
482 		} else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */
483 			adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
484 			dev_info(adev->dev, "Using BOCO for runtime pm\n");
485 		} else {
486 			if (!bamaco_support)
487 				goto no_runtime_pm;
488 
489 			switch (adev->asic_type) {
490 			case CHIP_VEGA20:
491 			case CHIP_ARCTURUS:
492 				/* BACO are not supported on vega20 and arctrus */
493 				break;
494 			case CHIP_VEGA10:
495 				/* enable BACO as runpm mode if noretry=0 */
496 				if (!adev->gmc.noretry)
497 					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
498 				break;
499 			default:
500 				/* enable BACO as runpm mode on CI+ */
501 				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
502 				break;
503 			}
504 
505 			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
506 				if (bamaco_support & MACO_SUPPORT) {
507 					adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
508 					dev_info(adev->dev, "Using BAMACO for runtime pm\n");
509 				} else {
510 					dev_info(adev->dev, "Using BACO for runtime pm\n");
511 				}
512 			}
513 		}
514 		break;
515 	case 0:
516 		dev_info(adev->dev, "runtime pm is manually disabled\n");
517 		break;
518 	default:
519 		break;
520 	}
521 
522 no_runtime_pm:
523 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
524 		dev_info(adev->dev, "Runtime PM not available\n");
525 }
526 /**
527  * amdgpu_device_supports_smart_shift - Is the device dGPU with
528  * smart shift support
529  *
530  * @dev: drm_device pointer
531  *
532  * Returns true if the device is a dGPU with Smart Shift support,
533  * otherwise returns false.
534  */
535 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
536 {
537 	return (amdgpu_device_supports_boco(dev) &&
538 		amdgpu_acpi_is_power_shift_control_supported());
539 }
540 
541 /*
542  * VRAM access helper functions
543  */
544 
545 /**
546  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
547  *
548  * @adev: amdgpu_device pointer
549  * @pos: offset of the buffer in vram
550  * @buf: virtual address of the buffer in system memory
551  * @size: read/write size, sizeof(@buf) must > @size
552  * @write: true - write to vram, otherwise - read from vram
553  */
554 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
555 			     void *buf, size_t size, bool write)
556 {
557 	unsigned long flags;
558 	uint32_t hi = ~0, tmp = 0;
559 	uint32_t *data = buf;
560 	uint64_t last;
561 	int idx;
562 
563 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
564 		return;
565 
566 	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
567 
568 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
569 	for (last = pos + size; pos < last; pos += 4) {
570 		tmp = pos >> 31;
571 
572 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
573 		if (tmp != hi) {
574 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
575 			hi = tmp;
576 		}
577 		if (write)
578 			WREG32_NO_KIQ(mmMM_DATA, *data++);
579 		else
580 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
581 	}
582 
583 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
584 	drm_dev_exit(idx);
585 }
586 
587 /**
588  * amdgpu_device_aper_access - access vram by vram aperture
589  *
590  * @adev: amdgpu_device pointer
591  * @pos: offset of the buffer in vram
592  * @buf: virtual address of the buffer in system memory
593  * @size: read/write size, sizeof(@buf) must > @size
594  * @write: true - write to vram, otherwise - read from vram
595  *
596  * The return value means how many bytes have been transferred.
597  */
598 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
599 				 void *buf, size_t size, bool write)
600 {
601 #ifdef CONFIG_64BIT
602 	void __iomem *addr;
603 	size_t count = 0;
604 	uint64_t last;
605 
606 	if (!adev->mman.aper_base_kaddr)
607 		return 0;
608 
609 	last = min(pos + size, adev->gmc.visible_vram_size);
610 	if (last > pos) {
611 		addr = adev->mman.aper_base_kaddr + pos;
612 		count = last - pos;
613 
614 		if (write) {
615 			memcpy_toio(addr, buf, count);
616 			/* Make sure HDP write cache flush happens without any reordering
617 			 * after the system memory contents are sent over PCIe device
618 			 */
619 			mb();
620 			amdgpu_device_flush_hdp(adev, NULL);
621 		} else {
622 			amdgpu_device_invalidate_hdp(adev, NULL);
623 			/* Make sure HDP read cache is invalidated before issuing a read
624 			 * to the PCIe device
625 			 */
626 			mb();
627 			memcpy_fromio(buf, addr, count);
628 		}
629 
630 	}
631 
632 	return count;
633 #else
634 	return 0;
635 #endif
636 }
637 
638 /**
639  * amdgpu_device_vram_access - read/write a buffer in vram
640  *
641  * @adev: amdgpu_device pointer
642  * @pos: offset of the buffer in vram
643  * @buf: virtual address of the buffer in system memory
644  * @size: read/write size, sizeof(@buf) must > @size
645  * @write: true - write to vram, otherwise - read from vram
646  */
647 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
648 			       void *buf, size_t size, bool write)
649 {
650 	size_t count;
651 
652 	/* try to using vram apreature to access vram first */
653 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
654 	size -= count;
655 	if (size) {
656 		/* using MM to access rest vram */
657 		pos += count;
658 		buf += count;
659 		amdgpu_device_mm_access(adev, pos, buf, size, write);
660 	}
661 }
662 
663 /*
664  * register access helper functions.
665  */
666 
667 /* Check if hw access should be skipped because of hotplug or device error */
668 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
669 {
670 	if (adev->no_hw_access)
671 		return true;
672 
673 #ifdef CONFIG_LOCKDEP
674 	/*
675 	 * This is a bit complicated to understand, so worth a comment. What we assert
676 	 * here is that the GPU reset is not running on another thread in parallel.
677 	 *
678 	 * For this we trylock the read side of the reset semaphore, if that succeeds
679 	 * we know that the reset is not running in parallel.
680 	 *
681 	 * If the trylock fails we assert that we are either already holding the read
682 	 * side of the lock or are the reset thread itself and hold the write side of
683 	 * the lock.
684 	 */
685 	if (in_task()) {
686 		if (down_read_trylock(&adev->reset_domain->sem))
687 			up_read(&adev->reset_domain->sem);
688 		else
689 			lockdep_assert_held(&adev->reset_domain->sem);
690 	}
691 #endif
692 	return false;
693 }
694 
695 /**
696  * amdgpu_device_rreg - read a memory mapped IO or indirect register
697  *
698  * @adev: amdgpu_device pointer
699  * @reg: dword aligned register offset
700  * @acc_flags: access flags which require special behavior
701  *
702  * Returns the 32 bit value from the offset specified.
703  */
704 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
705 			    uint32_t reg, uint32_t acc_flags)
706 {
707 	uint32_t ret;
708 
709 	if (amdgpu_device_skip_hw_access(adev))
710 		return 0;
711 
712 	if ((reg * 4) < adev->rmmio_size) {
713 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
714 		    amdgpu_sriov_runtime(adev) &&
715 		    down_read_trylock(&adev->reset_domain->sem)) {
716 			ret = amdgpu_kiq_rreg(adev, reg, 0);
717 			up_read(&adev->reset_domain->sem);
718 		} else {
719 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
720 		}
721 	} else {
722 		ret = adev->pcie_rreg(adev, reg * 4);
723 	}
724 
725 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
726 
727 	return ret;
728 }
729 
730 /*
731  * MMIO register read with bytes helper functions
732  * @offset:bytes offset from MMIO start
733  */
734 
735 /**
736  * amdgpu_mm_rreg8 - read a memory mapped IO register
737  *
738  * @adev: amdgpu_device pointer
739  * @offset: byte aligned register offset
740  *
741  * Returns the 8 bit value from the offset specified.
742  */
743 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
744 {
745 	if (amdgpu_device_skip_hw_access(adev))
746 		return 0;
747 
748 	if (offset < adev->rmmio_size)
749 		return (readb(adev->rmmio + offset));
750 	BUG();
751 }
752 
753 
754 /**
755  * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
756  *
757  * @adev: amdgpu_device pointer
758  * @reg: dword aligned register offset
759  * @acc_flags: access flags which require special behavior
760  * @xcc_id: xcc accelerated compute core id
761  *
762  * Returns the 32 bit value from the offset specified.
763  */
764 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
765 				uint32_t reg, uint32_t acc_flags,
766 				uint32_t xcc_id)
767 {
768 	uint32_t ret, rlcg_flag;
769 
770 	if (amdgpu_device_skip_hw_access(adev))
771 		return 0;
772 
773 	if ((reg * 4) < adev->rmmio_size) {
774 		if (amdgpu_sriov_vf(adev) &&
775 		    !amdgpu_sriov_runtime(adev) &&
776 		    adev->gfx.rlc.rlcg_reg_access_supported &&
777 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
778 							 GC_HWIP, false,
779 							 &rlcg_flag)) {
780 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
781 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
782 		    amdgpu_sriov_runtime(adev) &&
783 		    down_read_trylock(&adev->reset_domain->sem)) {
784 			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
785 			up_read(&adev->reset_domain->sem);
786 		} else {
787 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
788 		}
789 	} else {
790 		ret = adev->pcie_rreg(adev, reg * 4);
791 	}
792 
793 	return ret;
794 }
795 
796 /*
797  * MMIO register write with bytes helper functions
798  * @offset:bytes offset from MMIO start
799  * @value: the value want to be written to the register
800  */
801 
802 /**
803  * amdgpu_mm_wreg8 - read a memory mapped IO register
804  *
805  * @adev: amdgpu_device pointer
806  * @offset: byte aligned register offset
807  * @value: 8 bit value to write
808  *
809  * Writes the value specified to the offset specified.
810  */
811 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
812 {
813 	if (amdgpu_device_skip_hw_access(adev))
814 		return;
815 
816 	if (offset < adev->rmmio_size)
817 		writeb(value, adev->rmmio + offset);
818 	else
819 		BUG();
820 }
821 
822 /**
823  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
824  *
825  * @adev: amdgpu_device pointer
826  * @reg: dword aligned register offset
827  * @v: 32 bit value to write to the register
828  * @acc_flags: access flags which require special behavior
829  *
830  * Writes the value specified to the offset specified.
831  */
832 void amdgpu_device_wreg(struct amdgpu_device *adev,
833 			uint32_t reg, uint32_t v,
834 			uint32_t acc_flags)
835 {
836 	if (amdgpu_device_skip_hw_access(adev))
837 		return;
838 
839 	if ((reg * 4) < adev->rmmio_size) {
840 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
841 		    amdgpu_sriov_runtime(adev) &&
842 		    down_read_trylock(&adev->reset_domain->sem)) {
843 			amdgpu_kiq_wreg(adev, reg, v, 0);
844 			up_read(&adev->reset_domain->sem);
845 		} else {
846 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
847 		}
848 	} else {
849 		adev->pcie_wreg(adev, reg * 4, v);
850 	}
851 
852 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
853 }
854 
855 /**
856  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
857  *
858  * @adev: amdgpu_device pointer
859  * @reg: mmio/rlc register
860  * @v: value to write
861  * @xcc_id: xcc accelerated compute core id
862  *
863  * this function is invoked only for the debugfs register access
864  */
865 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
866 			     uint32_t reg, uint32_t v,
867 			     uint32_t xcc_id)
868 {
869 	if (amdgpu_device_skip_hw_access(adev))
870 		return;
871 
872 	if (amdgpu_sriov_fullaccess(adev) &&
873 	    adev->gfx.rlc.funcs &&
874 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
875 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
876 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
877 	} else if ((reg * 4) >= adev->rmmio_size) {
878 		adev->pcie_wreg(adev, reg * 4, v);
879 	} else {
880 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
881 	}
882 }
883 
884 /**
885  * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
886  *
887  * @adev: amdgpu_device pointer
888  * @reg: dword aligned register offset
889  * @v: 32 bit value to write to the register
890  * @acc_flags: access flags which require special behavior
891  * @xcc_id: xcc accelerated compute core id
892  *
893  * Writes the value specified to the offset specified.
894  */
895 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
896 			uint32_t reg, uint32_t v,
897 			uint32_t acc_flags, uint32_t xcc_id)
898 {
899 	uint32_t rlcg_flag;
900 
901 	if (amdgpu_device_skip_hw_access(adev))
902 		return;
903 
904 	if ((reg * 4) < adev->rmmio_size) {
905 		if (amdgpu_sriov_vf(adev) &&
906 		    !amdgpu_sriov_runtime(adev) &&
907 		    adev->gfx.rlc.rlcg_reg_access_supported &&
908 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
909 							 GC_HWIP, true,
910 							 &rlcg_flag)) {
911 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
912 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
913 		    amdgpu_sriov_runtime(adev) &&
914 		    down_read_trylock(&adev->reset_domain->sem)) {
915 			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
916 			up_read(&adev->reset_domain->sem);
917 		} else {
918 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
919 		}
920 	} else {
921 		adev->pcie_wreg(adev, reg * 4, v);
922 	}
923 }
924 
925 /**
926  * amdgpu_device_indirect_rreg - read an indirect register
927  *
928  * @adev: amdgpu_device pointer
929  * @reg_addr: indirect register address to read from
930  *
931  * Returns the value of indirect register @reg_addr
932  */
933 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
934 				u32 reg_addr)
935 {
936 	unsigned long flags, pcie_index, pcie_data;
937 	void __iomem *pcie_index_offset;
938 	void __iomem *pcie_data_offset;
939 	u32 r;
940 
941 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
942 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
943 
944 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
945 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
946 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
947 
948 	writel(reg_addr, pcie_index_offset);
949 	readl(pcie_index_offset);
950 	r = readl(pcie_data_offset);
951 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
952 
953 	return r;
954 }
955 
956 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
957 				    u64 reg_addr)
958 {
959 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
960 	u32 r;
961 	void __iomem *pcie_index_offset;
962 	void __iomem *pcie_index_hi_offset;
963 	void __iomem *pcie_data_offset;
964 
965 	if (unlikely(!adev->nbio.funcs)) {
966 		pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
967 		pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
968 	} else {
969 		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
970 		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
971 	}
972 
973 	if (reg_addr >> 32) {
974 		if (unlikely(!adev->nbio.funcs))
975 			pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
976 		else
977 			pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
978 	} else {
979 		pcie_index_hi = 0;
980 	}
981 
982 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
983 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
984 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
985 	if (pcie_index_hi != 0)
986 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
987 				pcie_index_hi * 4;
988 
989 	writel(reg_addr, pcie_index_offset);
990 	readl(pcie_index_offset);
991 	if (pcie_index_hi != 0) {
992 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
993 		readl(pcie_index_hi_offset);
994 	}
995 	r = readl(pcie_data_offset);
996 
997 	/* clear the high bits */
998 	if (pcie_index_hi != 0) {
999 		writel(0, pcie_index_hi_offset);
1000 		readl(pcie_index_hi_offset);
1001 	}
1002 
1003 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1004 
1005 	return r;
1006 }
1007 
1008 /**
1009  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
1010  *
1011  * @adev: amdgpu_device pointer
1012  * @reg_addr: indirect register address to read from
1013  *
1014  * Returns the value of indirect register @reg_addr
1015  */
1016 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
1017 				  u32 reg_addr)
1018 {
1019 	unsigned long flags, pcie_index, pcie_data;
1020 	void __iomem *pcie_index_offset;
1021 	void __iomem *pcie_data_offset;
1022 	u64 r;
1023 
1024 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1025 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1026 
1027 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1028 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1029 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1030 
1031 	/* read low 32 bits */
1032 	writel(reg_addr, pcie_index_offset);
1033 	readl(pcie_index_offset);
1034 	r = readl(pcie_data_offset);
1035 	/* read high 32 bits */
1036 	writel(reg_addr + 4, pcie_index_offset);
1037 	readl(pcie_index_offset);
1038 	r |= ((u64)readl(pcie_data_offset) << 32);
1039 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1040 
1041 	return r;
1042 }
1043 
1044 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
1045 				  u64 reg_addr)
1046 {
1047 	unsigned long flags, pcie_index, pcie_data;
1048 	unsigned long pcie_index_hi = 0;
1049 	void __iomem *pcie_index_offset;
1050 	void __iomem *pcie_index_hi_offset;
1051 	void __iomem *pcie_data_offset;
1052 	u64 r;
1053 
1054 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1055 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1056 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1057 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1058 
1059 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1060 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1061 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1062 	if (pcie_index_hi != 0)
1063 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1064 			pcie_index_hi * 4;
1065 
1066 	/* read low 32 bits */
1067 	writel(reg_addr, pcie_index_offset);
1068 	readl(pcie_index_offset);
1069 	if (pcie_index_hi != 0) {
1070 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1071 		readl(pcie_index_hi_offset);
1072 	}
1073 	r = readl(pcie_data_offset);
1074 	/* read high 32 bits */
1075 	writel(reg_addr + 4, pcie_index_offset);
1076 	readl(pcie_index_offset);
1077 	if (pcie_index_hi != 0) {
1078 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1079 		readl(pcie_index_hi_offset);
1080 	}
1081 	r |= ((u64)readl(pcie_data_offset) << 32);
1082 
1083 	/* clear the high bits */
1084 	if (pcie_index_hi != 0) {
1085 		writel(0, pcie_index_hi_offset);
1086 		readl(pcie_index_hi_offset);
1087 	}
1088 
1089 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1090 
1091 	return r;
1092 }
1093 
1094 /**
1095  * amdgpu_device_indirect_wreg - write an indirect register address
1096  *
1097  * @adev: amdgpu_device pointer
1098  * @reg_addr: indirect register offset
1099  * @reg_data: indirect register data
1100  *
1101  */
1102 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
1103 				 u32 reg_addr, u32 reg_data)
1104 {
1105 	unsigned long flags, pcie_index, pcie_data;
1106 	void __iomem *pcie_index_offset;
1107 	void __iomem *pcie_data_offset;
1108 
1109 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1110 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1111 
1112 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1113 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1114 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1115 
1116 	writel(reg_addr, pcie_index_offset);
1117 	readl(pcie_index_offset);
1118 	writel(reg_data, pcie_data_offset);
1119 	readl(pcie_data_offset);
1120 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1121 }
1122 
1123 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
1124 				     u64 reg_addr, u32 reg_data)
1125 {
1126 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
1127 	void __iomem *pcie_index_offset;
1128 	void __iomem *pcie_index_hi_offset;
1129 	void __iomem *pcie_data_offset;
1130 
1131 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1132 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1133 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1134 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1135 	else
1136 		pcie_index_hi = 0;
1137 
1138 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1139 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1140 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1141 	if (pcie_index_hi != 0)
1142 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1143 				pcie_index_hi * 4;
1144 
1145 	writel(reg_addr, pcie_index_offset);
1146 	readl(pcie_index_offset);
1147 	if (pcie_index_hi != 0) {
1148 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1149 		readl(pcie_index_hi_offset);
1150 	}
1151 	writel(reg_data, pcie_data_offset);
1152 	readl(pcie_data_offset);
1153 
1154 	/* clear the high bits */
1155 	if (pcie_index_hi != 0) {
1156 		writel(0, pcie_index_hi_offset);
1157 		readl(pcie_index_hi_offset);
1158 	}
1159 
1160 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1161 }
1162 
1163 /**
1164  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
1165  *
1166  * @adev: amdgpu_device pointer
1167  * @reg_addr: indirect register offset
1168  * @reg_data: indirect register data
1169  *
1170  */
1171 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
1172 				   u32 reg_addr, u64 reg_data)
1173 {
1174 	unsigned long flags, pcie_index, pcie_data;
1175 	void __iomem *pcie_index_offset;
1176 	void __iomem *pcie_data_offset;
1177 
1178 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1179 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1180 
1181 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1182 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1183 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1184 
1185 	/* write low 32 bits */
1186 	writel(reg_addr, pcie_index_offset);
1187 	readl(pcie_index_offset);
1188 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1189 	readl(pcie_data_offset);
1190 	/* write high 32 bits */
1191 	writel(reg_addr + 4, pcie_index_offset);
1192 	readl(pcie_index_offset);
1193 	writel((u32)(reg_data >> 32), pcie_data_offset);
1194 	readl(pcie_data_offset);
1195 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1196 }
1197 
1198 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1199 				   u64 reg_addr, u64 reg_data)
1200 {
1201 	unsigned long flags, pcie_index, pcie_data;
1202 	unsigned long pcie_index_hi = 0;
1203 	void __iomem *pcie_index_offset;
1204 	void __iomem *pcie_index_hi_offset;
1205 	void __iomem *pcie_data_offset;
1206 
1207 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1208 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1209 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1210 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1211 
1212 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1213 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1214 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1215 	if (pcie_index_hi != 0)
1216 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1217 				pcie_index_hi * 4;
1218 
1219 	/* write low 32 bits */
1220 	writel(reg_addr, pcie_index_offset);
1221 	readl(pcie_index_offset);
1222 	if (pcie_index_hi != 0) {
1223 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1224 		readl(pcie_index_hi_offset);
1225 	}
1226 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1227 	readl(pcie_data_offset);
1228 	/* write high 32 bits */
1229 	writel(reg_addr + 4, pcie_index_offset);
1230 	readl(pcie_index_offset);
1231 	if (pcie_index_hi != 0) {
1232 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1233 		readl(pcie_index_hi_offset);
1234 	}
1235 	writel((u32)(reg_data >> 32), pcie_data_offset);
1236 	readl(pcie_data_offset);
1237 
1238 	/* clear the high bits */
1239 	if (pcie_index_hi != 0) {
1240 		writel(0, pcie_index_hi_offset);
1241 		readl(pcie_index_hi_offset);
1242 	}
1243 
1244 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1245 }
1246 
1247 /**
1248  * amdgpu_device_get_rev_id - query device rev_id
1249  *
1250  * @adev: amdgpu_device pointer
1251  *
1252  * Return device rev_id
1253  */
1254 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1255 {
1256 	return adev->nbio.funcs->get_rev_id(adev);
1257 }
1258 
1259 /**
1260  * amdgpu_invalid_rreg - dummy reg read function
1261  *
1262  * @adev: amdgpu_device pointer
1263  * @reg: offset of register
1264  *
1265  * Dummy register read function.  Used for register blocks
1266  * that certain asics don't have (all asics).
1267  * Returns the value in the register.
1268  */
1269 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1270 {
1271 	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1272 	BUG();
1273 	return 0;
1274 }
1275 
1276 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1277 {
1278 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1279 	BUG();
1280 	return 0;
1281 }
1282 
1283 /**
1284  * amdgpu_invalid_wreg - dummy reg write function
1285  *
1286  * @adev: amdgpu_device pointer
1287  * @reg: offset of register
1288  * @v: value to write to the register
1289  *
1290  * Dummy register read function.  Used for register blocks
1291  * that certain asics don't have (all asics).
1292  */
1293 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1294 {
1295 	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1296 		  reg, v);
1297 	BUG();
1298 }
1299 
1300 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1301 {
1302 	DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1303 		  reg, v);
1304 	BUG();
1305 }
1306 
1307 /**
1308  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1309  *
1310  * @adev: amdgpu_device pointer
1311  * @reg: offset of register
1312  *
1313  * Dummy register read function.  Used for register blocks
1314  * that certain asics don't have (all asics).
1315  * Returns the value in the register.
1316  */
1317 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1318 {
1319 	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1320 	BUG();
1321 	return 0;
1322 }
1323 
1324 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1325 {
1326 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1327 	BUG();
1328 	return 0;
1329 }
1330 
1331 /**
1332  * amdgpu_invalid_wreg64 - dummy reg write function
1333  *
1334  * @adev: amdgpu_device pointer
1335  * @reg: offset of register
1336  * @v: value to write to the register
1337  *
1338  * Dummy register read function.  Used for register blocks
1339  * that certain asics don't have (all asics).
1340  */
1341 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1342 {
1343 	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1344 		  reg, v);
1345 	BUG();
1346 }
1347 
1348 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1349 {
1350 	DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1351 		  reg, v);
1352 	BUG();
1353 }
1354 
1355 /**
1356  * amdgpu_block_invalid_rreg - dummy reg read function
1357  *
1358  * @adev: amdgpu_device pointer
1359  * @block: offset of instance
1360  * @reg: offset of register
1361  *
1362  * Dummy register read function.  Used for register blocks
1363  * that certain asics don't have (all asics).
1364  * Returns the value in the register.
1365  */
1366 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1367 					  uint32_t block, uint32_t reg)
1368 {
1369 	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1370 		  reg, block);
1371 	BUG();
1372 	return 0;
1373 }
1374 
1375 /**
1376  * amdgpu_block_invalid_wreg - dummy reg write function
1377  *
1378  * @adev: amdgpu_device pointer
1379  * @block: offset of instance
1380  * @reg: offset of register
1381  * @v: value to write to the register
1382  *
1383  * Dummy register read function.  Used for register blocks
1384  * that certain asics don't have (all asics).
1385  */
1386 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1387 				      uint32_t block,
1388 				      uint32_t reg, uint32_t v)
1389 {
1390 	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1391 		  reg, block, v);
1392 	BUG();
1393 }
1394 
1395 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)
1396 {
1397 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1398 		return AMDGPU_VBIOS_SKIP;
1399 
1400 	if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))
1401 		return AMDGPU_VBIOS_OPTIONAL;
1402 
1403 	return 0;
1404 }
1405 
1406 /**
1407  * amdgpu_device_asic_init - Wrapper for atom asic_init
1408  *
1409  * @adev: amdgpu_device pointer
1410  *
1411  * Does any asic specific work and then calls atom asic init.
1412  */
1413 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1414 {
1415 	uint32_t flags;
1416 	bool optional;
1417 	int ret;
1418 
1419 	amdgpu_asic_pre_asic_init(adev);
1420 	flags = amdgpu_device_get_vbios_flags(adev);
1421 	optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP));
1422 
1423 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1424 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
1425 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
1426 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1427 		amdgpu_psp_wait_for_bootloader(adev);
1428 		if (optional && !adev->bios)
1429 			return 0;
1430 
1431 		ret = amdgpu_atomfirmware_asic_init(adev, true);
1432 		return ret;
1433 	} else {
1434 		if (optional && !adev->bios)
1435 			return 0;
1436 
1437 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1438 	}
1439 
1440 	return 0;
1441 }
1442 
1443 /**
1444  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1445  *
1446  * @adev: amdgpu_device pointer
1447  *
1448  * Allocates a scratch page of VRAM for use by various things in the
1449  * driver.
1450  */
1451 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1452 {
1453 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1454 				       AMDGPU_GEM_DOMAIN_VRAM |
1455 				       AMDGPU_GEM_DOMAIN_GTT,
1456 				       &adev->mem_scratch.robj,
1457 				       &adev->mem_scratch.gpu_addr,
1458 				       (void **)&adev->mem_scratch.ptr);
1459 }
1460 
1461 /**
1462  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1463  *
1464  * @adev: amdgpu_device pointer
1465  *
1466  * Frees the VRAM scratch page.
1467  */
1468 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1469 {
1470 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1471 }
1472 
1473 /**
1474  * amdgpu_device_program_register_sequence - program an array of registers.
1475  *
1476  * @adev: amdgpu_device pointer
1477  * @registers: pointer to the register array
1478  * @array_size: size of the register array
1479  *
1480  * Programs an array or registers with and or masks.
1481  * This is a helper for setting golden registers.
1482  */
1483 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1484 					     const u32 *registers,
1485 					     const u32 array_size)
1486 {
1487 	u32 tmp, reg, and_mask, or_mask;
1488 	int i;
1489 
1490 	if (array_size % 3)
1491 		return;
1492 
1493 	for (i = 0; i < array_size; i += 3) {
1494 		reg = registers[i + 0];
1495 		and_mask = registers[i + 1];
1496 		or_mask = registers[i + 2];
1497 
1498 		if (and_mask == 0xffffffff) {
1499 			tmp = or_mask;
1500 		} else {
1501 			tmp = RREG32(reg);
1502 			tmp &= ~and_mask;
1503 			if (adev->family >= AMDGPU_FAMILY_AI)
1504 				tmp |= (or_mask & and_mask);
1505 			else
1506 				tmp |= or_mask;
1507 		}
1508 		WREG32(reg, tmp);
1509 	}
1510 }
1511 
1512 /**
1513  * amdgpu_device_pci_config_reset - reset the GPU
1514  *
1515  * @adev: amdgpu_device pointer
1516  *
1517  * Resets the GPU using the pci config reset sequence.
1518  * Only applicable to asics prior to vega10.
1519  */
1520 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1521 {
1522 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1523 }
1524 
1525 /**
1526  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1527  *
1528  * @adev: amdgpu_device pointer
1529  *
1530  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1531  */
1532 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1533 {
1534 	return pci_reset_function(adev->pdev);
1535 }
1536 
1537 /*
1538  * amdgpu_device_wb_*()
1539  * Writeback is the method by which the GPU updates special pages in memory
1540  * with the status of certain GPU events (fences, ring pointers,etc.).
1541  */
1542 
1543 /**
1544  * amdgpu_device_wb_fini - Disable Writeback and free memory
1545  *
1546  * @adev: amdgpu_device pointer
1547  *
1548  * Disables Writeback and frees the Writeback memory (all asics).
1549  * Used at driver shutdown.
1550  */
1551 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1552 {
1553 	if (adev->wb.wb_obj) {
1554 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1555 				      &adev->wb.gpu_addr,
1556 				      (void **)&adev->wb.wb);
1557 		adev->wb.wb_obj = NULL;
1558 	}
1559 }
1560 
1561 /**
1562  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1563  *
1564  * @adev: amdgpu_device pointer
1565  *
1566  * Initializes writeback and allocates writeback memory (all asics).
1567  * Used at driver startup.
1568  * Returns 0 on success or an -error on failure.
1569  */
1570 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1571 {
1572 	int r;
1573 
1574 	if (adev->wb.wb_obj == NULL) {
1575 		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1576 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1577 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1578 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
1579 					    (void **)&adev->wb.wb);
1580 		if (r) {
1581 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1582 			return r;
1583 		}
1584 
1585 		adev->wb.num_wb = AMDGPU_MAX_WB;
1586 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1587 
1588 		/* clear wb memory */
1589 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1590 	}
1591 
1592 	return 0;
1593 }
1594 
1595 /**
1596  * amdgpu_device_wb_get - Allocate a wb entry
1597  *
1598  * @adev: amdgpu_device pointer
1599  * @wb: wb index
1600  *
1601  * Allocate a wb slot for use by the driver (all asics).
1602  * Returns 0 on success or -EINVAL on failure.
1603  */
1604 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1605 {
1606 	unsigned long flags, offset;
1607 
1608 	spin_lock_irqsave(&adev->wb.lock, flags);
1609 	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1610 	if (offset < adev->wb.num_wb) {
1611 		__set_bit(offset, adev->wb.used);
1612 		spin_unlock_irqrestore(&adev->wb.lock, flags);
1613 		*wb = offset << 3; /* convert to dw offset */
1614 		return 0;
1615 	} else {
1616 		spin_unlock_irqrestore(&adev->wb.lock, flags);
1617 		return -EINVAL;
1618 	}
1619 }
1620 
1621 /**
1622  * amdgpu_device_wb_free - Free a wb entry
1623  *
1624  * @adev: amdgpu_device pointer
1625  * @wb: wb index
1626  *
1627  * Free a wb slot allocated for use by the driver (all asics)
1628  */
1629 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1630 {
1631 	unsigned long flags;
1632 
1633 	wb >>= 3;
1634 	spin_lock_irqsave(&adev->wb.lock, flags);
1635 	if (wb < adev->wb.num_wb)
1636 		__clear_bit(wb, adev->wb.used);
1637 	spin_unlock_irqrestore(&adev->wb.lock, flags);
1638 }
1639 
1640 /**
1641  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1642  *
1643  * @adev: amdgpu_device pointer
1644  *
1645  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1646  * to fail, but if any of the BARs is not accessible after the size we abort
1647  * driver loading by returning -ENODEV.
1648  */
1649 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1650 {
1651 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1652 	struct pci_bus *root;
1653 	struct resource *res;
1654 	unsigned int i;
1655 	u16 cmd;
1656 	int r;
1657 
1658 	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1659 		return 0;
1660 
1661 	/* Bypass for VF */
1662 	if (amdgpu_sriov_vf(adev))
1663 		return 0;
1664 
1665 	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1666 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1667 		DRM_WARN("System can't access extended configuration space, please check!!\n");
1668 
1669 	/* skip if the bios has already enabled large BAR */
1670 	if (adev->gmc.real_vram_size &&
1671 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1672 		return 0;
1673 
1674 	/* Check if the root BUS has 64bit memory resources */
1675 	root = adev->pdev->bus;
1676 	while (root->parent)
1677 		root = root->parent;
1678 
1679 	pci_bus_for_each_resource(root, res, i) {
1680 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1681 		    res->start > 0x100000000ull)
1682 			break;
1683 	}
1684 
1685 	/* Trying to resize is pointless without a root hub window above 4GB */
1686 	if (!res)
1687 		return 0;
1688 
1689 	/* Limit the BAR size to what is available */
1690 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1691 			rbar_size);
1692 
1693 	/* Disable memory decoding while we change the BAR addresses and size */
1694 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1695 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1696 			      cmd & ~PCI_COMMAND_MEMORY);
1697 
1698 	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1699 	amdgpu_doorbell_fini(adev);
1700 	if (adev->asic_type >= CHIP_BONAIRE)
1701 		pci_release_resource(adev->pdev, 2);
1702 
1703 	pci_release_resource(adev->pdev, 0);
1704 
1705 	r = pci_resize_resource(adev->pdev, 0, rbar_size);
1706 	if (r == -ENOSPC)
1707 		DRM_INFO("Not enough PCI address space for a large BAR.");
1708 	else if (r && r != -ENOTSUPP)
1709 		DRM_ERROR("Problem resizing BAR0 (%d).", r);
1710 
1711 	pci_assign_unassigned_bus_resources(adev->pdev->bus);
1712 
1713 	/* When the doorbell or fb BAR isn't available we have no chance of
1714 	 * using the device.
1715 	 */
1716 	r = amdgpu_doorbell_init(adev);
1717 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1718 		return -ENODEV;
1719 
1720 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1721 
1722 	return 0;
1723 }
1724 
1725 /*
1726  * GPU helpers function.
1727  */
1728 /**
1729  * amdgpu_device_need_post - check if the hw need post or not
1730  *
1731  * @adev: amdgpu_device pointer
1732  *
1733  * Check if the asic has been initialized (all asics) at driver startup
1734  * or post is needed if  hw reset is performed.
1735  * Returns true if need or false if not.
1736  */
1737 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1738 {
1739 	uint32_t reg, flags;
1740 
1741 	if (amdgpu_sriov_vf(adev))
1742 		return false;
1743 
1744 	flags = amdgpu_device_get_vbios_flags(adev);
1745 	if (flags & AMDGPU_VBIOS_SKIP)
1746 		return false;
1747 	if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)
1748 		return false;
1749 
1750 	if (amdgpu_passthrough(adev)) {
1751 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1752 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1753 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1754 		 * vpost executed for smc version below 22.15
1755 		 */
1756 		if (adev->asic_type == CHIP_FIJI) {
1757 			int err;
1758 			uint32_t fw_ver;
1759 
1760 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1761 			/* force vPost if error occurred */
1762 			if (err)
1763 				return true;
1764 
1765 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1766 			release_firmware(adev->pm.fw);
1767 			if (fw_ver < 0x00160e00)
1768 				return true;
1769 		}
1770 	}
1771 
1772 	/* Don't post if we need to reset whole hive on init */
1773 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
1774 		return false;
1775 
1776 	if (adev->has_hw_reset) {
1777 		adev->has_hw_reset = false;
1778 		return true;
1779 	}
1780 
1781 	/* bios scratch used on CIK+ */
1782 	if (adev->asic_type >= CHIP_BONAIRE)
1783 		return amdgpu_atombios_scratch_need_asic_init(adev);
1784 
1785 	/* check MEM_SIZE for older asics */
1786 	reg = amdgpu_asic_get_config_memsize(adev);
1787 
1788 	if ((reg != 0) && (reg != 0xffffffff))
1789 		return false;
1790 
1791 	return true;
1792 }
1793 
1794 /*
1795  * Check whether seamless boot is supported.
1796  *
1797  * So far we only support seamless boot on DCE 3.0 or later.
1798  * If users report that it works on older ASICS as well, we may
1799  * loosen this.
1800  */
1801 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1802 {
1803 	switch (amdgpu_seamless) {
1804 	case -1:
1805 		break;
1806 	case 1:
1807 		return true;
1808 	case 0:
1809 		return false;
1810 	default:
1811 		DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1812 			  amdgpu_seamless);
1813 		return false;
1814 	}
1815 
1816 	if (!(adev->flags & AMD_IS_APU))
1817 		return false;
1818 
1819 	if (adev->mman.keep_stolen_vga_memory)
1820 		return false;
1821 
1822 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1823 }
1824 
1825 /*
1826  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1827  * don't support dynamic speed switching. Until we have confirmation from Intel
1828  * that a specific host supports it, it's safer that we keep it disabled for all.
1829  *
1830  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1831  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1832  */
1833 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1834 {
1835 #if IS_ENABLED(CONFIG_X86)
1836 	struct cpuinfo_x86 *c = &cpu_data(0);
1837 
1838 	/* eGPU change speeds based on USB4 fabric conditions */
1839 	if (dev_is_removable(adev->dev))
1840 		return true;
1841 
1842 	if (c->x86_vendor == X86_VENDOR_INTEL)
1843 		return false;
1844 #endif
1845 	return true;
1846 }
1847 
1848 /**
1849  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1850  *
1851  * @adev: amdgpu_device pointer
1852  *
1853  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1854  * be set for this device.
1855  *
1856  * Returns true if it should be used or false if not.
1857  */
1858 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1859 {
1860 	switch (amdgpu_aspm) {
1861 	case -1:
1862 		break;
1863 	case 0:
1864 		return false;
1865 	case 1:
1866 		return true;
1867 	default:
1868 		return false;
1869 	}
1870 	if (adev->flags & AMD_IS_APU)
1871 		return false;
1872 	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1873 		return false;
1874 	return pcie_aspm_enabled(adev->pdev);
1875 }
1876 
1877 /* if we get transitioned to only one device, take VGA back */
1878 /**
1879  * amdgpu_device_vga_set_decode - enable/disable vga decode
1880  *
1881  * @pdev: PCI device pointer
1882  * @state: enable/disable vga decode
1883  *
1884  * Enable/disable vga decode (all asics).
1885  * Returns VGA resource flags.
1886  */
1887 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1888 		bool state)
1889 {
1890 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1891 
1892 	amdgpu_asic_set_vga_state(adev, state);
1893 	if (state)
1894 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1895 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1896 	else
1897 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1898 }
1899 
1900 /**
1901  * amdgpu_device_check_block_size - validate the vm block size
1902  *
1903  * @adev: amdgpu_device pointer
1904  *
1905  * Validates the vm block size specified via module parameter.
1906  * The vm block size defines number of bits in page table versus page directory,
1907  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1908  * page table and the remaining bits are in the page directory.
1909  */
1910 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1911 {
1912 	/* defines number of bits in page table versus page directory,
1913 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1914 	 * page table and the remaining bits are in the page directory
1915 	 */
1916 	if (amdgpu_vm_block_size == -1)
1917 		return;
1918 
1919 	if (amdgpu_vm_block_size < 9) {
1920 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1921 			 amdgpu_vm_block_size);
1922 		amdgpu_vm_block_size = -1;
1923 	}
1924 }
1925 
1926 /**
1927  * amdgpu_device_check_vm_size - validate the vm size
1928  *
1929  * @adev: amdgpu_device pointer
1930  *
1931  * Validates the vm size in GB specified via module parameter.
1932  * The VM size is the size of the GPU virtual memory space in GB.
1933  */
1934 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1935 {
1936 	/* no need to check the default value */
1937 	if (amdgpu_vm_size == -1)
1938 		return;
1939 
1940 	if (amdgpu_vm_size < 1) {
1941 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1942 			 amdgpu_vm_size);
1943 		amdgpu_vm_size = -1;
1944 	}
1945 }
1946 
1947 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1948 {
1949 	struct sysinfo si;
1950 	bool is_os_64 = (sizeof(void *) == 8);
1951 	uint64_t total_memory;
1952 	uint64_t dram_size_seven_GB = 0x1B8000000;
1953 	uint64_t dram_size_three_GB = 0xB8000000;
1954 
1955 	if (amdgpu_smu_memory_pool_size == 0)
1956 		return;
1957 
1958 	if (!is_os_64) {
1959 		DRM_WARN("Not 64-bit OS, feature not supported\n");
1960 		goto def_value;
1961 	}
1962 	si_meminfo(&si);
1963 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1964 
1965 	if ((amdgpu_smu_memory_pool_size == 1) ||
1966 		(amdgpu_smu_memory_pool_size == 2)) {
1967 		if (total_memory < dram_size_three_GB)
1968 			goto def_value1;
1969 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1970 		(amdgpu_smu_memory_pool_size == 8)) {
1971 		if (total_memory < dram_size_seven_GB)
1972 			goto def_value1;
1973 	} else {
1974 		DRM_WARN("Smu memory pool size not supported\n");
1975 		goto def_value;
1976 	}
1977 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1978 
1979 	return;
1980 
1981 def_value1:
1982 	DRM_WARN("No enough system memory\n");
1983 def_value:
1984 	adev->pm.smu_prv_buffer_size = 0;
1985 }
1986 
1987 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1988 {
1989 	if (!(adev->flags & AMD_IS_APU) ||
1990 	    adev->asic_type < CHIP_RAVEN)
1991 		return 0;
1992 
1993 	switch (adev->asic_type) {
1994 	case CHIP_RAVEN:
1995 		if (adev->pdev->device == 0x15dd)
1996 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1997 		if (adev->pdev->device == 0x15d8)
1998 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1999 		break;
2000 	case CHIP_RENOIR:
2001 		if ((adev->pdev->device == 0x1636) ||
2002 		    (adev->pdev->device == 0x164c))
2003 			adev->apu_flags |= AMD_APU_IS_RENOIR;
2004 		else
2005 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
2006 		break;
2007 	case CHIP_VANGOGH:
2008 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
2009 		break;
2010 	case CHIP_YELLOW_CARP:
2011 		break;
2012 	case CHIP_CYAN_SKILLFISH:
2013 		if ((adev->pdev->device == 0x13FE) ||
2014 		    (adev->pdev->device == 0x143F))
2015 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
2016 		break;
2017 	default:
2018 		break;
2019 	}
2020 
2021 	return 0;
2022 }
2023 
2024 /**
2025  * amdgpu_device_check_arguments - validate module params
2026  *
2027  * @adev: amdgpu_device pointer
2028  *
2029  * Validates certain module parameters and updates
2030  * the associated values used by the driver (all asics).
2031  */
2032 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
2033 {
2034 	int i;
2035 
2036 	if (amdgpu_sched_jobs < 4) {
2037 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
2038 			 amdgpu_sched_jobs);
2039 		amdgpu_sched_jobs = 4;
2040 	} else if (!is_power_of_2(amdgpu_sched_jobs)) {
2041 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
2042 			 amdgpu_sched_jobs);
2043 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
2044 	}
2045 
2046 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
2047 		/* gart size must be greater or equal to 32M */
2048 		dev_warn(adev->dev, "gart size (%d) too small\n",
2049 			 amdgpu_gart_size);
2050 		amdgpu_gart_size = -1;
2051 	}
2052 
2053 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
2054 		/* gtt size must be greater or equal to 32M */
2055 		dev_warn(adev->dev, "gtt size (%d) too small\n",
2056 				 amdgpu_gtt_size);
2057 		amdgpu_gtt_size = -1;
2058 	}
2059 
2060 	/* valid range is between 4 and 9 inclusive */
2061 	if (amdgpu_vm_fragment_size != -1 &&
2062 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
2063 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
2064 		amdgpu_vm_fragment_size = -1;
2065 	}
2066 
2067 	if (amdgpu_sched_hw_submission < 2) {
2068 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
2069 			 amdgpu_sched_hw_submission);
2070 		amdgpu_sched_hw_submission = 2;
2071 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
2072 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
2073 			 amdgpu_sched_hw_submission);
2074 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
2075 	}
2076 
2077 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
2078 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
2079 		amdgpu_reset_method = -1;
2080 	}
2081 
2082 	amdgpu_device_check_smu_prv_buffer_size(adev);
2083 
2084 	amdgpu_device_check_vm_size(adev);
2085 
2086 	amdgpu_device_check_block_size(adev);
2087 
2088 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
2089 
2090 	for (i = 0; i < MAX_XCP; i++)
2091 		adev->enforce_isolation[i] = !!enforce_isolation;
2092 
2093 	return 0;
2094 }
2095 
2096 /**
2097  * amdgpu_switcheroo_set_state - set switcheroo state
2098  *
2099  * @pdev: pci dev pointer
2100  * @state: vga_switcheroo state
2101  *
2102  * Callback for the switcheroo driver.  Suspends or resumes
2103  * the asics before or after it is powered up using ACPI methods.
2104  */
2105 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
2106 					enum vga_switcheroo_state state)
2107 {
2108 	struct drm_device *dev = pci_get_drvdata(pdev);
2109 	int r;
2110 
2111 	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
2112 		return;
2113 
2114 	if (state == VGA_SWITCHEROO_ON) {
2115 		pr_info("switched on\n");
2116 		/* don't suspend or resume card normally */
2117 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2118 
2119 		pci_set_power_state(pdev, PCI_D0);
2120 		amdgpu_device_load_pci_state(pdev);
2121 		r = pci_enable_device(pdev);
2122 		if (r)
2123 			DRM_WARN("pci_enable_device failed (%d)\n", r);
2124 		amdgpu_device_resume(dev, true);
2125 
2126 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
2127 	} else {
2128 		pr_info("switched off\n");
2129 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2130 		amdgpu_device_prepare(dev);
2131 		amdgpu_device_suspend(dev, true);
2132 		amdgpu_device_cache_pci_state(pdev);
2133 		/* Shut down the device */
2134 		pci_disable_device(pdev);
2135 		pci_set_power_state(pdev, PCI_D3cold);
2136 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
2137 	}
2138 }
2139 
2140 /**
2141  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
2142  *
2143  * @pdev: pci dev pointer
2144  *
2145  * Callback for the switcheroo driver.  Check of the switcheroo
2146  * state can be changed.
2147  * Returns true if the state can be changed, false if not.
2148  */
2149 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
2150 {
2151 	struct drm_device *dev = pci_get_drvdata(pdev);
2152 
2153        /*
2154 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
2155 	* locking inversion with the driver load path. And the access here is
2156 	* completely racy anyway. So don't bother with locking for now.
2157 	*/
2158 	return atomic_read(&dev->open_count) == 0;
2159 }
2160 
2161 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
2162 	.set_gpu_state = amdgpu_switcheroo_set_state,
2163 	.reprobe = NULL,
2164 	.can_switch = amdgpu_switcheroo_can_switch,
2165 };
2166 
2167 /**
2168  * amdgpu_device_ip_set_clockgating_state - set the CG state
2169  *
2170  * @dev: amdgpu_device pointer
2171  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2172  * @state: clockgating state (gate or ungate)
2173  *
2174  * Sets the requested clockgating state for all instances of
2175  * the hardware IP specified.
2176  * Returns the error code from the last instance.
2177  */
2178 int amdgpu_device_ip_set_clockgating_state(void *dev,
2179 					   enum amd_ip_block_type block_type,
2180 					   enum amd_clockgating_state state)
2181 {
2182 	struct amdgpu_device *adev = dev;
2183 	int i, r = 0;
2184 
2185 	for (i = 0; i < adev->num_ip_blocks; i++) {
2186 		if (!adev->ip_blocks[i].status.valid)
2187 			continue;
2188 		if (adev->ip_blocks[i].version->type != block_type)
2189 			continue;
2190 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
2191 			continue;
2192 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
2193 			&adev->ip_blocks[i], state);
2194 		if (r)
2195 			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
2196 				  adev->ip_blocks[i].version->funcs->name, r);
2197 	}
2198 	return r;
2199 }
2200 
2201 /**
2202  * amdgpu_device_ip_set_powergating_state - set the PG state
2203  *
2204  * @dev: amdgpu_device pointer
2205  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2206  * @state: powergating state (gate or ungate)
2207  *
2208  * Sets the requested powergating state for all instances of
2209  * the hardware IP specified.
2210  * Returns the error code from the last instance.
2211  */
2212 int amdgpu_device_ip_set_powergating_state(void *dev,
2213 					   enum amd_ip_block_type block_type,
2214 					   enum amd_powergating_state state)
2215 {
2216 	struct amdgpu_device *adev = dev;
2217 	int i, r = 0;
2218 
2219 	for (i = 0; i < adev->num_ip_blocks; i++) {
2220 		if (!adev->ip_blocks[i].status.valid)
2221 			continue;
2222 		if (adev->ip_blocks[i].version->type != block_type)
2223 			continue;
2224 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2225 			continue;
2226 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2227 			&adev->ip_blocks[i], state);
2228 		if (r)
2229 			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2230 				  adev->ip_blocks[i].version->funcs->name, r);
2231 	}
2232 	return r;
2233 }
2234 
2235 /**
2236  * amdgpu_device_ip_get_clockgating_state - get the CG state
2237  *
2238  * @adev: amdgpu_device pointer
2239  * @flags: clockgating feature flags
2240  *
2241  * Walks the list of IPs on the device and updates the clockgating
2242  * flags for each IP.
2243  * Updates @flags with the feature flags for each hardware IP where
2244  * clockgating is enabled.
2245  */
2246 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2247 					    u64 *flags)
2248 {
2249 	int i;
2250 
2251 	for (i = 0; i < adev->num_ip_blocks; i++) {
2252 		if (!adev->ip_blocks[i].status.valid)
2253 			continue;
2254 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2255 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2256 	}
2257 }
2258 
2259 /**
2260  * amdgpu_device_ip_wait_for_idle - wait for idle
2261  *
2262  * @adev: amdgpu_device pointer
2263  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2264  *
2265  * Waits for the request hardware IP to be idle.
2266  * Returns 0 for success or a negative error code on failure.
2267  */
2268 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2269 				   enum amd_ip_block_type block_type)
2270 {
2271 	int i, r;
2272 
2273 	for (i = 0; i < adev->num_ip_blocks; i++) {
2274 		if (!adev->ip_blocks[i].status.valid)
2275 			continue;
2276 		if (adev->ip_blocks[i].version->type == block_type) {
2277 			if (adev->ip_blocks[i].version->funcs->wait_for_idle) {
2278 				r = adev->ip_blocks[i].version->funcs->wait_for_idle(
2279 								&adev->ip_blocks[i]);
2280 				if (r)
2281 					return r;
2282 			}
2283 			break;
2284 		}
2285 	}
2286 	return 0;
2287 
2288 }
2289 
2290 /**
2291  * amdgpu_device_ip_is_valid - is the hardware IP enabled
2292  *
2293  * @adev: amdgpu_device pointer
2294  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2295  *
2296  * Check if the hardware IP is enable or not.
2297  * Returns true if it the IP is enable, false if not.
2298  */
2299 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
2300 			       enum amd_ip_block_type block_type)
2301 {
2302 	int i;
2303 
2304 	for (i = 0; i < adev->num_ip_blocks; i++) {
2305 		if (adev->ip_blocks[i].version->type == block_type)
2306 			return adev->ip_blocks[i].status.valid;
2307 	}
2308 	return false;
2309 
2310 }
2311 
2312 /**
2313  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2314  *
2315  * @adev: amdgpu_device pointer
2316  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2317  *
2318  * Returns a pointer to the hardware IP block structure
2319  * if it exists for the asic, otherwise NULL.
2320  */
2321 struct amdgpu_ip_block *
2322 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2323 			      enum amd_ip_block_type type)
2324 {
2325 	int i;
2326 
2327 	for (i = 0; i < adev->num_ip_blocks; i++)
2328 		if (adev->ip_blocks[i].version->type == type)
2329 			return &adev->ip_blocks[i];
2330 
2331 	return NULL;
2332 }
2333 
2334 /**
2335  * amdgpu_device_ip_block_version_cmp
2336  *
2337  * @adev: amdgpu_device pointer
2338  * @type: enum amd_ip_block_type
2339  * @major: major version
2340  * @minor: minor version
2341  *
2342  * return 0 if equal or greater
2343  * return 1 if smaller or the ip_block doesn't exist
2344  */
2345 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2346 				       enum amd_ip_block_type type,
2347 				       u32 major, u32 minor)
2348 {
2349 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2350 
2351 	if (ip_block && ((ip_block->version->major > major) ||
2352 			((ip_block->version->major == major) &&
2353 			(ip_block->version->minor >= minor))))
2354 		return 0;
2355 
2356 	return 1;
2357 }
2358 
2359 /**
2360  * amdgpu_device_ip_block_add
2361  *
2362  * @adev: amdgpu_device pointer
2363  * @ip_block_version: pointer to the IP to add
2364  *
2365  * Adds the IP block driver information to the collection of IPs
2366  * on the asic.
2367  */
2368 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2369 			       const struct amdgpu_ip_block_version *ip_block_version)
2370 {
2371 	if (!ip_block_version)
2372 		return -EINVAL;
2373 
2374 	switch (ip_block_version->type) {
2375 	case AMD_IP_BLOCK_TYPE_VCN:
2376 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2377 			return 0;
2378 		break;
2379 	case AMD_IP_BLOCK_TYPE_JPEG:
2380 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2381 			return 0;
2382 		break;
2383 	default:
2384 		break;
2385 	}
2386 
2387 	dev_info(adev->dev, "detected ip block number %d <%s>\n",
2388 		 adev->num_ip_blocks, ip_block_version->funcs->name);
2389 
2390 	adev->ip_blocks[adev->num_ip_blocks].adev = adev;
2391 
2392 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2393 
2394 	return 0;
2395 }
2396 
2397 /**
2398  * amdgpu_device_enable_virtual_display - enable virtual display feature
2399  *
2400  * @adev: amdgpu_device pointer
2401  *
2402  * Enabled the virtual display feature if the user has enabled it via
2403  * the module parameter virtual_display.  This feature provides a virtual
2404  * display hardware on headless boards or in virtualized environments.
2405  * This function parses and validates the configuration string specified by
2406  * the user and configures the virtual display configuration (number of
2407  * virtual connectors, crtcs, etc.) specified.
2408  */
2409 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2410 {
2411 	adev->enable_virtual_display = false;
2412 
2413 	if (amdgpu_virtual_display) {
2414 		const char *pci_address_name = pci_name(adev->pdev);
2415 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2416 
2417 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2418 		pciaddstr_tmp = pciaddstr;
2419 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2420 			pciaddname = strsep(&pciaddname_tmp, ",");
2421 			if (!strcmp("all", pciaddname)
2422 			    || !strcmp(pci_address_name, pciaddname)) {
2423 				long num_crtc;
2424 				int res = -1;
2425 
2426 				adev->enable_virtual_display = true;
2427 
2428 				if (pciaddname_tmp)
2429 					res = kstrtol(pciaddname_tmp, 10,
2430 						      &num_crtc);
2431 
2432 				if (!res) {
2433 					if (num_crtc < 1)
2434 						num_crtc = 1;
2435 					if (num_crtc > 6)
2436 						num_crtc = 6;
2437 					adev->mode_info.num_crtc = num_crtc;
2438 				} else {
2439 					adev->mode_info.num_crtc = 1;
2440 				}
2441 				break;
2442 			}
2443 		}
2444 
2445 		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2446 			 amdgpu_virtual_display, pci_address_name,
2447 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2448 
2449 		kfree(pciaddstr);
2450 	}
2451 }
2452 
2453 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2454 {
2455 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2456 		adev->mode_info.num_crtc = 1;
2457 		adev->enable_virtual_display = true;
2458 		DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2459 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2460 	}
2461 }
2462 
2463 /**
2464  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2465  *
2466  * @adev: amdgpu_device pointer
2467  *
2468  * Parses the asic configuration parameters specified in the gpu info
2469  * firmware and makes them available to the driver for use in configuring
2470  * the asic.
2471  * Returns 0 on success, -EINVAL on failure.
2472  */
2473 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2474 {
2475 	const char *chip_name;
2476 	int err;
2477 	const struct gpu_info_firmware_header_v1_0 *hdr;
2478 
2479 	adev->firmware.gpu_info_fw = NULL;
2480 
2481 	if (adev->mman.discovery_bin)
2482 		return 0;
2483 
2484 	switch (adev->asic_type) {
2485 	default:
2486 		return 0;
2487 	case CHIP_VEGA10:
2488 		chip_name = "vega10";
2489 		break;
2490 	case CHIP_VEGA12:
2491 		chip_name = "vega12";
2492 		break;
2493 	case CHIP_RAVEN:
2494 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2495 			chip_name = "raven2";
2496 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2497 			chip_name = "picasso";
2498 		else
2499 			chip_name = "raven";
2500 		break;
2501 	case CHIP_ARCTURUS:
2502 		chip_name = "arcturus";
2503 		break;
2504 	case CHIP_NAVI12:
2505 		chip_name = "navi12";
2506 		break;
2507 	}
2508 
2509 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,
2510 				   AMDGPU_UCODE_OPTIONAL,
2511 				   "amdgpu/%s_gpu_info.bin", chip_name);
2512 	if (err) {
2513 		dev_err(adev->dev,
2514 			"Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n",
2515 			chip_name);
2516 		goto out;
2517 	}
2518 
2519 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2520 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2521 
2522 	switch (hdr->version_major) {
2523 	case 1:
2524 	{
2525 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2526 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2527 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2528 
2529 		/*
2530 		 * Should be dropped when DAL no longer needs it.
2531 		 */
2532 		if (adev->asic_type == CHIP_NAVI12)
2533 			goto parse_soc_bounding_box;
2534 
2535 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2536 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2537 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2538 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2539 		adev->gfx.config.max_texture_channel_caches =
2540 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
2541 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2542 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2543 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2544 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2545 		adev->gfx.config.double_offchip_lds_buf =
2546 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2547 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2548 		adev->gfx.cu_info.max_waves_per_simd =
2549 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2550 		adev->gfx.cu_info.max_scratch_slots_per_cu =
2551 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2552 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2553 		if (hdr->version_minor >= 1) {
2554 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2555 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2556 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2557 			adev->gfx.config.num_sc_per_sh =
2558 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2559 			adev->gfx.config.num_packer_per_sc =
2560 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2561 		}
2562 
2563 parse_soc_bounding_box:
2564 		/*
2565 		 * soc bounding box info is not integrated in disocovery table,
2566 		 * we always need to parse it from gpu info firmware if needed.
2567 		 */
2568 		if (hdr->version_minor == 2) {
2569 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2570 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2571 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2572 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2573 		}
2574 		break;
2575 	}
2576 	default:
2577 		dev_err(adev->dev,
2578 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2579 		err = -EINVAL;
2580 		goto out;
2581 	}
2582 out:
2583 	return err;
2584 }
2585 
2586 /**
2587  * amdgpu_device_ip_early_init - run early init for hardware IPs
2588  *
2589  * @adev: amdgpu_device pointer
2590  *
2591  * Early initialization pass for hardware IPs.  The hardware IPs that make
2592  * up each asic are discovered each IP's early_init callback is run.  This
2593  * is the first stage in initializing the asic.
2594  * Returns 0 on success, negative error code on failure.
2595  */
2596 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2597 {
2598 	struct amdgpu_ip_block *ip_block;
2599 	struct pci_dev *parent;
2600 	bool total, skip_bios;
2601 	uint32_t bios_flags;
2602 	int i, r;
2603 
2604 	amdgpu_device_enable_virtual_display(adev);
2605 
2606 	if (amdgpu_sriov_vf(adev)) {
2607 		r = amdgpu_virt_request_full_gpu(adev, true);
2608 		if (r)
2609 			return r;
2610 	}
2611 
2612 	switch (adev->asic_type) {
2613 #ifdef CONFIG_DRM_AMDGPU_SI
2614 	case CHIP_VERDE:
2615 	case CHIP_TAHITI:
2616 	case CHIP_PITCAIRN:
2617 	case CHIP_OLAND:
2618 	case CHIP_HAINAN:
2619 		adev->family = AMDGPU_FAMILY_SI;
2620 		r = si_set_ip_blocks(adev);
2621 		if (r)
2622 			return r;
2623 		break;
2624 #endif
2625 #ifdef CONFIG_DRM_AMDGPU_CIK
2626 	case CHIP_BONAIRE:
2627 	case CHIP_HAWAII:
2628 	case CHIP_KAVERI:
2629 	case CHIP_KABINI:
2630 	case CHIP_MULLINS:
2631 		if (adev->flags & AMD_IS_APU)
2632 			adev->family = AMDGPU_FAMILY_KV;
2633 		else
2634 			adev->family = AMDGPU_FAMILY_CI;
2635 
2636 		r = cik_set_ip_blocks(adev);
2637 		if (r)
2638 			return r;
2639 		break;
2640 #endif
2641 	case CHIP_TOPAZ:
2642 	case CHIP_TONGA:
2643 	case CHIP_FIJI:
2644 	case CHIP_POLARIS10:
2645 	case CHIP_POLARIS11:
2646 	case CHIP_POLARIS12:
2647 	case CHIP_VEGAM:
2648 	case CHIP_CARRIZO:
2649 	case CHIP_STONEY:
2650 		if (adev->flags & AMD_IS_APU)
2651 			adev->family = AMDGPU_FAMILY_CZ;
2652 		else
2653 			adev->family = AMDGPU_FAMILY_VI;
2654 
2655 		r = vi_set_ip_blocks(adev);
2656 		if (r)
2657 			return r;
2658 		break;
2659 	default:
2660 		r = amdgpu_discovery_set_ip_blocks(adev);
2661 		if (r)
2662 			return r;
2663 		break;
2664 	}
2665 
2666 	if (amdgpu_has_atpx() &&
2667 	    (amdgpu_is_atpx_hybrid() ||
2668 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2669 	    ((adev->flags & AMD_IS_APU) == 0) &&
2670 	    !dev_is_removable(&adev->pdev->dev))
2671 		adev->flags |= AMD_IS_PX;
2672 
2673 	if (!(adev->flags & AMD_IS_APU)) {
2674 		parent = pcie_find_root_port(adev->pdev);
2675 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2676 	}
2677 
2678 
2679 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2680 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2681 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2682 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2683 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2684 	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2685 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2686 
2687 	total = true;
2688 	for (i = 0; i < adev->num_ip_blocks; i++) {
2689 		ip_block = &adev->ip_blocks[i];
2690 
2691 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2692 			DRM_WARN("disabled ip block: %d <%s>\n",
2693 				  i, adev->ip_blocks[i].version->funcs->name);
2694 			adev->ip_blocks[i].status.valid = false;
2695 		} else if (ip_block->version->funcs->early_init) {
2696 			r = ip_block->version->funcs->early_init(ip_block);
2697 			if (r == -ENOENT) {
2698 				adev->ip_blocks[i].status.valid = false;
2699 			} else if (r) {
2700 				DRM_ERROR("early_init of IP block <%s> failed %d\n",
2701 					  adev->ip_blocks[i].version->funcs->name, r);
2702 				total = false;
2703 			} else {
2704 				adev->ip_blocks[i].status.valid = true;
2705 			}
2706 		} else {
2707 			adev->ip_blocks[i].status.valid = true;
2708 		}
2709 		/* get the vbios after the asic_funcs are set up */
2710 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2711 			r = amdgpu_device_parse_gpu_info_fw(adev);
2712 			if (r)
2713 				return r;
2714 
2715 			bios_flags = amdgpu_device_get_vbios_flags(adev);
2716 			skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP);
2717 			/* Read BIOS */
2718 			if (!skip_bios) {
2719 				bool optional =
2720 					!!(bios_flags & AMDGPU_VBIOS_OPTIONAL);
2721 				if (!amdgpu_get_bios(adev) && !optional)
2722 					return -EINVAL;
2723 
2724 				if (optional && !adev->bios)
2725 					dev_info(
2726 						adev->dev,
2727 						"VBIOS image optional, proceeding without VBIOS image");
2728 
2729 				if (adev->bios) {
2730 					r = amdgpu_atombios_init(adev);
2731 					if (r) {
2732 						dev_err(adev->dev,
2733 							"amdgpu_atombios_init failed\n");
2734 						amdgpu_vf_error_put(
2735 							adev,
2736 							AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL,
2737 							0, 0);
2738 						return r;
2739 					}
2740 				}
2741 			}
2742 
2743 			/*get pf2vf msg info at it's earliest time*/
2744 			if (amdgpu_sriov_vf(adev))
2745 				amdgpu_virt_init_data_exchange(adev);
2746 
2747 		}
2748 	}
2749 	if (!total)
2750 		return -ENODEV;
2751 
2752 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
2753 	if (ip_block->status.valid != false)
2754 		amdgpu_amdkfd_device_probe(adev);
2755 
2756 	adev->cg_flags &= amdgpu_cg_mask;
2757 	adev->pg_flags &= amdgpu_pg_mask;
2758 
2759 	return 0;
2760 }
2761 
2762 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2763 {
2764 	int i, r;
2765 
2766 	for (i = 0; i < adev->num_ip_blocks; i++) {
2767 		if (!adev->ip_blocks[i].status.sw)
2768 			continue;
2769 		if (adev->ip_blocks[i].status.hw)
2770 			continue;
2771 		if (!amdgpu_ip_member_of_hwini(
2772 			    adev, adev->ip_blocks[i].version->type))
2773 			continue;
2774 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2775 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2776 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2777 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2778 			if (r) {
2779 				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2780 					  adev->ip_blocks[i].version->funcs->name, r);
2781 				return r;
2782 			}
2783 			adev->ip_blocks[i].status.hw = true;
2784 		}
2785 	}
2786 
2787 	return 0;
2788 }
2789 
2790 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2791 {
2792 	int i, r;
2793 
2794 	for (i = 0; i < adev->num_ip_blocks; i++) {
2795 		if (!adev->ip_blocks[i].status.sw)
2796 			continue;
2797 		if (adev->ip_blocks[i].status.hw)
2798 			continue;
2799 		if (!amdgpu_ip_member_of_hwini(
2800 			    adev, adev->ip_blocks[i].version->type))
2801 			continue;
2802 		r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2803 		if (r) {
2804 			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2805 				  adev->ip_blocks[i].version->funcs->name, r);
2806 			return r;
2807 		}
2808 		adev->ip_blocks[i].status.hw = true;
2809 	}
2810 
2811 	return 0;
2812 }
2813 
2814 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2815 {
2816 	int r = 0;
2817 	int i;
2818 	uint32_t smu_version;
2819 
2820 	if (adev->asic_type >= CHIP_VEGA10) {
2821 		for (i = 0; i < adev->num_ip_blocks; i++) {
2822 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2823 				continue;
2824 
2825 			if (!amdgpu_ip_member_of_hwini(adev,
2826 						       AMD_IP_BLOCK_TYPE_PSP))
2827 				break;
2828 
2829 			if (!adev->ip_blocks[i].status.sw)
2830 				continue;
2831 
2832 			/* no need to do the fw loading again if already done*/
2833 			if (adev->ip_blocks[i].status.hw == true)
2834 				break;
2835 
2836 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2837 				r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
2838 				if (r)
2839 					return r;
2840 			} else {
2841 				r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2842 				if (r) {
2843 					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2844 							  adev->ip_blocks[i].version->funcs->name, r);
2845 					return r;
2846 				}
2847 				adev->ip_blocks[i].status.hw = true;
2848 			}
2849 			break;
2850 		}
2851 	}
2852 
2853 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2854 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2855 
2856 	return r;
2857 }
2858 
2859 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2860 {
2861 	long timeout;
2862 	int r, i;
2863 
2864 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2865 		struct amdgpu_ring *ring = adev->rings[i];
2866 
2867 		/* No need to setup the GPU scheduler for rings that don't need it */
2868 		if (!ring || ring->no_scheduler)
2869 			continue;
2870 
2871 		switch (ring->funcs->type) {
2872 		case AMDGPU_RING_TYPE_GFX:
2873 			timeout = adev->gfx_timeout;
2874 			break;
2875 		case AMDGPU_RING_TYPE_COMPUTE:
2876 			timeout = adev->compute_timeout;
2877 			break;
2878 		case AMDGPU_RING_TYPE_SDMA:
2879 			timeout = adev->sdma_timeout;
2880 			break;
2881 		default:
2882 			timeout = adev->video_timeout;
2883 			break;
2884 		}
2885 
2886 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2887 				   DRM_SCHED_PRIORITY_COUNT,
2888 				   ring->num_hw_submission, 0,
2889 				   timeout, adev->reset_domain->wq,
2890 				   ring->sched_score, ring->name,
2891 				   adev->dev);
2892 		if (r) {
2893 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
2894 				  ring->name);
2895 			return r;
2896 		}
2897 		r = amdgpu_uvd_entity_init(adev, ring);
2898 		if (r) {
2899 			DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2900 				  ring->name);
2901 			return r;
2902 		}
2903 		r = amdgpu_vce_entity_init(adev, ring);
2904 		if (r) {
2905 			DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2906 				  ring->name);
2907 			return r;
2908 		}
2909 	}
2910 
2911 	amdgpu_xcp_update_partition_sched_list(adev);
2912 
2913 	return 0;
2914 }
2915 
2916 
2917 /**
2918  * amdgpu_device_ip_init - run init for hardware IPs
2919  *
2920  * @adev: amdgpu_device pointer
2921  *
2922  * Main initialization pass for hardware IPs.  The list of all the hardware
2923  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2924  * are run.  sw_init initializes the software state associated with each IP
2925  * and hw_init initializes the hardware associated with each IP.
2926  * Returns 0 on success, negative error code on failure.
2927  */
2928 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2929 {
2930 	bool init_badpage;
2931 	int i, r;
2932 
2933 	r = amdgpu_ras_init(adev);
2934 	if (r)
2935 		return r;
2936 
2937 	for (i = 0; i < adev->num_ip_blocks; i++) {
2938 		if (!adev->ip_blocks[i].status.valid)
2939 			continue;
2940 		if (adev->ip_blocks[i].version->funcs->sw_init) {
2941 			r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);
2942 			if (r) {
2943 				DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2944 					  adev->ip_blocks[i].version->funcs->name, r);
2945 				goto init_failed;
2946 			}
2947 		}
2948 		adev->ip_blocks[i].status.sw = true;
2949 
2950 		if (!amdgpu_ip_member_of_hwini(
2951 			    adev, adev->ip_blocks[i].version->type))
2952 			continue;
2953 
2954 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2955 			/* need to do common hw init early so everything is set up for gmc */
2956 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2957 			if (r) {
2958 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2959 				goto init_failed;
2960 			}
2961 			adev->ip_blocks[i].status.hw = true;
2962 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2963 			/* need to do gmc hw init early so we can allocate gpu mem */
2964 			/* Try to reserve bad pages early */
2965 			if (amdgpu_sriov_vf(adev))
2966 				amdgpu_virt_exchange_data(adev);
2967 
2968 			r = amdgpu_device_mem_scratch_init(adev);
2969 			if (r) {
2970 				DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2971 				goto init_failed;
2972 			}
2973 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2974 			if (r) {
2975 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2976 				goto init_failed;
2977 			}
2978 			r = amdgpu_device_wb_init(adev);
2979 			if (r) {
2980 				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2981 				goto init_failed;
2982 			}
2983 			adev->ip_blocks[i].status.hw = true;
2984 
2985 			/* right after GMC hw init, we create CSA */
2986 			if (adev->gfx.mcbp) {
2987 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2988 							       AMDGPU_GEM_DOMAIN_VRAM |
2989 							       AMDGPU_GEM_DOMAIN_GTT,
2990 							       AMDGPU_CSA_SIZE);
2991 				if (r) {
2992 					DRM_ERROR("allocate CSA failed %d\n", r);
2993 					goto init_failed;
2994 				}
2995 			}
2996 
2997 			r = amdgpu_seq64_init(adev);
2998 			if (r) {
2999 				DRM_ERROR("allocate seq64 failed %d\n", r);
3000 				goto init_failed;
3001 			}
3002 		}
3003 	}
3004 
3005 	if (amdgpu_sriov_vf(adev))
3006 		amdgpu_virt_init_data_exchange(adev);
3007 
3008 	r = amdgpu_ib_pool_init(adev);
3009 	if (r) {
3010 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
3011 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
3012 		goto init_failed;
3013 	}
3014 
3015 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
3016 	if (r)
3017 		goto init_failed;
3018 
3019 	r = amdgpu_device_ip_hw_init_phase1(adev);
3020 	if (r)
3021 		goto init_failed;
3022 
3023 	r = amdgpu_device_fw_loading(adev);
3024 	if (r)
3025 		goto init_failed;
3026 
3027 	r = amdgpu_device_ip_hw_init_phase2(adev);
3028 	if (r)
3029 		goto init_failed;
3030 
3031 	/*
3032 	 * retired pages will be loaded from eeprom and reserved here,
3033 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
3034 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
3035 	 * for I2C communication which only true at this point.
3036 	 *
3037 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
3038 	 * failure from bad gpu situation and stop amdgpu init process
3039 	 * accordingly. For other failed cases, it will still release all
3040 	 * the resource and print error message, rather than returning one
3041 	 * negative value to upper level.
3042 	 *
3043 	 * Note: theoretically, this should be called before all vram allocations
3044 	 * to protect retired page from abusing
3045 	 */
3046 	init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
3047 	r = amdgpu_ras_recovery_init(adev, init_badpage);
3048 	if (r)
3049 		goto init_failed;
3050 
3051 	/**
3052 	 * In case of XGMI grab extra reference for reset domain for this device
3053 	 */
3054 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3055 		if (amdgpu_xgmi_add_device(adev) == 0) {
3056 			if (!amdgpu_sriov_vf(adev)) {
3057 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3058 
3059 				if (WARN_ON(!hive)) {
3060 					r = -ENOENT;
3061 					goto init_failed;
3062 				}
3063 
3064 				if (!hive->reset_domain ||
3065 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
3066 					r = -ENOENT;
3067 					amdgpu_put_xgmi_hive(hive);
3068 					goto init_failed;
3069 				}
3070 
3071 				/* Drop the early temporary reset domain we created for device */
3072 				amdgpu_reset_put_reset_domain(adev->reset_domain);
3073 				adev->reset_domain = hive->reset_domain;
3074 				amdgpu_put_xgmi_hive(hive);
3075 			}
3076 		}
3077 	}
3078 
3079 	r = amdgpu_device_init_schedulers(adev);
3080 	if (r)
3081 		goto init_failed;
3082 
3083 	if (adev->mman.buffer_funcs_ring->sched.ready)
3084 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
3085 
3086 	/* Don't init kfd if whole hive need to be reset during init */
3087 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
3088 		kgd2kfd_init_zone_device(adev);
3089 		amdgpu_amdkfd_device_init(adev);
3090 	}
3091 
3092 	amdgpu_fru_get_product_info(adev);
3093 
3094 init_failed:
3095 
3096 	return r;
3097 }
3098 
3099 /**
3100  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
3101  *
3102  * @adev: amdgpu_device pointer
3103  *
3104  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
3105  * this function before a GPU reset.  If the value is retained after a
3106  * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents.
3107  */
3108 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
3109 {
3110 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
3111 }
3112 
3113 /**
3114  * amdgpu_device_check_vram_lost - check if vram is valid
3115  *
3116  * @adev: amdgpu_device pointer
3117  *
3118  * Checks the reset magic value written to the gart pointer in VRAM.
3119  * The driver calls this after a GPU reset to see if the contents of
3120  * VRAM is lost or now.
3121  * returns true if vram is lost, false if not.
3122  */
3123 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
3124 {
3125 	if (memcmp(adev->gart.ptr, adev->reset_magic,
3126 			AMDGPU_RESET_MAGIC_NUM))
3127 		return true;
3128 
3129 	if (!amdgpu_in_reset(adev))
3130 		return false;
3131 
3132 	/*
3133 	 * For all ASICs with baco/mode1 reset, the VRAM is
3134 	 * always assumed to be lost.
3135 	 */
3136 	switch (amdgpu_asic_reset_method(adev)) {
3137 	case AMD_RESET_METHOD_BACO:
3138 	case AMD_RESET_METHOD_MODE1:
3139 		return true;
3140 	default:
3141 		return false;
3142 	}
3143 }
3144 
3145 /**
3146  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
3147  *
3148  * @adev: amdgpu_device pointer
3149  * @state: clockgating state (gate or ungate)
3150  *
3151  * The list of all the hardware IPs that make up the asic is walked and the
3152  * set_clockgating_state callbacks are run.
3153  * Late initialization pass enabling clockgating for hardware IPs.
3154  * Fini or suspend, pass disabling clockgating for hardware IPs.
3155  * Returns 0 on success, negative error code on failure.
3156  */
3157 
3158 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
3159 			       enum amd_clockgating_state state)
3160 {
3161 	int i, j, r;
3162 
3163 	if (amdgpu_emu_mode == 1)
3164 		return 0;
3165 
3166 	for (j = 0; j < adev->num_ip_blocks; j++) {
3167 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3168 		if (!adev->ip_blocks[i].status.late_initialized)
3169 			continue;
3170 		/* skip CG for GFX, SDMA on S0ix */
3171 		if (adev->in_s0ix &&
3172 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3173 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3174 			continue;
3175 		/* skip CG for VCE/UVD, it's handled specially */
3176 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3177 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3178 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3179 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3180 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
3181 			/* enable clockgating to save power */
3182 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i],
3183 										     state);
3184 			if (r) {
3185 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
3186 					  adev->ip_blocks[i].version->funcs->name, r);
3187 				return r;
3188 			}
3189 		}
3190 	}
3191 
3192 	return 0;
3193 }
3194 
3195 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
3196 			       enum amd_powergating_state state)
3197 {
3198 	int i, j, r;
3199 
3200 	if (amdgpu_emu_mode == 1)
3201 		return 0;
3202 
3203 	for (j = 0; j < adev->num_ip_blocks; j++) {
3204 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3205 		if (!adev->ip_blocks[i].status.late_initialized)
3206 			continue;
3207 		/* skip PG for GFX, SDMA on S0ix */
3208 		if (adev->in_s0ix &&
3209 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3210 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3211 			continue;
3212 		/* skip CG for VCE/UVD, it's handled specially */
3213 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3214 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3215 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3216 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3217 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
3218 			/* enable powergating to save power */
3219 			r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i],
3220 											state);
3221 			if (r) {
3222 				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
3223 					  adev->ip_blocks[i].version->funcs->name, r);
3224 				return r;
3225 			}
3226 		}
3227 	}
3228 	return 0;
3229 }
3230 
3231 static int amdgpu_device_enable_mgpu_fan_boost(void)
3232 {
3233 	struct amdgpu_gpu_instance *gpu_ins;
3234 	struct amdgpu_device *adev;
3235 	int i, ret = 0;
3236 
3237 	mutex_lock(&mgpu_info.mutex);
3238 
3239 	/*
3240 	 * MGPU fan boost feature should be enabled
3241 	 * only when there are two or more dGPUs in
3242 	 * the system
3243 	 */
3244 	if (mgpu_info.num_dgpu < 2)
3245 		goto out;
3246 
3247 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
3248 		gpu_ins = &(mgpu_info.gpu_ins[i]);
3249 		adev = gpu_ins->adev;
3250 		if (!(adev->flags & AMD_IS_APU) &&
3251 		    !gpu_ins->mgpu_fan_enabled) {
3252 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3253 			if (ret)
3254 				break;
3255 
3256 			gpu_ins->mgpu_fan_enabled = 1;
3257 		}
3258 	}
3259 
3260 out:
3261 	mutex_unlock(&mgpu_info.mutex);
3262 
3263 	return ret;
3264 }
3265 
3266 /**
3267  * amdgpu_device_ip_late_init - run late init for hardware IPs
3268  *
3269  * @adev: amdgpu_device pointer
3270  *
3271  * Late initialization pass for hardware IPs.  The list of all the hardware
3272  * IPs that make up the asic is walked and the late_init callbacks are run.
3273  * late_init covers any special initialization that an IP requires
3274  * after all of the have been initialized or something that needs to happen
3275  * late in the init process.
3276  * Returns 0 on success, negative error code on failure.
3277  */
3278 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3279 {
3280 	struct amdgpu_gpu_instance *gpu_instance;
3281 	int i = 0, r;
3282 
3283 	for (i = 0; i < adev->num_ip_blocks; i++) {
3284 		if (!adev->ip_blocks[i].status.hw)
3285 			continue;
3286 		if (adev->ip_blocks[i].version->funcs->late_init) {
3287 			r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
3288 			if (r) {
3289 				DRM_ERROR("late_init of IP block <%s> failed %d\n",
3290 					  adev->ip_blocks[i].version->funcs->name, r);
3291 				return r;
3292 			}
3293 		}
3294 		adev->ip_blocks[i].status.late_initialized = true;
3295 	}
3296 
3297 	r = amdgpu_ras_late_init(adev);
3298 	if (r) {
3299 		DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3300 		return r;
3301 	}
3302 
3303 	if (!amdgpu_reset_in_recovery(adev))
3304 		amdgpu_ras_set_error_query_ready(adev, true);
3305 
3306 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3307 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3308 
3309 	amdgpu_device_fill_reset_magic(adev);
3310 
3311 	r = amdgpu_device_enable_mgpu_fan_boost();
3312 	if (r)
3313 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3314 
3315 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3316 	if (amdgpu_passthrough(adev) &&
3317 	    ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3318 	     adev->asic_type == CHIP_ALDEBARAN))
3319 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
3320 
3321 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3322 		mutex_lock(&mgpu_info.mutex);
3323 
3324 		/*
3325 		 * Reset device p-state to low as this was booted with high.
3326 		 *
3327 		 * This should be performed only after all devices from the same
3328 		 * hive get initialized.
3329 		 *
3330 		 * However, it's unknown how many device in the hive in advance.
3331 		 * As this is counted one by one during devices initializations.
3332 		 *
3333 		 * So, we wait for all XGMI interlinked devices initialized.
3334 		 * This may bring some delays as those devices may come from
3335 		 * different hives. But that should be OK.
3336 		 */
3337 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3338 			for (i = 0; i < mgpu_info.num_gpu; i++) {
3339 				gpu_instance = &(mgpu_info.gpu_ins[i]);
3340 				if (gpu_instance->adev->flags & AMD_IS_APU)
3341 					continue;
3342 
3343 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3344 						AMDGPU_XGMI_PSTATE_MIN);
3345 				if (r) {
3346 					DRM_ERROR("pstate setting failed (%d).\n", r);
3347 					break;
3348 				}
3349 			}
3350 		}
3351 
3352 		mutex_unlock(&mgpu_info.mutex);
3353 	}
3354 
3355 	return 0;
3356 }
3357 
3358 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block)
3359 {
3360 	int r;
3361 
3362 	if (!ip_block->version->funcs->hw_fini) {
3363 		DRM_ERROR("hw_fini of IP block <%s> not defined\n",
3364 			  ip_block->version->funcs->name);
3365 	} else {
3366 		r = ip_block->version->funcs->hw_fini(ip_block);
3367 		/* XXX handle errors */
3368 		if (r) {
3369 			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3370 				  ip_block->version->funcs->name, r);
3371 		}
3372 	}
3373 
3374 	ip_block->status.hw = false;
3375 }
3376 
3377 /**
3378  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3379  *
3380  * @adev: amdgpu_device pointer
3381  *
3382  * For ASICs need to disable SMC first
3383  */
3384 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3385 {
3386 	int i;
3387 
3388 	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3389 		return;
3390 
3391 	for (i = 0; i < adev->num_ip_blocks; i++) {
3392 		if (!adev->ip_blocks[i].status.hw)
3393 			continue;
3394 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3395 			amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3396 			break;
3397 		}
3398 	}
3399 }
3400 
3401 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3402 {
3403 	int i, r;
3404 
3405 	for (i = 0; i < adev->num_ip_blocks; i++) {
3406 		if (!adev->ip_blocks[i].version->funcs->early_fini)
3407 			continue;
3408 
3409 		r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);
3410 		if (r) {
3411 			DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3412 				  adev->ip_blocks[i].version->funcs->name, r);
3413 		}
3414 	}
3415 
3416 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3417 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3418 
3419 	amdgpu_amdkfd_suspend(adev, false);
3420 
3421 	/* Workaround for ASICs need to disable SMC first */
3422 	amdgpu_device_smu_fini_early(adev);
3423 
3424 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3425 		if (!adev->ip_blocks[i].status.hw)
3426 			continue;
3427 
3428 		amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3429 	}
3430 
3431 	if (amdgpu_sriov_vf(adev)) {
3432 		if (amdgpu_virt_release_full_gpu(adev, false))
3433 			DRM_ERROR("failed to release exclusive mode on fini\n");
3434 	}
3435 
3436 	return 0;
3437 }
3438 
3439 /**
3440  * amdgpu_device_ip_fini - run fini for hardware IPs
3441  *
3442  * @adev: amdgpu_device pointer
3443  *
3444  * Main teardown pass for hardware IPs.  The list of all the hardware
3445  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3446  * are run.  hw_fini tears down the hardware associated with each IP
3447  * and sw_fini tears down any software state associated with each IP.
3448  * Returns 0 on success, negative error code on failure.
3449  */
3450 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3451 {
3452 	int i, r;
3453 
3454 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3455 		amdgpu_virt_release_ras_err_handler_data(adev);
3456 
3457 	if (adev->gmc.xgmi.num_physical_nodes > 1)
3458 		amdgpu_xgmi_remove_device(adev);
3459 
3460 	amdgpu_amdkfd_device_fini_sw(adev);
3461 
3462 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3463 		if (!adev->ip_blocks[i].status.sw)
3464 			continue;
3465 
3466 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3467 			amdgpu_ucode_free_bo(adev);
3468 			amdgpu_free_static_csa(&adev->virt.csa_obj);
3469 			amdgpu_device_wb_fini(adev);
3470 			amdgpu_device_mem_scratch_fini(adev);
3471 			amdgpu_ib_pool_fini(adev);
3472 			amdgpu_seq64_fini(adev);
3473 		}
3474 		if (adev->ip_blocks[i].version->funcs->sw_fini) {
3475 			r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);
3476 			/* XXX handle errors */
3477 			if (r) {
3478 				DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3479 					  adev->ip_blocks[i].version->funcs->name, r);
3480 			}
3481 		}
3482 		adev->ip_blocks[i].status.sw = false;
3483 		adev->ip_blocks[i].status.valid = false;
3484 	}
3485 
3486 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3487 		if (!adev->ip_blocks[i].status.late_initialized)
3488 			continue;
3489 		if (adev->ip_blocks[i].version->funcs->late_fini)
3490 			adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);
3491 		adev->ip_blocks[i].status.late_initialized = false;
3492 	}
3493 
3494 	amdgpu_ras_fini(adev);
3495 
3496 	return 0;
3497 }
3498 
3499 /**
3500  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3501  *
3502  * @work: work_struct.
3503  */
3504 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3505 {
3506 	struct amdgpu_device *adev =
3507 		container_of(work, struct amdgpu_device, delayed_init_work.work);
3508 	int r;
3509 
3510 	r = amdgpu_ib_ring_tests(adev);
3511 	if (r)
3512 		DRM_ERROR("ib ring test failed (%d).\n", r);
3513 }
3514 
3515 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3516 {
3517 	struct amdgpu_device *adev =
3518 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3519 
3520 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3521 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3522 
3523 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
3524 		adev->gfx.gfx_off_state = true;
3525 }
3526 
3527 /**
3528  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3529  *
3530  * @adev: amdgpu_device pointer
3531  *
3532  * Main suspend function for hardware IPs.  The list of all the hardware
3533  * IPs that make up the asic is walked, clockgating is disabled and the
3534  * suspend callbacks are run.  suspend puts the hardware and software state
3535  * in each IP into a state suitable for suspend.
3536  * Returns 0 on success, negative error code on failure.
3537  */
3538 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3539 {
3540 	int i, r;
3541 
3542 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3543 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3544 
3545 	/*
3546 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
3547 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3548 	 * scenario. Add the missing df cstate disablement here.
3549 	 */
3550 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3551 		dev_warn(adev->dev, "Failed to disallow df cstate");
3552 
3553 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3554 		if (!adev->ip_blocks[i].status.valid)
3555 			continue;
3556 
3557 		/* displays are handled separately */
3558 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3559 			continue;
3560 
3561 		/* XXX handle errors */
3562 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3563 		if (r)
3564 			return r;
3565 	}
3566 
3567 	return 0;
3568 }
3569 
3570 /**
3571  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3572  *
3573  * @adev: amdgpu_device pointer
3574  *
3575  * Main suspend function for hardware IPs.  The list of all the hardware
3576  * IPs that make up the asic is walked, clockgating is disabled and the
3577  * suspend callbacks are run.  suspend puts the hardware and software state
3578  * in each IP into a state suitable for suspend.
3579  * Returns 0 on success, negative error code on failure.
3580  */
3581 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3582 {
3583 	int i, r;
3584 
3585 	if (adev->in_s0ix)
3586 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3587 
3588 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3589 		if (!adev->ip_blocks[i].status.valid)
3590 			continue;
3591 		/* displays are handled in phase1 */
3592 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3593 			continue;
3594 		/* PSP lost connection when err_event_athub occurs */
3595 		if (amdgpu_ras_intr_triggered() &&
3596 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3597 			adev->ip_blocks[i].status.hw = false;
3598 			continue;
3599 		}
3600 
3601 		/* skip unnecessary suspend if we do not initialize them yet */
3602 		if (!amdgpu_ip_member_of_hwini(
3603 			    adev, adev->ip_blocks[i].version->type))
3604 			continue;
3605 
3606 		/* skip suspend of gfx/mes and psp for S0ix
3607 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3608 		 * like at runtime. PSP is also part of the always on hardware
3609 		 * so no need to suspend it.
3610 		 */
3611 		if (adev->in_s0ix &&
3612 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3613 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3614 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3615 			continue;
3616 
3617 		/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3618 		if (adev->in_s0ix &&
3619 		    (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3620 		     IP_VERSION(5, 0, 0)) &&
3621 		    (adev->ip_blocks[i].version->type ==
3622 		     AMD_IP_BLOCK_TYPE_SDMA))
3623 			continue;
3624 
3625 		/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3626 		 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3627 		 * from this location and RLC Autoload automatically also gets loaded
3628 		 * from here based on PMFW -> PSP message during re-init sequence.
3629 		 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3630 		 * the TMR and reload FWs again for IMU enabled APU ASICs.
3631 		 */
3632 		if (amdgpu_in_reset(adev) &&
3633 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3634 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3635 			continue;
3636 
3637 		/* XXX handle errors */
3638 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3639 		adev->ip_blocks[i].status.hw = false;
3640 
3641 		/* handle putting the SMC in the appropriate state */
3642 		if (!amdgpu_sriov_vf(adev)) {
3643 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3644 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3645 				if (r) {
3646 					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3647 							adev->mp1_state, r);
3648 					return r;
3649 				}
3650 			}
3651 		}
3652 	}
3653 
3654 	return 0;
3655 }
3656 
3657 /**
3658  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3659  *
3660  * @adev: amdgpu_device pointer
3661  *
3662  * Main suspend function for hardware IPs.  The list of all the hardware
3663  * IPs that make up the asic is walked, clockgating is disabled and the
3664  * suspend callbacks are run.  suspend puts the hardware and software state
3665  * in each IP into a state suitable for suspend.
3666  * Returns 0 on success, negative error code on failure.
3667  */
3668 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3669 {
3670 	int r;
3671 
3672 	if (amdgpu_sriov_vf(adev)) {
3673 		amdgpu_virt_fini_data_exchange(adev);
3674 		amdgpu_virt_request_full_gpu(adev, false);
3675 	}
3676 
3677 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
3678 
3679 	r = amdgpu_device_ip_suspend_phase1(adev);
3680 	if (r)
3681 		return r;
3682 	r = amdgpu_device_ip_suspend_phase2(adev);
3683 
3684 	if (amdgpu_sriov_vf(adev))
3685 		amdgpu_virt_release_full_gpu(adev, false);
3686 
3687 	return r;
3688 }
3689 
3690 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3691 {
3692 	int i, r;
3693 
3694 	static enum amd_ip_block_type ip_order[] = {
3695 		AMD_IP_BLOCK_TYPE_COMMON,
3696 		AMD_IP_BLOCK_TYPE_GMC,
3697 		AMD_IP_BLOCK_TYPE_PSP,
3698 		AMD_IP_BLOCK_TYPE_IH,
3699 	};
3700 
3701 	for (i = 0; i < adev->num_ip_blocks; i++) {
3702 		int j;
3703 		struct amdgpu_ip_block *block;
3704 
3705 		block = &adev->ip_blocks[i];
3706 		block->status.hw = false;
3707 
3708 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3709 
3710 			if (block->version->type != ip_order[j] ||
3711 				!block->status.valid)
3712 				continue;
3713 
3714 			r = block->version->funcs->hw_init(&adev->ip_blocks[i]);
3715 			if (r) {
3716 				dev_err(adev->dev, "RE-INIT-early: %s failed\n",
3717 					 block->version->funcs->name);
3718 				return r;
3719 			}
3720 			block->status.hw = true;
3721 		}
3722 	}
3723 
3724 	return 0;
3725 }
3726 
3727 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3728 {
3729 	struct amdgpu_ip_block *block;
3730 	int i, r = 0;
3731 
3732 	static enum amd_ip_block_type ip_order[] = {
3733 		AMD_IP_BLOCK_TYPE_SMC,
3734 		AMD_IP_BLOCK_TYPE_DCE,
3735 		AMD_IP_BLOCK_TYPE_GFX,
3736 		AMD_IP_BLOCK_TYPE_SDMA,
3737 		AMD_IP_BLOCK_TYPE_MES,
3738 		AMD_IP_BLOCK_TYPE_UVD,
3739 		AMD_IP_BLOCK_TYPE_VCE,
3740 		AMD_IP_BLOCK_TYPE_VCN,
3741 		AMD_IP_BLOCK_TYPE_JPEG
3742 	};
3743 
3744 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3745 		block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);
3746 
3747 		if (!block)
3748 			continue;
3749 
3750 		if (block->status.valid && !block->status.hw) {
3751 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) {
3752 				r = amdgpu_ip_block_resume(block);
3753 			} else {
3754 				r = block->version->funcs->hw_init(block);
3755 			}
3756 
3757 			if (r) {
3758 				dev_err(adev->dev, "RE-INIT-late: %s failed\n",
3759 					 block->version->funcs->name);
3760 				break;
3761 			}
3762 			block->status.hw = true;
3763 		}
3764 	}
3765 
3766 	return r;
3767 }
3768 
3769 /**
3770  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3771  *
3772  * @adev: amdgpu_device pointer
3773  *
3774  * First resume function for hardware IPs.  The list of all the hardware
3775  * IPs that make up the asic is walked and the resume callbacks are run for
3776  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3777  * after a suspend and updates the software state as necessary.  This
3778  * function is also used for restoring the GPU after a GPU reset.
3779  * Returns 0 on success, negative error code on failure.
3780  */
3781 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3782 {
3783 	int i, r;
3784 
3785 	for (i = 0; i < adev->num_ip_blocks; i++) {
3786 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3787 			continue;
3788 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3789 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3790 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3791 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3792 
3793 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3794 			if (r)
3795 				return r;
3796 		}
3797 	}
3798 
3799 	return 0;
3800 }
3801 
3802 /**
3803  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3804  *
3805  * @adev: amdgpu_device pointer
3806  *
3807  * Second resume function for hardware IPs.  The list of all the hardware
3808  * IPs that make up the asic is walked and the resume callbacks are run for
3809  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3810  * functional state after a suspend and updates the software state as
3811  * necessary.  This function is also used for restoring the GPU after a GPU
3812  * reset.
3813  * Returns 0 on success, negative error code on failure.
3814  */
3815 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3816 {
3817 	int i, r;
3818 
3819 	for (i = 0; i < adev->num_ip_blocks; i++) {
3820 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3821 			continue;
3822 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3823 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3824 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3825 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||
3826 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3827 			continue;
3828 		r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3829 		if (r)
3830 			return r;
3831 	}
3832 
3833 	return 0;
3834 }
3835 
3836 /**
3837  * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3838  *
3839  * @adev: amdgpu_device pointer
3840  *
3841  * Third resume function for hardware IPs.  The list of all the hardware
3842  * IPs that make up the asic is walked and the resume callbacks are run for
3843  * all DCE.  resume puts the hardware into a functional state after a suspend
3844  * and updates the software state as necessary.  This function is also used
3845  * for restoring the GPU after a GPU reset.
3846  *
3847  * Returns 0 on success, negative error code on failure.
3848  */
3849 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)
3850 {
3851 	int i, r;
3852 
3853 	for (i = 0; i < adev->num_ip_blocks; i++) {
3854 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3855 			continue;
3856 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
3857 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3858 			if (r)
3859 				return r;
3860 		}
3861 	}
3862 
3863 	return 0;
3864 }
3865 
3866 /**
3867  * amdgpu_device_ip_resume - run resume for hardware IPs
3868  *
3869  * @adev: amdgpu_device pointer
3870  *
3871  * Main resume function for hardware IPs.  The hardware IPs
3872  * are split into two resume functions because they are
3873  * also used in recovering from a GPU reset and some additional
3874  * steps need to be take between them.  In this case (S3/S4) they are
3875  * run sequentially.
3876  * Returns 0 on success, negative error code on failure.
3877  */
3878 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3879 {
3880 	int r;
3881 
3882 	r = amdgpu_device_ip_resume_phase1(adev);
3883 	if (r)
3884 		return r;
3885 
3886 	r = amdgpu_device_fw_loading(adev);
3887 	if (r)
3888 		return r;
3889 
3890 	r = amdgpu_device_ip_resume_phase2(adev);
3891 
3892 	if (adev->mman.buffer_funcs_ring->sched.ready)
3893 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
3894 
3895 	if (r)
3896 		return r;
3897 
3898 	amdgpu_fence_driver_hw_init(adev);
3899 
3900 	r = amdgpu_device_ip_resume_phase3(adev);
3901 
3902 	return r;
3903 }
3904 
3905 /**
3906  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3907  *
3908  * @adev: amdgpu_device pointer
3909  *
3910  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3911  */
3912 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3913 {
3914 	if (amdgpu_sriov_vf(adev)) {
3915 		if (adev->is_atom_fw) {
3916 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3917 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3918 		} else {
3919 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3920 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3921 		}
3922 
3923 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3924 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3925 	}
3926 }
3927 
3928 /**
3929  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3930  *
3931  * @asic_type: AMD asic type
3932  *
3933  * Check if there is DC (new modesetting infrastructre) support for an asic.
3934  * returns true if DC has support, false if not.
3935  */
3936 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3937 {
3938 	switch (asic_type) {
3939 #ifdef CONFIG_DRM_AMDGPU_SI
3940 	case CHIP_HAINAN:
3941 #endif
3942 	case CHIP_TOPAZ:
3943 		/* chips with no display hardware */
3944 		return false;
3945 #if defined(CONFIG_DRM_AMD_DC)
3946 	case CHIP_TAHITI:
3947 	case CHIP_PITCAIRN:
3948 	case CHIP_VERDE:
3949 	case CHIP_OLAND:
3950 		/*
3951 		 * We have systems in the wild with these ASICs that require
3952 		 * LVDS and VGA support which is not supported with DC.
3953 		 *
3954 		 * Fallback to the non-DC driver here by default so as not to
3955 		 * cause regressions.
3956 		 */
3957 #if defined(CONFIG_DRM_AMD_DC_SI)
3958 		return amdgpu_dc > 0;
3959 #else
3960 		return false;
3961 #endif
3962 	case CHIP_BONAIRE:
3963 	case CHIP_KAVERI:
3964 	case CHIP_KABINI:
3965 	case CHIP_MULLINS:
3966 		/*
3967 		 * We have systems in the wild with these ASICs that require
3968 		 * VGA support which is not supported with DC.
3969 		 *
3970 		 * Fallback to the non-DC driver here by default so as not to
3971 		 * cause regressions.
3972 		 */
3973 		return amdgpu_dc > 0;
3974 	default:
3975 		return amdgpu_dc != 0;
3976 #else
3977 	default:
3978 		if (amdgpu_dc > 0)
3979 			DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3980 		return false;
3981 #endif
3982 	}
3983 }
3984 
3985 /**
3986  * amdgpu_device_has_dc_support - check if dc is supported
3987  *
3988  * @adev: amdgpu_device pointer
3989  *
3990  * Returns true for supported, false for not supported
3991  */
3992 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3993 {
3994 	if (adev->enable_virtual_display ||
3995 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3996 		return false;
3997 
3998 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
3999 }
4000 
4001 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
4002 {
4003 	struct amdgpu_device *adev =
4004 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
4005 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4006 
4007 	/* It's a bug to not have a hive within this function */
4008 	if (WARN_ON(!hive))
4009 		return;
4010 
4011 	/*
4012 	 * Use task barrier to synchronize all xgmi reset works across the
4013 	 * hive. task_barrier_enter and task_barrier_exit will block
4014 	 * until all the threads running the xgmi reset works reach
4015 	 * those points. task_barrier_full will do both blocks.
4016 	 */
4017 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
4018 
4019 		task_barrier_enter(&hive->tb);
4020 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
4021 
4022 		if (adev->asic_reset_res)
4023 			goto fail;
4024 
4025 		task_barrier_exit(&hive->tb);
4026 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
4027 
4028 		if (adev->asic_reset_res)
4029 			goto fail;
4030 
4031 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
4032 	} else {
4033 
4034 		task_barrier_full(&hive->tb);
4035 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
4036 	}
4037 
4038 fail:
4039 	if (adev->asic_reset_res)
4040 		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
4041 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
4042 	amdgpu_put_xgmi_hive(hive);
4043 }
4044 
4045 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
4046 {
4047 	char *input = amdgpu_lockup_timeout;
4048 	char *timeout_setting = NULL;
4049 	int index = 0;
4050 	long timeout;
4051 	int ret = 0;
4052 
4053 	/*
4054 	 * By default timeout for non compute jobs is 10000
4055 	 * and 60000 for compute jobs.
4056 	 * In SR-IOV or passthrough mode, timeout for compute
4057 	 * jobs are 60000 by default.
4058 	 */
4059 	adev->gfx_timeout = msecs_to_jiffies(10000);
4060 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4061 	if (amdgpu_sriov_vf(adev))
4062 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
4063 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
4064 	else
4065 		adev->compute_timeout =  msecs_to_jiffies(60000);
4066 
4067 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4068 		while ((timeout_setting = strsep(&input, ",")) &&
4069 				strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4070 			ret = kstrtol(timeout_setting, 0, &timeout);
4071 			if (ret)
4072 				return ret;
4073 
4074 			if (timeout == 0) {
4075 				index++;
4076 				continue;
4077 			} else if (timeout < 0) {
4078 				timeout = MAX_SCHEDULE_TIMEOUT;
4079 				dev_warn(adev->dev, "lockup timeout disabled");
4080 				add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
4081 			} else {
4082 				timeout = msecs_to_jiffies(timeout);
4083 			}
4084 
4085 			switch (index++) {
4086 			case 0:
4087 				adev->gfx_timeout = timeout;
4088 				break;
4089 			case 1:
4090 				adev->compute_timeout = timeout;
4091 				break;
4092 			case 2:
4093 				adev->sdma_timeout = timeout;
4094 				break;
4095 			case 3:
4096 				adev->video_timeout = timeout;
4097 				break;
4098 			default:
4099 				break;
4100 			}
4101 		}
4102 		/*
4103 		 * There is only one value specified and
4104 		 * it should apply to all non-compute jobs.
4105 		 */
4106 		if (index == 1) {
4107 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4108 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
4109 				adev->compute_timeout = adev->gfx_timeout;
4110 		}
4111 	}
4112 
4113 	return ret;
4114 }
4115 
4116 /**
4117  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
4118  *
4119  * @adev: amdgpu_device pointer
4120  *
4121  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
4122  */
4123 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
4124 {
4125 	struct iommu_domain *domain;
4126 
4127 	domain = iommu_get_domain_for_dev(adev->dev);
4128 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
4129 		adev->ram_is_direct_mapped = true;
4130 }
4131 
4132 #if defined(CONFIG_HSA_AMD_P2P)
4133 /**
4134  * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
4135  *
4136  * @adev: amdgpu_device pointer
4137  *
4138  * return if IOMMU remapping bar address
4139  */
4140 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)
4141 {
4142 	struct iommu_domain *domain;
4143 
4144 	domain = iommu_get_domain_for_dev(adev->dev);
4145 	if (domain && (domain->type == IOMMU_DOMAIN_DMA ||
4146 		domain->type ==	IOMMU_DOMAIN_DMA_FQ))
4147 		return true;
4148 
4149 	return false;
4150 }
4151 #endif
4152 
4153 static const struct attribute *amdgpu_dev_attributes[] = {
4154 	&dev_attr_pcie_replay_count.attr,
4155 	NULL
4156 };
4157 
4158 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
4159 {
4160 	if (amdgpu_mcbp == 1)
4161 		adev->gfx.mcbp = true;
4162 	else if (amdgpu_mcbp == 0)
4163 		adev->gfx.mcbp = false;
4164 
4165 	if (amdgpu_sriov_vf(adev))
4166 		adev->gfx.mcbp = true;
4167 
4168 	if (adev->gfx.mcbp)
4169 		DRM_INFO("MCBP is enabled\n");
4170 }
4171 
4172 /**
4173  * amdgpu_device_init - initialize the driver
4174  *
4175  * @adev: amdgpu_device pointer
4176  * @flags: driver flags
4177  *
4178  * Initializes the driver info and hw (all asics).
4179  * Returns 0 for success or an error on failure.
4180  * Called at driver startup.
4181  */
4182 int amdgpu_device_init(struct amdgpu_device *adev,
4183 		       uint32_t flags)
4184 {
4185 	struct drm_device *ddev = adev_to_drm(adev);
4186 	struct pci_dev *pdev = adev->pdev;
4187 	int r, i;
4188 	bool px = false;
4189 	u32 max_MBps;
4190 	int tmp;
4191 
4192 	adev->shutdown = false;
4193 	adev->flags = flags;
4194 
4195 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
4196 		adev->asic_type = amdgpu_force_asic_type;
4197 	else
4198 		adev->asic_type = flags & AMD_ASIC_MASK;
4199 
4200 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
4201 	if (amdgpu_emu_mode == 1)
4202 		adev->usec_timeout *= 10;
4203 	adev->gmc.gart_size = 512 * 1024 * 1024;
4204 	adev->accel_working = false;
4205 	adev->num_rings = 0;
4206 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
4207 	adev->mman.buffer_funcs = NULL;
4208 	adev->mman.buffer_funcs_ring = NULL;
4209 	adev->vm_manager.vm_pte_funcs = NULL;
4210 	adev->vm_manager.vm_pte_num_scheds = 0;
4211 	adev->gmc.gmc_funcs = NULL;
4212 	adev->harvest_ip_mask = 0x0;
4213 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
4214 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
4215 
4216 	adev->smc_rreg = &amdgpu_invalid_rreg;
4217 	adev->smc_wreg = &amdgpu_invalid_wreg;
4218 	adev->pcie_rreg = &amdgpu_invalid_rreg;
4219 	adev->pcie_wreg = &amdgpu_invalid_wreg;
4220 	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
4221 	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
4222 	adev->pciep_rreg = &amdgpu_invalid_rreg;
4223 	adev->pciep_wreg = &amdgpu_invalid_wreg;
4224 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
4225 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
4226 	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
4227 	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
4228 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
4229 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
4230 	adev->didt_rreg = &amdgpu_invalid_rreg;
4231 	adev->didt_wreg = &amdgpu_invalid_wreg;
4232 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
4233 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
4234 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
4235 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
4236 
4237 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
4238 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
4239 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
4240 
4241 	/* mutex initialization are all done here so we
4242 	 * can recall function without having locking issues
4243 	 */
4244 	mutex_init(&adev->firmware.mutex);
4245 	mutex_init(&adev->pm.mutex);
4246 	mutex_init(&adev->gfx.gpu_clock_mutex);
4247 	mutex_init(&adev->srbm_mutex);
4248 	mutex_init(&adev->gfx.pipe_reserve_mutex);
4249 	mutex_init(&adev->gfx.gfx_off_mutex);
4250 	mutex_init(&adev->gfx.partition_mutex);
4251 	mutex_init(&adev->grbm_idx_mutex);
4252 	mutex_init(&adev->mn_lock);
4253 	mutex_init(&adev->virt.vf_errors.lock);
4254 	mutex_init(&adev->virt.rlcg_reg_lock);
4255 	hash_init(adev->mn_hash);
4256 	mutex_init(&adev->psp.mutex);
4257 	mutex_init(&adev->notifier_lock);
4258 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
4259 	mutex_init(&adev->benchmark_mutex);
4260 	mutex_init(&adev->gfx.reset_sem_mutex);
4261 	/* Initialize the mutex for cleaner shader isolation between GFX and compute processes */
4262 	mutex_init(&adev->enforce_isolation_mutex);
4263 	mutex_init(&adev->gfx.kfd_sch_mutex);
4264 
4265 	amdgpu_device_init_apu_flags(adev);
4266 
4267 	r = amdgpu_device_check_arguments(adev);
4268 	if (r)
4269 		return r;
4270 
4271 	spin_lock_init(&adev->mmio_idx_lock);
4272 	spin_lock_init(&adev->smc_idx_lock);
4273 	spin_lock_init(&adev->pcie_idx_lock);
4274 	spin_lock_init(&adev->uvd_ctx_idx_lock);
4275 	spin_lock_init(&adev->didt_idx_lock);
4276 	spin_lock_init(&adev->gc_cac_idx_lock);
4277 	spin_lock_init(&adev->se_cac_idx_lock);
4278 	spin_lock_init(&adev->audio_endpt_idx_lock);
4279 	spin_lock_init(&adev->mm_stats.lock);
4280 	spin_lock_init(&adev->wb.lock);
4281 
4282 	INIT_LIST_HEAD(&adev->reset_list);
4283 
4284 	INIT_LIST_HEAD(&adev->ras_list);
4285 
4286 	INIT_LIST_HEAD(&adev->pm.od_kobj_list);
4287 
4288 	INIT_DELAYED_WORK(&adev->delayed_init_work,
4289 			  amdgpu_device_delayed_init_work_handler);
4290 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
4291 			  amdgpu_device_delay_enable_gfx_off);
4292 	/*
4293 	 * Initialize the enforce_isolation work structures for each XCP
4294 	 * partition.  This work handler is responsible for enforcing shader
4295 	 * isolation on AMD GPUs.  It counts the number of emitted fences for
4296 	 * each GFX and compute ring.  If there are any fences, it schedules
4297 	 * the `enforce_isolation_work` to be run after a delay.  If there are
4298 	 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the
4299 	 * runqueue.
4300 	 */
4301 	for (i = 0; i < MAX_XCP; i++) {
4302 		INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,
4303 				  amdgpu_gfx_enforce_isolation_handler);
4304 		adev->gfx.enforce_isolation[i].adev = adev;
4305 		adev->gfx.enforce_isolation[i].xcp_id = i;
4306 	}
4307 
4308 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4309 
4310 	adev->gfx.gfx_off_req_count = 1;
4311 	adev->gfx.gfx_off_residency = 0;
4312 	adev->gfx.gfx_off_entrycount = 0;
4313 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4314 
4315 	atomic_set(&adev->throttling_logging_enabled, 1);
4316 	/*
4317 	 * If throttling continues, logging will be performed every minute
4318 	 * to avoid log flooding. "-1" is subtracted since the thermal
4319 	 * throttling interrupt comes every second. Thus, the total logging
4320 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
4321 	 * for throttling interrupt) = 60 seconds.
4322 	 */
4323 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4324 	ratelimit_state_init(&adev->virt.ras_telemetry_rs, 5 * HZ, 1);
4325 
4326 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4327 	ratelimit_set_flags(&adev->virt.ras_telemetry_rs, RATELIMIT_MSG_ON_RELEASE);
4328 
4329 	/* Registers mapping */
4330 	/* TODO: block userspace mapping of io register */
4331 	if (adev->asic_type >= CHIP_BONAIRE) {
4332 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4333 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4334 	} else {
4335 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4336 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4337 	}
4338 
4339 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4340 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4341 
4342 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4343 	if (!adev->rmmio)
4344 		return -ENOMEM;
4345 
4346 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4347 	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4348 
4349 	/*
4350 	 * Reset domain needs to be present early, before XGMI hive discovered
4351 	 * (if any) and initialized to use reset sem and in_gpu reset flag
4352 	 * early on during init and before calling to RREG32.
4353 	 */
4354 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4355 	if (!adev->reset_domain)
4356 		return -ENOMEM;
4357 
4358 	/* detect hw virtualization here */
4359 	amdgpu_detect_virtualization(adev);
4360 
4361 	amdgpu_device_get_pcie_info(adev);
4362 
4363 	r = amdgpu_device_get_job_timeout_settings(adev);
4364 	if (r) {
4365 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4366 		return r;
4367 	}
4368 
4369 	amdgpu_device_set_mcbp(adev);
4370 
4371 	/*
4372 	 * By default, use default mode where all blocks are expected to be
4373 	 * initialized. At present a 'swinit' of blocks is required to be
4374 	 * completed before the need for a different level is detected.
4375 	 */
4376 	amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
4377 	/* early init functions */
4378 	r = amdgpu_device_ip_early_init(adev);
4379 	if (r)
4380 		return r;
4381 
4382 	/* Get rid of things like offb */
4383 	r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);
4384 	if (r)
4385 		return r;
4386 
4387 	/* Enable TMZ based on IP_VERSION */
4388 	amdgpu_gmc_tmz_set(adev);
4389 
4390 	if (amdgpu_sriov_vf(adev) &&
4391 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
4392 		/* VF MMIO access (except mailbox range) from CPU
4393 		 * will be blocked during sriov runtime
4394 		 */
4395 		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
4396 
4397 	amdgpu_gmc_noretry_set(adev);
4398 	/* Need to get xgmi info early to decide the reset behavior*/
4399 	if (adev->gmc.xgmi.supported) {
4400 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
4401 		if (r)
4402 			return r;
4403 	}
4404 
4405 	/* enable PCIE atomic ops */
4406 	if (amdgpu_sriov_vf(adev)) {
4407 		if (adev->virt.fw_reserve.p_pf2vf)
4408 			adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4409 						      adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4410 				(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4411 	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4412 	 * internal path natively support atomics, set have_atomics_support to true.
4413 	 */
4414 	} else if ((adev->flags & AMD_IS_APU) &&
4415 		   (amdgpu_ip_version(adev, GC_HWIP, 0) >
4416 		    IP_VERSION(9, 0, 0))) {
4417 		adev->have_atomics_support = true;
4418 	} else {
4419 		adev->have_atomics_support =
4420 			!pci_enable_atomic_ops_to_root(adev->pdev,
4421 					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4422 					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4423 	}
4424 
4425 	if (!adev->have_atomics_support)
4426 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4427 
4428 	/* doorbell bar mapping and doorbell index init*/
4429 	amdgpu_doorbell_init(adev);
4430 
4431 	if (amdgpu_emu_mode == 1) {
4432 		/* post the asic on emulation mode */
4433 		emu_soc_asic_init(adev);
4434 		goto fence_driver_init;
4435 	}
4436 
4437 	amdgpu_reset_init(adev);
4438 
4439 	/* detect if we are with an SRIOV vbios */
4440 	if (adev->bios)
4441 		amdgpu_device_detect_sriov_bios(adev);
4442 
4443 	/* check if we need to reset the asic
4444 	 *  E.g., driver was not cleanly unloaded previously, etc.
4445 	 */
4446 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4447 		if (adev->gmc.xgmi.num_physical_nodes) {
4448 			dev_info(adev->dev, "Pending hive reset.\n");
4449 			amdgpu_set_init_level(adev,
4450 					      AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
4451 		} else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) &&
4452 				   !amdgpu_device_has_display_hardware(adev)) {
4453 					r = psp_gpu_reset(adev);
4454 		} else {
4455 				tmp = amdgpu_reset_method;
4456 				/* It should do a default reset when loading or reloading the driver,
4457 				 * regardless of the module parameter reset_method.
4458 				 */
4459 				amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4460 				r = amdgpu_asic_reset(adev);
4461 				amdgpu_reset_method = tmp;
4462 		}
4463 
4464 		if (r) {
4465 		  dev_err(adev->dev, "asic reset on init failed\n");
4466 		  goto failed;
4467 		}
4468 	}
4469 
4470 	/* Post card if necessary */
4471 	if (amdgpu_device_need_post(adev)) {
4472 		if (!adev->bios) {
4473 			dev_err(adev->dev, "no vBIOS found\n");
4474 			r = -EINVAL;
4475 			goto failed;
4476 		}
4477 		DRM_INFO("GPU posting now...\n");
4478 		r = amdgpu_device_asic_init(adev);
4479 		if (r) {
4480 			dev_err(adev->dev, "gpu post error!\n");
4481 			goto failed;
4482 		}
4483 	}
4484 
4485 	if (adev->bios) {
4486 		if (adev->is_atom_fw) {
4487 			/* Initialize clocks */
4488 			r = amdgpu_atomfirmware_get_clock_info(adev);
4489 			if (r) {
4490 				dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4491 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4492 				goto failed;
4493 			}
4494 		} else {
4495 			/* Initialize clocks */
4496 			r = amdgpu_atombios_get_clock_info(adev);
4497 			if (r) {
4498 				dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4499 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4500 				goto failed;
4501 			}
4502 			/* init i2c buses */
4503 			amdgpu_i2c_init(adev);
4504 		}
4505 	}
4506 
4507 fence_driver_init:
4508 	/* Fence driver */
4509 	r = amdgpu_fence_driver_sw_init(adev);
4510 	if (r) {
4511 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4512 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4513 		goto failed;
4514 	}
4515 
4516 	/* init the mode config */
4517 	drm_mode_config_init(adev_to_drm(adev));
4518 
4519 	r = amdgpu_device_ip_init(adev);
4520 	if (r) {
4521 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4522 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4523 		goto release_ras_con;
4524 	}
4525 
4526 	amdgpu_fence_driver_hw_init(adev);
4527 
4528 	dev_info(adev->dev,
4529 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4530 			adev->gfx.config.max_shader_engines,
4531 			adev->gfx.config.max_sh_per_se,
4532 			adev->gfx.config.max_cu_per_sh,
4533 			adev->gfx.cu_info.number);
4534 
4535 	adev->accel_working = true;
4536 
4537 	amdgpu_vm_check_compute_bug(adev);
4538 
4539 	/* Initialize the buffer migration limit. */
4540 	if (amdgpu_moverate >= 0)
4541 		max_MBps = amdgpu_moverate;
4542 	else
4543 		max_MBps = 8; /* Allow 8 MB/s. */
4544 	/* Get a log2 for easy divisions. */
4545 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4546 
4547 	/*
4548 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4549 	 * Otherwise the mgpu fan boost feature will be skipped due to the
4550 	 * gpu instance is counted less.
4551 	 */
4552 	amdgpu_register_gpu_instance(adev);
4553 
4554 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4555 	 * explicit gating rather than handling it automatically.
4556 	 */
4557 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4558 		r = amdgpu_device_ip_late_init(adev);
4559 		if (r) {
4560 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4561 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4562 			goto release_ras_con;
4563 		}
4564 		/* must succeed. */
4565 		amdgpu_ras_resume(adev);
4566 		queue_delayed_work(system_wq, &adev->delayed_init_work,
4567 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
4568 	}
4569 
4570 	if (amdgpu_sriov_vf(adev)) {
4571 		amdgpu_virt_release_full_gpu(adev, true);
4572 		flush_delayed_work(&adev->delayed_init_work);
4573 	}
4574 
4575 	/*
4576 	 * Place those sysfs registering after `late_init`. As some of those
4577 	 * operations performed in `late_init` might affect the sysfs
4578 	 * interfaces creating.
4579 	 */
4580 	r = amdgpu_atombios_sysfs_init(adev);
4581 	if (r)
4582 		drm_err(&adev->ddev,
4583 			"registering atombios sysfs failed (%d).\n", r);
4584 
4585 	r = amdgpu_pm_sysfs_init(adev);
4586 	if (r)
4587 		DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4588 
4589 	r = amdgpu_ucode_sysfs_init(adev);
4590 	if (r) {
4591 		adev->ucode_sysfs_en = false;
4592 		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4593 	} else
4594 		adev->ucode_sysfs_en = true;
4595 
4596 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4597 	if (r)
4598 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
4599 
4600 	r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4601 	if (r)
4602 		dev_err(adev->dev,
4603 			"Could not create amdgpu board attributes\n");
4604 
4605 	amdgpu_fru_sysfs_init(adev);
4606 	amdgpu_reg_state_sysfs_init(adev);
4607 	amdgpu_xcp_cfg_sysfs_init(adev);
4608 
4609 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4610 		r = amdgpu_pmu_init(adev);
4611 	if (r)
4612 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4613 
4614 	/* Have stored pci confspace at hand for restore in sudden PCI error */
4615 	if (amdgpu_device_cache_pci_state(adev->pdev))
4616 		pci_restore_state(pdev);
4617 
4618 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4619 	/* this will fail for cards that aren't VGA class devices, just
4620 	 * ignore it
4621 	 */
4622 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4623 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4624 
4625 	px = amdgpu_device_supports_px(ddev);
4626 
4627 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4628 				apple_gmux_detect(NULL, NULL)))
4629 		vga_switcheroo_register_client(adev->pdev,
4630 					       &amdgpu_switcheroo_ops, px);
4631 
4632 	if (px)
4633 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4634 
4635 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
4636 		amdgpu_xgmi_reset_on_init(adev);
4637 
4638 	amdgpu_device_check_iommu_direct_map(adev);
4639 
4640 	adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
4641 	r = register_pm_notifier(&adev->pm_nb);
4642 	if (r)
4643 		goto failed;
4644 
4645 	return 0;
4646 
4647 release_ras_con:
4648 	if (amdgpu_sriov_vf(adev))
4649 		amdgpu_virt_release_full_gpu(adev, true);
4650 
4651 	/* failed in exclusive mode due to timeout */
4652 	if (amdgpu_sriov_vf(adev) &&
4653 		!amdgpu_sriov_runtime(adev) &&
4654 		amdgpu_virt_mmio_blocked(adev) &&
4655 		!amdgpu_virt_wait_reset(adev)) {
4656 		dev_err(adev->dev, "VF exclusive mode timeout\n");
4657 		/* Don't send request since VF is inactive. */
4658 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4659 		adev->virt.ops = NULL;
4660 		r = -EAGAIN;
4661 	}
4662 	amdgpu_release_ras_context(adev);
4663 
4664 failed:
4665 	amdgpu_vf_error_trans_all(adev);
4666 
4667 	return r;
4668 }
4669 
4670 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4671 {
4672 
4673 	/* Clear all CPU mappings pointing to this device */
4674 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4675 
4676 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4677 	amdgpu_doorbell_fini(adev);
4678 
4679 	iounmap(adev->rmmio);
4680 	adev->rmmio = NULL;
4681 	if (adev->mman.aper_base_kaddr)
4682 		iounmap(adev->mman.aper_base_kaddr);
4683 	adev->mman.aper_base_kaddr = NULL;
4684 
4685 	/* Memory manager related */
4686 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4687 		arch_phys_wc_del(adev->gmc.vram_mtrr);
4688 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4689 	}
4690 }
4691 
4692 /**
4693  * amdgpu_device_fini_hw - tear down the driver
4694  *
4695  * @adev: amdgpu_device pointer
4696  *
4697  * Tear down the driver info (all asics).
4698  * Called at driver shutdown.
4699  */
4700 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4701 {
4702 	dev_info(adev->dev, "amdgpu: finishing device.\n");
4703 	flush_delayed_work(&adev->delayed_init_work);
4704 
4705 	if (adev->mman.initialized)
4706 		drain_workqueue(adev->mman.bdev.wq);
4707 	adev->shutdown = true;
4708 
4709 	unregister_pm_notifier(&adev->pm_nb);
4710 
4711 	/* make sure IB test finished before entering exclusive mode
4712 	 * to avoid preemption on IB test
4713 	 */
4714 	if (amdgpu_sriov_vf(adev)) {
4715 		amdgpu_virt_request_full_gpu(adev, false);
4716 		amdgpu_virt_fini_data_exchange(adev);
4717 	}
4718 
4719 	/* disable all interrupts */
4720 	amdgpu_irq_disable_all(adev);
4721 	if (adev->mode_info.mode_config_initialized) {
4722 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4723 			drm_helper_force_disable_all(adev_to_drm(adev));
4724 		else
4725 			drm_atomic_helper_shutdown(adev_to_drm(adev));
4726 	}
4727 	amdgpu_fence_driver_hw_fini(adev);
4728 
4729 	if (adev->pm.sysfs_initialized)
4730 		amdgpu_pm_sysfs_fini(adev);
4731 	if (adev->ucode_sysfs_en)
4732 		amdgpu_ucode_sysfs_fini(adev);
4733 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4734 	amdgpu_fru_sysfs_fini(adev);
4735 
4736 	amdgpu_reg_state_sysfs_fini(adev);
4737 	amdgpu_xcp_cfg_sysfs_fini(adev);
4738 
4739 	/* disable ras feature must before hw fini */
4740 	amdgpu_ras_pre_fini(adev);
4741 
4742 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4743 
4744 	amdgpu_device_ip_fini_early(adev);
4745 
4746 	amdgpu_irq_fini_hw(adev);
4747 
4748 	if (adev->mman.initialized)
4749 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4750 
4751 	amdgpu_gart_dummy_page_fini(adev);
4752 
4753 	if (drm_dev_is_unplugged(adev_to_drm(adev)))
4754 		amdgpu_device_unmap_mmio(adev);
4755 
4756 }
4757 
4758 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4759 {
4760 	int idx;
4761 	bool px;
4762 
4763 	amdgpu_device_ip_fini(adev);
4764 	amdgpu_fence_driver_sw_fini(adev);
4765 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4766 	adev->accel_working = false;
4767 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4768 
4769 	amdgpu_reset_fini(adev);
4770 
4771 	/* free i2c buses */
4772 	amdgpu_i2c_fini(adev);
4773 
4774 	if (adev->bios) {
4775 		if (amdgpu_emu_mode != 1)
4776 			amdgpu_atombios_fini(adev);
4777 		amdgpu_bios_release(adev);
4778 	}
4779 
4780 	kfree(adev->fru_info);
4781 	adev->fru_info = NULL;
4782 
4783 	px = amdgpu_device_supports_px(adev_to_drm(adev));
4784 
4785 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4786 				apple_gmux_detect(NULL, NULL)))
4787 		vga_switcheroo_unregister_client(adev->pdev);
4788 
4789 	if (px)
4790 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4791 
4792 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4793 		vga_client_unregister(adev->pdev);
4794 
4795 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4796 
4797 		iounmap(adev->rmmio);
4798 		adev->rmmio = NULL;
4799 		amdgpu_doorbell_fini(adev);
4800 		drm_dev_exit(idx);
4801 	}
4802 
4803 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4804 		amdgpu_pmu_fini(adev);
4805 	if (adev->mman.discovery_bin)
4806 		amdgpu_discovery_fini(adev);
4807 
4808 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4809 	adev->reset_domain = NULL;
4810 
4811 	kfree(adev->pci_state);
4812 
4813 }
4814 
4815 /**
4816  * amdgpu_device_evict_resources - evict device resources
4817  * @adev: amdgpu device object
4818  *
4819  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4820  * of the vram memory type. Mainly used for evicting device resources
4821  * at suspend time.
4822  *
4823  */
4824 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4825 {
4826 	int ret;
4827 
4828 	/* No need to evict vram on APUs unless going to S4 */
4829 	if (!adev->in_s4 && (adev->flags & AMD_IS_APU))
4830 		return 0;
4831 
4832 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4833 	if (ret)
4834 		DRM_WARN("evicting device resources failed\n");
4835 	return ret;
4836 }
4837 
4838 /*
4839  * Suspend & resume.
4840  */
4841 /**
4842  * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events
4843  * @nb: notifier block
4844  * @mode: suspend mode
4845  * @data: data
4846  *
4847  * This function is called when the system is about to suspend or hibernate.
4848  * It is used to evict resources from the device before the system goes to
4849  * sleep while there is still access to swap.
4850  */
4851 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
4852 				     void *data)
4853 {
4854 	struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);
4855 	int r;
4856 
4857 	switch (mode) {
4858 	case PM_HIBERNATION_PREPARE:
4859 		adev->in_s4 = true;
4860 		fallthrough;
4861 	case PM_SUSPEND_PREPARE:
4862 		r = amdgpu_device_evict_resources(adev);
4863 		/*
4864 		 * This is considered non-fatal at this time because
4865 		 * amdgpu_device_prepare() will also fatally evict resources.
4866 		 * See https://gitlab.freedesktop.org/drm/amd/-/issues/3781
4867 		 */
4868 		if (r)
4869 			drm_warn(adev_to_drm(adev), "Failed to evict resources, freeze active processes if problems occur: %d\n", r);
4870 		break;
4871 	}
4872 
4873 	return NOTIFY_DONE;
4874 }
4875 
4876 /**
4877  * amdgpu_device_prepare - prepare for device suspend
4878  *
4879  * @dev: drm dev pointer
4880  *
4881  * Prepare to put the hw in the suspend state (all asics).
4882  * Returns 0 for success or an error on failure.
4883  * Called at driver suspend.
4884  */
4885 int amdgpu_device_prepare(struct drm_device *dev)
4886 {
4887 	struct amdgpu_device *adev = drm_to_adev(dev);
4888 	int i, r;
4889 
4890 	amdgpu_choose_low_power_state(adev);
4891 
4892 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4893 		return 0;
4894 
4895 	/* Evict the majority of BOs before starting suspend sequence */
4896 	r = amdgpu_device_evict_resources(adev);
4897 	if (r)
4898 		goto unprepare;
4899 
4900 	flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4901 
4902 	for (i = 0; i < adev->num_ip_blocks; i++) {
4903 		if (!adev->ip_blocks[i].status.valid)
4904 			continue;
4905 		if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4906 			continue;
4907 		r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);
4908 		if (r)
4909 			goto unprepare;
4910 	}
4911 
4912 	return 0;
4913 
4914 unprepare:
4915 	adev->in_s0ix = adev->in_s3 = adev->in_s4 = false;
4916 
4917 	return r;
4918 }
4919 
4920 /**
4921  * amdgpu_device_suspend - initiate device suspend
4922  *
4923  * @dev: drm dev pointer
4924  * @notify_clients: notify in-kernel DRM clients
4925  *
4926  * Puts the hw in the suspend state (all asics).
4927  * Returns 0 for success or an error on failure.
4928  * Called at driver suspend.
4929  */
4930 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
4931 {
4932 	struct amdgpu_device *adev = drm_to_adev(dev);
4933 	int r = 0;
4934 
4935 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4936 		return 0;
4937 
4938 	adev->in_suspend = true;
4939 
4940 	if (amdgpu_sriov_vf(adev)) {
4941 		amdgpu_virt_fini_data_exchange(adev);
4942 		r = amdgpu_virt_request_full_gpu(adev, false);
4943 		if (r)
4944 			return r;
4945 	}
4946 
4947 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4948 		DRM_WARN("smart shift update failed\n");
4949 
4950 	if (notify_clients)
4951 		drm_client_dev_suspend(adev_to_drm(adev), false);
4952 
4953 	cancel_delayed_work_sync(&adev->delayed_init_work);
4954 
4955 	amdgpu_ras_suspend(adev);
4956 
4957 	amdgpu_device_ip_suspend_phase1(adev);
4958 
4959 	if (!adev->in_s0ix)
4960 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4961 
4962 	r = amdgpu_device_evict_resources(adev);
4963 	if (r)
4964 		return r;
4965 
4966 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4967 
4968 	amdgpu_fence_driver_hw_fini(adev);
4969 
4970 	amdgpu_device_ip_suspend_phase2(adev);
4971 
4972 	if (amdgpu_sriov_vf(adev))
4973 		amdgpu_virt_release_full_gpu(adev, false);
4974 
4975 	r = amdgpu_dpm_notify_rlc_state(adev, false);
4976 	if (r)
4977 		return r;
4978 
4979 	return 0;
4980 }
4981 
4982 /**
4983  * amdgpu_device_resume - initiate device resume
4984  *
4985  * @dev: drm dev pointer
4986  * @notify_clients: notify in-kernel DRM clients
4987  *
4988  * Bring the hw back to operating state (all asics).
4989  * Returns 0 for success or an error on failure.
4990  * Called at driver resume.
4991  */
4992 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
4993 {
4994 	struct amdgpu_device *adev = drm_to_adev(dev);
4995 	int r = 0;
4996 
4997 	if (amdgpu_sriov_vf(adev)) {
4998 		r = amdgpu_virt_request_full_gpu(adev, true);
4999 		if (r)
5000 			return r;
5001 	}
5002 
5003 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
5004 		return 0;
5005 
5006 	if (adev->in_s0ix)
5007 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
5008 
5009 	/* post card */
5010 	if (amdgpu_device_need_post(adev)) {
5011 		r = amdgpu_device_asic_init(adev);
5012 		if (r)
5013 			dev_err(adev->dev, "amdgpu asic init failed\n");
5014 	}
5015 
5016 	r = amdgpu_device_ip_resume(adev);
5017 
5018 	if (r) {
5019 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
5020 		goto exit;
5021 	}
5022 
5023 	if (!adev->in_s0ix) {
5024 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
5025 		if (r)
5026 			goto exit;
5027 	}
5028 
5029 	r = amdgpu_device_ip_late_init(adev);
5030 	if (r)
5031 		goto exit;
5032 
5033 	queue_delayed_work(system_wq, &adev->delayed_init_work,
5034 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
5035 exit:
5036 	if (amdgpu_sriov_vf(adev)) {
5037 		amdgpu_virt_init_data_exchange(adev);
5038 		amdgpu_virt_release_full_gpu(adev, true);
5039 	}
5040 
5041 	if (r)
5042 		return r;
5043 
5044 	/* Make sure IB tests flushed */
5045 	flush_delayed_work(&adev->delayed_init_work);
5046 
5047 	if (notify_clients)
5048 		drm_client_dev_resume(adev_to_drm(adev), false);
5049 
5050 	amdgpu_ras_resume(adev);
5051 
5052 	if (adev->mode_info.num_crtc) {
5053 		/*
5054 		 * Most of the connector probing functions try to acquire runtime pm
5055 		 * refs to ensure that the GPU is powered on when connector polling is
5056 		 * performed. Since we're calling this from a runtime PM callback,
5057 		 * trying to acquire rpm refs will cause us to deadlock.
5058 		 *
5059 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
5060 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
5061 		 */
5062 #ifdef CONFIG_PM
5063 		dev->dev->power.disable_depth++;
5064 #endif
5065 		if (!adev->dc_enabled)
5066 			drm_helper_hpd_irq_event(dev);
5067 		else
5068 			drm_kms_helper_hotplug_event(dev);
5069 #ifdef CONFIG_PM
5070 		dev->dev->power.disable_depth--;
5071 #endif
5072 	}
5073 	adev->in_suspend = false;
5074 
5075 	if (adev->enable_mes)
5076 		amdgpu_mes_self_test(adev);
5077 
5078 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
5079 		DRM_WARN("smart shift update failed\n");
5080 
5081 	return 0;
5082 }
5083 
5084 /**
5085  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
5086  *
5087  * @adev: amdgpu_device pointer
5088  *
5089  * The list of all the hardware IPs that make up the asic is walked and
5090  * the check_soft_reset callbacks are run.  check_soft_reset determines
5091  * if the asic is still hung or not.
5092  * Returns true if any of the IPs are still in a hung state, false if not.
5093  */
5094 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
5095 {
5096 	int i;
5097 	bool asic_hang = false;
5098 
5099 	if (amdgpu_sriov_vf(adev))
5100 		return true;
5101 
5102 	if (amdgpu_asic_need_full_reset(adev))
5103 		return true;
5104 
5105 	for (i = 0; i < adev->num_ip_blocks; i++) {
5106 		if (!adev->ip_blocks[i].status.valid)
5107 			continue;
5108 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
5109 			adev->ip_blocks[i].status.hang =
5110 				adev->ip_blocks[i].version->funcs->check_soft_reset(
5111 					&adev->ip_blocks[i]);
5112 		if (adev->ip_blocks[i].status.hang) {
5113 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
5114 			asic_hang = true;
5115 		}
5116 	}
5117 	return asic_hang;
5118 }
5119 
5120 /**
5121  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
5122  *
5123  * @adev: amdgpu_device pointer
5124  *
5125  * The list of all the hardware IPs that make up the asic is walked and the
5126  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
5127  * handles any IP specific hardware or software state changes that are
5128  * necessary for a soft reset to succeed.
5129  * Returns 0 on success, negative error code on failure.
5130  */
5131 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
5132 {
5133 	int i, r = 0;
5134 
5135 	for (i = 0; i < adev->num_ip_blocks; i++) {
5136 		if (!adev->ip_blocks[i].status.valid)
5137 			continue;
5138 		if (adev->ip_blocks[i].status.hang &&
5139 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
5140 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]);
5141 			if (r)
5142 				return r;
5143 		}
5144 	}
5145 
5146 	return 0;
5147 }
5148 
5149 /**
5150  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
5151  *
5152  * @adev: amdgpu_device pointer
5153  *
5154  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
5155  * reset is necessary to recover.
5156  * Returns true if a full asic reset is required, false if not.
5157  */
5158 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
5159 {
5160 	int i;
5161 
5162 	if (amdgpu_asic_need_full_reset(adev))
5163 		return true;
5164 
5165 	for (i = 0; i < adev->num_ip_blocks; i++) {
5166 		if (!adev->ip_blocks[i].status.valid)
5167 			continue;
5168 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
5169 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
5170 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
5171 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
5172 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
5173 			if (adev->ip_blocks[i].status.hang) {
5174 				dev_info(adev->dev, "Some block need full reset!\n");
5175 				return true;
5176 			}
5177 		}
5178 	}
5179 	return false;
5180 }
5181 
5182 /**
5183  * amdgpu_device_ip_soft_reset - do a soft reset
5184  *
5185  * @adev: amdgpu_device pointer
5186  *
5187  * The list of all the hardware IPs that make up the asic is walked and the
5188  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
5189  * IP specific hardware or software state changes that are necessary to soft
5190  * reset the IP.
5191  * Returns 0 on success, negative error code on failure.
5192  */
5193 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
5194 {
5195 	int i, r = 0;
5196 
5197 	for (i = 0; i < adev->num_ip_blocks; i++) {
5198 		if (!adev->ip_blocks[i].status.valid)
5199 			continue;
5200 		if (adev->ip_blocks[i].status.hang &&
5201 		    adev->ip_blocks[i].version->funcs->soft_reset) {
5202 			r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]);
5203 			if (r)
5204 				return r;
5205 		}
5206 	}
5207 
5208 	return 0;
5209 }
5210 
5211 /**
5212  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
5213  *
5214  * @adev: amdgpu_device pointer
5215  *
5216  * The list of all the hardware IPs that make up the asic is walked and the
5217  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
5218  * handles any IP specific hardware or software state changes that are
5219  * necessary after the IP has been soft reset.
5220  * Returns 0 on success, negative error code on failure.
5221  */
5222 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
5223 {
5224 	int i, r = 0;
5225 
5226 	for (i = 0; i < adev->num_ip_blocks; i++) {
5227 		if (!adev->ip_blocks[i].status.valid)
5228 			continue;
5229 		if (adev->ip_blocks[i].status.hang &&
5230 		    adev->ip_blocks[i].version->funcs->post_soft_reset)
5231 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]);
5232 		if (r)
5233 			return r;
5234 	}
5235 
5236 	return 0;
5237 }
5238 
5239 /**
5240  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
5241  *
5242  * @adev: amdgpu_device pointer
5243  * @reset_context: amdgpu reset context pointer
5244  *
5245  * do VF FLR and reinitialize Asic
5246  * return 0 means succeeded otherwise failed
5247  */
5248 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
5249 				     struct amdgpu_reset_context *reset_context)
5250 {
5251 	int r;
5252 	struct amdgpu_hive_info *hive = NULL;
5253 
5254 	if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) {
5255 		if (!amdgpu_ras_get_fed_status(adev))
5256 			amdgpu_virt_ready_to_reset(adev);
5257 		amdgpu_virt_wait_reset(adev);
5258 		clear_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5259 		r = amdgpu_virt_request_full_gpu(adev, true);
5260 	} else {
5261 		r = amdgpu_virt_reset_gpu(adev);
5262 	}
5263 	if (r)
5264 		return r;
5265 
5266 	amdgpu_ras_clear_err_state(adev);
5267 	amdgpu_irq_gpu_reset_resume_helper(adev);
5268 
5269 	/* some sw clean up VF needs to do before recover */
5270 	amdgpu_virt_post_reset(adev);
5271 
5272 	/* Resume IP prior to SMC */
5273 	r = amdgpu_device_ip_reinit_early_sriov(adev);
5274 	if (r)
5275 		return r;
5276 
5277 	amdgpu_virt_init_data_exchange(adev);
5278 
5279 	r = amdgpu_device_fw_loading(adev);
5280 	if (r)
5281 		return r;
5282 
5283 	/* now we are okay to resume SMC/CP/SDMA */
5284 	r = amdgpu_device_ip_reinit_late_sriov(adev);
5285 	if (r)
5286 		return r;
5287 
5288 	hive = amdgpu_get_xgmi_hive(adev);
5289 	/* Update PSP FW topology after reset */
5290 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5291 		r = amdgpu_xgmi_update_topology(hive, adev);
5292 	if (hive)
5293 		amdgpu_put_xgmi_hive(hive);
5294 	if (r)
5295 		return r;
5296 
5297 	r = amdgpu_ib_ring_tests(adev);
5298 	if (r)
5299 		return r;
5300 
5301 	if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)
5302 		amdgpu_inc_vram_lost(adev);
5303 
5304 	/* need to be called during full access so we can't do it later like
5305 	 * bare-metal does.
5306 	 */
5307 	amdgpu_amdkfd_post_reset(adev);
5308 	amdgpu_virt_release_full_gpu(adev, true);
5309 
5310 	/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5311 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
5312 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
5313 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
5314 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5315 		amdgpu_ras_resume(adev);
5316 
5317 	amdgpu_virt_ras_telemetry_post_reset(adev);
5318 
5319 	return 0;
5320 }
5321 
5322 /**
5323  * amdgpu_device_has_job_running - check if there is any unfinished job
5324  *
5325  * @adev: amdgpu_device pointer
5326  *
5327  * check if there is any job running on the device when guest driver receives
5328  * FLR notification from host driver. If there are still jobs running, then
5329  * the guest driver will not respond the FLR reset. Instead, let the job hit
5330  * the timeout and guest driver then issue the reset request.
5331  */
5332 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5333 {
5334 	int i;
5335 
5336 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5337 		struct amdgpu_ring *ring = adev->rings[i];
5338 
5339 		if (!amdgpu_ring_sched_ready(ring))
5340 			continue;
5341 
5342 		if (amdgpu_fence_count_emitted(ring))
5343 			return true;
5344 	}
5345 	return false;
5346 }
5347 
5348 /**
5349  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5350  *
5351  * @adev: amdgpu_device pointer
5352  *
5353  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5354  * a hung GPU.
5355  */
5356 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5357 {
5358 
5359 	if (amdgpu_gpu_recovery == 0)
5360 		goto disabled;
5361 
5362 	/* Skip soft reset check in fatal error mode */
5363 	if (!amdgpu_ras_is_poison_mode_supported(adev))
5364 		return true;
5365 
5366 	if (amdgpu_sriov_vf(adev))
5367 		return true;
5368 
5369 	if (amdgpu_gpu_recovery == -1) {
5370 		switch (adev->asic_type) {
5371 #ifdef CONFIG_DRM_AMDGPU_SI
5372 		case CHIP_VERDE:
5373 		case CHIP_TAHITI:
5374 		case CHIP_PITCAIRN:
5375 		case CHIP_OLAND:
5376 		case CHIP_HAINAN:
5377 #endif
5378 #ifdef CONFIG_DRM_AMDGPU_CIK
5379 		case CHIP_KAVERI:
5380 		case CHIP_KABINI:
5381 		case CHIP_MULLINS:
5382 #endif
5383 		case CHIP_CARRIZO:
5384 		case CHIP_STONEY:
5385 		case CHIP_CYAN_SKILLFISH:
5386 			goto disabled;
5387 		default:
5388 			break;
5389 		}
5390 	}
5391 
5392 	return true;
5393 
5394 disabled:
5395 		dev_info(adev->dev, "GPU recovery disabled.\n");
5396 		return false;
5397 }
5398 
5399 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5400 {
5401 	u32 i;
5402 	int ret = 0;
5403 
5404 	if (adev->bios)
5405 		amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5406 
5407 	dev_info(adev->dev, "GPU mode1 reset\n");
5408 
5409 	/* Cache the state before bus master disable. The saved config space
5410 	 * values are used in other cases like restore after mode-2 reset.
5411 	 */
5412 	amdgpu_device_cache_pci_state(adev->pdev);
5413 
5414 	/* disable BM */
5415 	pci_clear_master(adev->pdev);
5416 
5417 	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5418 		dev_info(adev->dev, "GPU smu mode1 reset\n");
5419 		ret = amdgpu_dpm_mode1_reset(adev);
5420 	} else {
5421 		dev_info(adev->dev, "GPU psp mode1 reset\n");
5422 		ret = psp_gpu_reset(adev);
5423 	}
5424 
5425 	if (ret)
5426 		goto mode1_reset_failed;
5427 
5428 	amdgpu_device_load_pci_state(adev->pdev);
5429 	ret = amdgpu_psp_wait_for_bootloader(adev);
5430 	if (ret)
5431 		goto mode1_reset_failed;
5432 
5433 	/* wait for asic to come out of reset */
5434 	for (i = 0; i < adev->usec_timeout; i++) {
5435 		u32 memsize = adev->nbio.funcs->get_memsize(adev);
5436 
5437 		if (memsize != 0xffffffff)
5438 			break;
5439 		udelay(1);
5440 	}
5441 
5442 	if (i >= adev->usec_timeout) {
5443 		ret = -ETIMEDOUT;
5444 		goto mode1_reset_failed;
5445 	}
5446 
5447 	if (adev->bios)
5448 		amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5449 
5450 	return 0;
5451 
5452 mode1_reset_failed:
5453 	dev_err(adev->dev, "GPU mode1 reset failed\n");
5454 	return ret;
5455 }
5456 
5457 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5458 				 struct amdgpu_reset_context *reset_context)
5459 {
5460 	int i, r = 0;
5461 	struct amdgpu_job *job = NULL;
5462 	struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
5463 	bool need_full_reset =
5464 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5465 
5466 	if (reset_context->reset_req_dev == adev)
5467 		job = reset_context->job;
5468 
5469 	if (amdgpu_sriov_vf(adev))
5470 		amdgpu_virt_pre_reset(adev);
5471 
5472 	amdgpu_fence_driver_isr_toggle(adev, true);
5473 
5474 	/* block all schedulers and reset given job's ring */
5475 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5476 		struct amdgpu_ring *ring = adev->rings[i];
5477 
5478 		if (!amdgpu_ring_sched_ready(ring))
5479 			continue;
5480 
5481 		/* Clear job fence from fence drv to avoid force_completion
5482 		 * leave NULL and vm flush fence in fence drv
5483 		 */
5484 		amdgpu_fence_driver_clear_job_fences(ring);
5485 
5486 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5487 		amdgpu_fence_driver_force_completion(ring);
5488 	}
5489 
5490 	amdgpu_fence_driver_isr_toggle(adev, false);
5491 
5492 	if (job && job->vm)
5493 		drm_sched_increase_karma(&job->base);
5494 
5495 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5496 	/* If reset handler not implemented, continue; otherwise return */
5497 	if (r == -EOPNOTSUPP)
5498 		r = 0;
5499 	else
5500 		return r;
5501 
5502 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5503 	if (!amdgpu_sriov_vf(adev)) {
5504 
5505 		if (!need_full_reset)
5506 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5507 
5508 		if (!need_full_reset && amdgpu_gpu_recovery &&
5509 		    amdgpu_device_ip_check_soft_reset(adev)) {
5510 			amdgpu_device_ip_pre_soft_reset(adev);
5511 			r = amdgpu_device_ip_soft_reset(adev);
5512 			amdgpu_device_ip_post_soft_reset(adev);
5513 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5514 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5515 				need_full_reset = true;
5516 			}
5517 		}
5518 
5519 		if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
5520 			dev_info(tmp_adev->dev, "Dumping IP State\n");
5521 			/* Trigger ip dump before we reset the asic */
5522 			for (i = 0; i < tmp_adev->num_ip_blocks; i++)
5523 				if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
5524 					tmp_adev->ip_blocks[i].version->funcs
5525 						->dump_ip_state((void *)&tmp_adev->ip_blocks[i]);
5526 			dev_info(tmp_adev->dev, "Dumping IP State Completed\n");
5527 		}
5528 
5529 		if (need_full_reset)
5530 			r = amdgpu_device_ip_suspend(adev);
5531 		if (need_full_reset)
5532 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5533 		else
5534 			clear_bit(AMDGPU_NEED_FULL_RESET,
5535 				  &reset_context->flags);
5536 	}
5537 
5538 	return r;
5539 }
5540 
5541 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
5542 {
5543 	struct list_head *device_list_handle;
5544 	bool full_reset, vram_lost = false;
5545 	struct amdgpu_device *tmp_adev;
5546 	int r, init_level;
5547 
5548 	device_list_handle = reset_context->reset_device_list;
5549 
5550 	if (!device_list_handle)
5551 		return -EINVAL;
5552 
5553 	full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5554 
5555 	/**
5556 	 * If it's reset on init, it's default init level, otherwise keep level
5557 	 * as recovery level.
5558 	 */
5559 	if (reset_context->method == AMD_RESET_METHOD_ON_INIT)
5560 			init_level = AMDGPU_INIT_LEVEL_DEFAULT;
5561 	else
5562 			init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY;
5563 
5564 	r = 0;
5565 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5566 		amdgpu_set_init_level(tmp_adev, init_level);
5567 		if (full_reset) {
5568 			/* post card */
5569 			amdgpu_ras_clear_err_state(tmp_adev);
5570 			r = amdgpu_device_asic_init(tmp_adev);
5571 			if (r) {
5572 				dev_warn(tmp_adev->dev, "asic atom init failed!");
5573 			} else {
5574 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5575 
5576 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
5577 				if (r)
5578 					goto out;
5579 
5580 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5581 
5582 				if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
5583 					amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job);
5584 
5585 				if (vram_lost) {
5586 					DRM_INFO("VRAM is lost due to GPU reset!\n");
5587 					amdgpu_inc_vram_lost(tmp_adev);
5588 				}
5589 
5590 				r = amdgpu_device_fw_loading(tmp_adev);
5591 				if (r)
5592 					return r;
5593 
5594 				r = amdgpu_xcp_restore_partition_mode(
5595 					tmp_adev->xcp_mgr);
5596 				if (r)
5597 					goto out;
5598 
5599 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
5600 				if (r)
5601 					goto out;
5602 
5603 				if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5604 					amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5605 
5606 				r = amdgpu_device_ip_resume_phase3(tmp_adev);
5607 				if (r)
5608 					goto out;
5609 
5610 				if (vram_lost)
5611 					amdgpu_device_fill_reset_magic(tmp_adev);
5612 
5613 				/*
5614 				 * Add this ASIC as tracked as reset was already
5615 				 * complete successfully.
5616 				 */
5617 				amdgpu_register_gpu_instance(tmp_adev);
5618 
5619 				if (!reset_context->hive &&
5620 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5621 					amdgpu_xgmi_add_device(tmp_adev);
5622 
5623 				r = amdgpu_device_ip_late_init(tmp_adev);
5624 				if (r)
5625 					goto out;
5626 
5627 				drm_client_dev_resume(adev_to_drm(tmp_adev), false);
5628 
5629 				/*
5630 				 * The GPU enters bad state once faulty pages
5631 				 * by ECC has reached the threshold, and ras
5632 				 * recovery is scheduled next. So add one check
5633 				 * here to break recovery if it indeed exceeds
5634 				 * bad page threshold, and remind user to
5635 				 * retire this GPU or setting one bigger
5636 				 * bad_page_threshold value to fix this once
5637 				 * probing driver again.
5638 				 */
5639 				if (!amdgpu_ras_is_rma(tmp_adev)) {
5640 					/* must succeed. */
5641 					amdgpu_ras_resume(tmp_adev);
5642 				} else {
5643 					r = -EINVAL;
5644 					goto out;
5645 				}
5646 
5647 				/* Update PSP FW topology after reset */
5648 				if (reset_context->hive &&
5649 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5650 					r = amdgpu_xgmi_update_topology(
5651 						reset_context->hive, tmp_adev);
5652 			}
5653 		}
5654 
5655 out:
5656 		if (!r) {
5657 			/* IP init is complete now, set level as default */
5658 			amdgpu_set_init_level(tmp_adev,
5659 					      AMDGPU_INIT_LEVEL_DEFAULT);
5660 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5661 			r = amdgpu_ib_ring_tests(tmp_adev);
5662 			if (r) {
5663 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5664 				r = -EAGAIN;
5665 				goto end;
5666 			}
5667 		}
5668 
5669 		if (r)
5670 			tmp_adev->asic_reset_res = r;
5671 	}
5672 
5673 end:
5674 	return r;
5675 }
5676 
5677 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5678 			 struct amdgpu_reset_context *reset_context)
5679 {
5680 	struct amdgpu_device *tmp_adev = NULL;
5681 	bool need_full_reset, skip_hw_reset;
5682 	int r = 0;
5683 
5684 	/* Try reset handler method first */
5685 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5686 				    reset_list);
5687 
5688 	reset_context->reset_device_list = device_list_handle;
5689 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5690 	/* If reset handler not implemented, continue; otherwise return */
5691 	if (r == -EOPNOTSUPP)
5692 		r = 0;
5693 	else
5694 		return r;
5695 
5696 	/* Reset handler not implemented, use the default method */
5697 	need_full_reset =
5698 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5699 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5700 
5701 	/*
5702 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
5703 	 * to allow proper links negotiation in FW (within 1 sec)
5704 	 */
5705 	if (!skip_hw_reset && need_full_reset) {
5706 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5707 			/* For XGMI run all resets in parallel to speed up the process */
5708 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5709 				if (!queue_work(system_unbound_wq,
5710 						&tmp_adev->xgmi_reset_work))
5711 					r = -EALREADY;
5712 			} else
5713 				r = amdgpu_asic_reset(tmp_adev);
5714 
5715 			if (r) {
5716 				dev_err(tmp_adev->dev,
5717 					"ASIC reset failed with error, %d for drm dev, %s",
5718 					r, adev_to_drm(tmp_adev)->unique);
5719 				goto out;
5720 			}
5721 		}
5722 
5723 		/* For XGMI wait for all resets to complete before proceed */
5724 		if (!r) {
5725 			list_for_each_entry(tmp_adev, device_list_handle,
5726 					    reset_list) {
5727 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5728 					flush_work(&tmp_adev->xgmi_reset_work);
5729 					r = tmp_adev->asic_reset_res;
5730 					if (r)
5731 						break;
5732 				}
5733 			}
5734 		}
5735 	}
5736 
5737 	if (!r && amdgpu_ras_intr_triggered()) {
5738 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5739 			amdgpu_ras_reset_error_count(tmp_adev,
5740 						     AMDGPU_RAS_BLOCK__MMHUB);
5741 		}
5742 
5743 		amdgpu_ras_intr_cleared();
5744 	}
5745 
5746 	r = amdgpu_device_reinit_after_reset(reset_context);
5747 	if (r == -EAGAIN)
5748 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5749 	else
5750 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5751 
5752 out:
5753 	return r;
5754 }
5755 
5756 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5757 {
5758 
5759 	switch (amdgpu_asic_reset_method(adev)) {
5760 	case AMD_RESET_METHOD_MODE1:
5761 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5762 		break;
5763 	case AMD_RESET_METHOD_MODE2:
5764 		adev->mp1_state = PP_MP1_STATE_RESET;
5765 		break;
5766 	default:
5767 		adev->mp1_state = PP_MP1_STATE_NONE;
5768 		break;
5769 	}
5770 }
5771 
5772 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5773 {
5774 	amdgpu_vf_error_trans_all(adev);
5775 	adev->mp1_state = PP_MP1_STATE_NONE;
5776 }
5777 
5778 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5779 {
5780 	struct pci_dev *p = NULL;
5781 
5782 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5783 			adev->pdev->bus->number, 1);
5784 	if (p) {
5785 		pm_runtime_enable(&(p->dev));
5786 		pm_runtime_resume(&(p->dev));
5787 	}
5788 
5789 	pci_dev_put(p);
5790 }
5791 
5792 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5793 {
5794 	enum amd_reset_method reset_method;
5795 	struct pci_dev *p = NULL;
5796 	u64 expires;
5797 
5798 	/*
5799 	 * For now, only BACO and mode1 reset are confirmed
5800 	 * to suffer the audio issue without proper suspended.
5801 	 */
5802 	reset_method = amdgpu_asic_reset_method(adev);
5803 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5804 	     (reset_method != AMD_RESET_METHOD_MODE1))
5805 		return -EINVAL;
5806 
5807 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5808 			adev->pdev->bus->number, 1);
5809 	if (!p)
5810 		return -ENODEV;
5811 
5812 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5813 	if (!expires)
5814 		/*
5815 		 * If we cannot get the audio device autosuspend delay,
5816 		 * a fixed 4S interval will be used. Considering 3S is
5817 		 * the audio controller default autosuspend delay setting.
5818 		 * 4S used here is guaranteed to cover that.
5819 		 */
5820 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5821 
5822 	while (!pm_runtime_status_suspended(&(p->dev))) {
5823 		if (!pm_runtime_suspend(&(p->dev)))
5824 			break;
5825 
5826 		if (expires < ktime_get_mono_fast_ns()) {
5827 			dev_warn(adev->dev, "failed to suspend display audio\n");
5828 			pci_dev_put(p);
5829 			/* TODO: abort the succeeding gpu reset? */
5830 			return -ETIMEDOUT;
5831 		}
5832 	}
5833 
5834 	pm_runtime_disable(&(p->dev));
5835 
5836 	pci_dev_put(p);
5837 	return 0;
5838 }
5839 
5840 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5841 {
5842 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5843 
5844 #if defined(CONFIG_DEBUG_FS)
5845 	if (!amdgpu_sriov_vf(adev))
5846 		cancel_work(&adev->reset_work);
5847 #endif
5848 
5849 	if (adev->kfd.dev)
5850 		cancel_work(&adev->kfd.reset_work);
5851 
5852 	if (amdgpu_sriov_vf(adev))
5853 		cancel_work(&adev->virt.flr_work);
5854 
5855 	if (con && adev->ras_enabled)
5856 		cancel_work(&con->recovery_work);
5857 
5858 }
5859 
5860 static int amdgpu_device_health_check(struct list_head *device_list_handle)
5861 {
5862 	struct amdgpu_device *tmp_adev;
5863 	int ret = 0;
5864 	u32 status;
5865 
5866 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5867 		pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status);
5868 		if (PCI_POSSIBLE_ERROR(status)) {
5869 			dev_err(tmp_adev->dev, "device lost from bus!");
5870 			ret = -ENODEV;
5871 		}
5872 	}
5873 
5874 	return ret;
5875 }
5876 
5877 /**
5878  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5879  *
5880  * @adev: amdgpu_device pointer
5881  * @job: which job trigger hang
5882  * @reset_context: amdgpu reset context pointer
5883  *
5884  * Attempt to reset the GPU if it has hung (all asics).
5885  * Attempt to do soft-reset or full-reset and reinitialize Asic
5886  * Returns 0 for success or an error on failure.
5887  */
5888 
5889 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5890 			      struct amdgpu_job *job,
5891 			      struct amdgpu_reset_context *reset_context)
5892 {
5893 	struct list_head device_list, *device_list_handle =  NULL;
5894 	bool job_signaled = false;
5895 	struct amdgpu_hive_info *hive = NULL;
5896 	struct amdgpu_device *tmp_adev = NULL;
5897 	int i, r = 0;
5898 	bool need_emergency_restart = false;
5899 	bool audio_suspended = false;
5900 	int retry_limit = AMDGPU_MAX_RETRY_LIMIT;
5901 
5902 	/*
5903 	 * If it reaches here because of hang/timeout and a RAS error is
5904 	 * detected at the same time, let RAS recovery take care of it.
5905 	 */
5906 	if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) &&
5907 	    !amdgpu_sriov_vf(adev) &&
5908 	    reset_context->src != AMDGPU_RESET_SRC_RAS) {
5909 		dev_dbg(adev->dev,
5910 			"Gpu recovery from source: %d yielding to RAS error recovery handling",
5911 			reset_context->src);
5912 		return 0;
5913 	}
5914 	/*
5915 	 * Special case: RAS triggered and full reset isn't supported
5916 	 */
5917 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5918 
5919 	/*
5920 	 * Flush RAM to disk so that after reboot
5921 	 * the user can read log and see why the system rebooted.
5922 	 */
5923 	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5924 		amdgpu_ras_get_context(adev)->reboot) {
5925 		DRM_WARN("Emergency reboot.");
5926 
5927 		ksys_sync_helper();
5928 		emergency_restart();
5929 	}
5930 
5931 	dev_info(adev->dev, "GPU %s begin!\n",
5932 		need_emergency_restart ? "jobs stop":"reset");
5933 
5934 	if (!amdgpu_sriov_vf(adev))
5935 		hive = amdgpu_get_xgmi_hive(adev);
5936 	if (hive)
5937 		mutex_lock(&hive->hive_lock);
5938 
5939 	reset_context->job = job;
5940 	reset_context->hive = hive;
5941 	/*
5942 	 * Build list of devices to reset.
5943 	 * In case we are in XGMI hive mode, resort the device list
5944 	 * to put adev in the 1st position.
5945 	 */
5946 	INIT_LIST_HEAD(&device_list);
5947 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
5948 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5949 			list_add_tail(&tmp_adev->reset_list, &device_list);
5950 			if (adev->shutdown)
5951 				tmp_adev->shutdown = true;
5952 		}
5953 		if (!list_is_first(&adev->reset_list, &device_list))
5954 			list_rotate_to_front(&adev->reset_list, &device_list);
5955 		device_list_handle = &device_list;
5956 	} else {
5957 		list_add_tail(&adev->reset_list, &device_list);
5958 		device_list_handle = &device_list;
5959 	}
5960 
5961 	if (!amdgpu_sriov_vf(adev)) {
5962 		r = amdgpu_device_health_check(device_list_handle);
5963 		if (r)
5964 			goto end_reset;
5965 	}
5966 
5967 	/* We need to lock reset domain only once both for XGMI and single device */
5968 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5969 				    reset_list);
5970 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5971 
5972 	/* block all schedulers and reset given job's ring */
5973 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5974 
5975 		amdgpu_device_set_mp1_state(tmp_adev);
5976 
5977 		/*
5978 		 * Try to put the audio codec into suspend state
5979 		 * before gpu reset started.
5980 		 *
5981 		 * Due to the power domain of the graphics device
5982 		 * is shared with AZ power domain. Without this,
5983 		 * we may change the audio hardware from behind
5984 		 * the audio driver's back. That will trigger
5985 		 * some audio codec errors.
5986 		 */
5987 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5988 			audio_suspended = true;
5989 
5990 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5991 
5992 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5993 
5994 		amdgpu_amdkfd_pre_reset(tmp_adev, reset_context);
5995 
5996 		/*
5997 		 * Mark these ASICs to be reset as untracked first
5998 		 * And add them back after reset completed
5999 		 */
6000 		amdgpu_unregister_gpu_instance(tmp_adev);
6001 
6002 		drm_client_dev_suspend(adev_to_drm(tmp_adev), false);
6003 
6004 		/* disable ras on ALL IPs */
6005 		if (!need_emergency_restart &&
6006 		      amdgpu_device_ip_need_full_reset(tmp_adev))
6007 			amdgpu_ras_suspend(tmp_adev);
6008 
6009 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6010 			struct amdgpu_ring *ring = tmp_adev->rings[i];
6011 
6012 			if (!amdgpu_ring_sched_ready(ring))
6013 				continue;
6014 
6015 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
6016 
6017 			if (need_emergency_restart)
6018 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
6019 		}
6020 		atomic_inc(&tmp_adev->gpu_reset_counter);
6021 	}
6022 
6023 	if (need_emergency_restart)
6024 		goto skip_sched_resume;
6025 
6026 	/*
6027 	 * Must check guilty signal here since after this point all old
6028 	 * HW fences are force signaled.
6029 	 *
6030 	 * job->base holds a reference to parent fence
6031 	 */
6032 	if (job && dma_fence_is_signaled(&job->hw_fence)) {
6033 		job_signaled = true;
6034 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
6035 		goto skip_hw_reset;
6036 	}
6037 
6038 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
6039 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6040 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
6041 		/*TODO Should we stop ?*/
6042 		if (r) {
6043 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
6044 				  r, adev_to_drm(tmp_adev)->unique);
6045 			tmp_adev->asic_reset_res = r;
6046 		}
6047 	}
6048 
6049 	/* Actual ASIC resets if needed.*/
6050 	/* Host driver will handle XGMI hive reset for SRIOV */
6051 	if (amdgpu_sriov_vf(adev)) {
6052 		if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
6053 			dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
6054 			amdgpu_ras_set_fed(adev, true);
6055 			set_bit(AMDGPU_HOST_FLR, &reset_context->flags);
6056 		}
6057 
6058 		r = amdgpu_device_reset_sriov(adev, reset_context);
6059 		if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) {
6060 			amdgpu_virt_release_full_gpu(adev, true);
6061 			goto retry;
6062 		}
6063 		if (r)
6064 			adev->asic_reset_res = r;
6065 	} else {
6066 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
6067 		if (r && r == -EAGAIN)
6068 			goto retry;
6069 	}
6070 
6071 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6072 		/*
6073 		 * Drop any pending non scheduler resets queued before reset is done.
6074 		 * Any reset scheduled after this point would be valid. Scheduler resets
6075 		 * were already dropped during drm_sched_stop and no new ones can come
6076 		 * in before drm_sched_start.
6077 		 */
6078 		amdgpu_device_stop_pending_resets(tmp_adev);
6079 	}
6080 
6081 skip_hw_reset:
6082 
6083 	/* Post ASIC reset for all devs .*/
6084 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6085 
6086 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6087 			struct amdgpu_ring *ring = tmp_adev->rings[i];
6088 
6089 			if (!amdgpu_ring_sched_ready(ring))
6090 				continue;
6091 
6092 			drm_sched_start(&ring->sched, 0);
6093 		}
6094 
6095 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
6096 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
6097 
6098 		if (tmp_adev->asic_reset_res)
6099 			r = tmp_adev->asic_reset_res;
6100 
6101 		tmp_adev->asic_reset_res = 0;
6102 
6103 		if (r) {
6104 			/* bad news, how to tell it to userspace ?
6105 			 * for ras error, we should report GPU bad status instead of
6106 			 * reset failure
6107 			 */
6108 			if (reset_context->src != AMDGPU_RESET_SRC_RAS ||
6109 			    !amdgpu_ras_eeprom_check_err_threshold(tmp_adev))
6110 				dev_info(tmp_adev->dev, "GPU reset(%d) failed\n",
6111 					atomic_read(&tmp_adev->gpu_reset_counter));
6112 			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
6113 		} else {
6114 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
6115 			if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
6116 				DRM_WARN("smart shift update failed\n");
6117 		}
6118 	}
6119 
6120 skip_sched_resume:
6121 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6122 		/* unlock kfd: SRIOV would do it separately */
6123 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
6124 			amdgpu_amdkfd_post_reset(tmp_adev);
6125 
6126 		/* kfd_post_reset will do nothing if kfd device is not initialized,
6127 		 * need to bring up kfd here if it's not be initialized before
6128 		 */
6129 		if (!adev->kfd.init_complete)
6130 			amdgpu_amdkfd_device_init(adev);
6131 
6132 		if (audio_suspended)
6133 			amdgpu_device_resume_display_audio(tmp_adev);
6134 
6135 		amdgpu_device_unset_mp1_state(tmp_adev);
6136 
6137 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
6138 	}
6139 
6140 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
6141 					    reset_list);
6142 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
6143 
6144 end_reset:
6145 	if (hive) {
6146 		mutex_unlock(&hive->hive_lock);
6147 		amdgpu_put_xgmi_hive(hive);
6148 	}
6149 
6150 	if (r)
6151 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
6152 
6153 	atomic_set(&adev->reset_domain->reset_res, r);
6154 	return r;
6155 }
6156 
6157 /**
6158  * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
6159  *
6160  * @adev: amdgpu_device pointer
6161  * @speed: pointer to the speed of the link
6162  * @width: pointer to the width of the link
6163  *
6164  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
6165  * first physical partner to an AMD dGPU.
6166  * This will exclude any virtual switches and links.
6167  */
6168 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
6169 					    enum pci_bus_speed *speed,
6170 					    enum pcie_link_width *width)
6171 {
6172 	struct pci_dev *parent = adev->pdev;
6173 
6174 	if (!speed || !width)
6175 		return;
6176 
6177 	*speed = PCI_SPEED_UNKNOWN;
6178 	*width = PCIE_LNK_WIDTH_UNKNOWN;
6179 
6180 	if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
6181 		while ((parent = pci_upstream_bridge(parent))) {
6182 			/* skip upstream/downstream switches internal to dGPU*/
6183 			if (parent->vendor == PCI_VENDOR_ID_ATI)
6184 				continue;
6185 			*speed = pcie_get_speed_cap(parent);
6186 			*width = pcie_get_width_cap(parent);
6187 			break;
6188 		}
6189 	} else {
6190 		/* use the current speeds rather than max if switching is not supported */
6191 		pcie_bandwidth_available(adev->pdev, NULL, speed, width);
6192 	}
6193 }
6194 
6195 /**
6196  * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU
6197  *
6198  * @adev: amdgpu_device pointer
6199  * @speed: pointer to the speed of the link
6200  * @width: pointer to the width of the link
6201  *
6202  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
6203  * AMD dGPU which may be a virtual upstream bridge.
6204  */
6205 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,
6206 					enum pci_bus_speed *speed,
6207 					enum pcie_link_width *width)
6208 {
6209 	struct pci_dev *parent = adev->pdev;
6210 
6211 	if (!speed || !width)
6212 		return;
6213 
6214 	parent = pci_upstream_bridge(parent);
6215 	if (parent && parent->vendor == PCI_VENDOR_ID_ATI) {
6216 		/* use the upstream/downstream switches internal to dGPU */
6217 		*speed = pcie_get_speed_cap(parent);
6218 		*width = pcie_get_width_cap(parent);
6219 		while ((parent = pci_upstream_bridge(parent))) {
6220 			if (parent->vendor == PCI_VENDOR_ID_ATI) {
6221 				/* use the upstream/downstream switches internal to dGPU */
6222 				*speed = pcie_get_speed_cap(parent);
6223 				*width = pcie_get_width_cap(parent);
6224 			}
6225 		}
6226 	} else {
6227 		/* use the device itself */
6228 		*speed = pcie_get_speed_cap(adev->pdev);
6229 		*width = pcie_get_width_cap(adev->pdev);
6230 	}
6231 }
6232 
6233 /**
6234  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
6235  *
6236  * @adev: amdgpu_device pointer
6237  *
6238  * Fetches and stores in the driver the PCIE capabilities (gen speed
6239  * and lanes) of the slot the device is in. Handles APUs and
6240  * virtualized environments where PCIE config space may not be available.
6241  */
6242 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
6243 {
6244 	enum pci_bus_speed speed_cap, platform_speed_cap;
6245 	enum pcie_link_width platform_link_width, link_width;
6246 
6247 	if (amdgpu_pcie_gen_cap)
6248 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
6249 
6250 	if (amdgpu_pcie_lane_cap)
6251 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
6252 
6253 	/* covers APUs as well */
6254 	if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
6255 		if (adev->pm.pcie_gen_mask == 0)
6256 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
6257 		if (adev->pm.pcie_mlw_mask == 0)
6258 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
6259 		return;
6260 	}
6261 
6262 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
6263 		return;
6264 
6265 	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
6266 					&platform_link_width);
6267 	amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width);
6268 
6269 	if (adev->pm.pcie_gen_mask == 0) {
6270 		/* asic caps */
6271 		if (speed_cap == PCI_SPEED_UNKNOWN) {
6272 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6273 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6274 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6275 		} else {
6276 			if (speed_cap == PCIE_SPEED_32_0GT)
6277 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6278 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6279 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6280 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6281 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
6282 			else if (speed_cap == PCIE_SPEED_16_0GT)
6283 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6284 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6285 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6286 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
6287 			else if (speed_cap == PCIE_SPEED_8_0GT)
6288 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6289 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6290 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6291 			else if (speed_cap == PCIE_SPEED_5_0GT)
6292 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6293 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
6294 			else
6295 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
6296 		}
6297 		/* platform caps */
6298 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
6299 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6300 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6301 		} else {
6302 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
6303 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6304 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6305 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6306 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6307 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
6308 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
6309 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6310 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6311 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6312 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
6313 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
6314 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6315 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6316 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
6317 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
6318 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6319 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6320 			else
6321 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
6322 
6323 		}
6324 	}
6325 	if (adev->pm.pcie_mlw_mask == 0) {
6326 		/* asic caps */
6327 		if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6328 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK;
6329 		} else {
6330 			switch (link_width) {
6331 			case PCIE_LNK_X32:
6332 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 |
6333 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6334 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6335 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6336 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6337 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6338 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6339 				break;
6340 			case PCIE_LNK_X16:
6341 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6342 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6343 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6344 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6345 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6346 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6347 				break;
6348 			case PCIE_LNK_X12:
6349 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6350 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6351 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6352 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6353 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6354 				break;
6355 			case PCIE_LNK_X8:
6356 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6357 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6358 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6359 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6360 				break;
6361 			case PCIE_LNK_X4:
6362 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6363 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6364 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6365 				break;
6366 			case PCIE_LNK_X2:
6367 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6368 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6369 				break;
6370 			case PCIE_LNK_X1:
6371 				adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1;
6372 				break;
6373 			default:
6374 				break;
6375 			}
6376 		}
6377 		/* platform caps */
6378 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6379 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
6380 		} else {
6381 			switch (platform_link_width) {
6382 			case PCIE_LNK_X32:
6383 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
6384 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6385 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6386 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6387 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6388 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6389 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6390 				break;
6391 			case PCIE_LNK_X16:
6392 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6393 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6394 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6395 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6396 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6397 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6398 				break;
6399 			case PCIE_LNK_X12:
6400 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6401 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6402 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6403 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6404 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6405 				break;
6406 			case PCIE_LNK_X8:
6407 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6408 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6409 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6410 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6411 				break;
6412 			case PCIE_LNK_X4:
6413 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6414 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6415 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6416 				break;
6417 			case PCIE_LNK_X2:
6418 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6419 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6420 				break;
6421 			case PCIE_LNK_X1:
6422 				adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
6423 				break;
6424 			default:
6425 				break;
6426 			}
6427 		}
6428 	}
6429 }
6430 
6431 /**
6432  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6433  *
6434  * @adev: amdgpu_device pointer
6435  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6436  *
6437  * Return true if @peer_adev can access (DMA) @adev through the PCIe
6438  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6439  * @peer_adev.
6440  */
6441 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6442 				      struct amdgpu_device *peer_adev)
6443 {
6444 #ifdef CONFIG_HSA_AMD_P2P
6445 	bool p2p_access =
6446 		!adev->gmc.xgmi.connected_to_cpu &&
6447 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6448 	if (!p2p_access)
6449 		dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",
6450 			pci_name(peer_adev->pdev));
6451 
6452 	bool is_large_bar = adev->gmc.visible_vram_size &&
6453 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size;
6454 	bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev);
6455 
6456 	if (!p2p_addressable) {
6457 		uint64_t address_mask = peer_adev->dev->dma_mask ?
6458 			~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6459 		resource_size_t aper_limit =
6460 			adev->gmc.aper_base + adev->gmc.aper_size - 1;
6461 
6462 		p2p_addressable = !(adev->gmc.aper_base & address_mask ||
6463 				     aper_limit & address_mask);
6464 	}
6465 	return pcie_p2p && is_large_bar && p2p_access && p2p_addressable;
6466 #else
6467 	return false;
6468 #endif
6469 }
6470 
6471 int amdgpu_device_baco_enter(struct drm_device *dev)
6472 {
6473 	struct amdgpu_device *adev = drm_to_adev(dev);
6474 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6475 
6476 	if (!amdgpu_device_supports_baco(dev))
6477 		return -ENOTSUPP;
6478 
6479 	if (ras && adev->ras_enabled &&
6480 	    adev->nbio.funcs->enable_doorbell_interrupt)
6481 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6482 
6483 	return amdgpu_dpm_baco_enter(adev);
6484 }
6485 
6486 int amdgpu_device_baco_exit(struct drm_device *dev)
6487 {
6488 	struct amdgpu_device *adev = drm_to_adev(dev);
6489 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6490 	int ret = 0;
6491 
6492 	if (!amdgpu_device_supports_baco(dev))
6493 		return -ENOTSUPP;
6494 
6495 	ret = amdgpu_dpm_baco_exit(adev);
6496 	if (ret)
6497 		return ret;
6498 
6499 	if (ras && adev->ras_enabled &&
6500 	    adev->nbio.funcs->enable_doorbell_interrupt)
6501 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6502 
6503 	if (amdgpu_passthrough(adev) && adev->nbio.funcs &&
6504 	    adev->nbio.funcs->clear_doorbell_interrupt)
6505 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
6506 
6507 	return 0;
6508 }
6509 
6510 /**
6511  * amdgpu_pci_error_detected - Called when a PCI error is detected.
6512  * @pdev: PCI device struct
6513  * @state: PCI channel state
6514  *
6515  * Description: Called when a PCI error is detected.
6516  *
6517  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6518  */
6519 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6520 {
6521 	struct drm_device *dev = pci_get_drvdata(pdev);
6522 	struct amdgpu_device *adev = drm_to_adev(dev);
6523 	int i;
6524 
6525 	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6526 
6527 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
6528 		DRM_WARN("No support for XGMI hive yet...");
6529 		return PCI_ERS_RESULT_DISCONNECT;
6530 	}
6531 
6532 	adev->pci_channel_state = state;
6533 
6534 	switch (state) {
6535 	case pci_channel_io_normal:
6536 		return PCI_ERS_RESULT_CAN_RECOVER;
6537 	/* Fatal error, prepare for slot reset */
6538 	case pci_channel_io_frozen:
6539 		/*
6540 		 * Locking adev->reset_domain->sem will prevent any external access
6541 		 * to GPU during PCI error recovery
6542 		 */
6543 		amdgpu_device_lock_reset_domain(adev->reset_domain);
6544 		amdgpu_device_set_mp1_state(adev);
6545 
6546 		/*
6547 		 * Block any work scheduling as we do for regular GPU reset
6548 		 * for the duration of the recovery
6549 		 */
6550 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6551 			struct amdgpu_ring *ring = adev->rings[i];
6552 
6553 			if (!amdgpu_ring_sched_ready(ring))
6554 				continue;
6555 
6556 			drm_sched_stop(&ring->sched, NULL);
6557 		}
6558 		atomic_inc(&adev->gpu_reset_counter);
6559 		return PCI_ERS_RESULT_NEED_RESET;
6560 	case pci_channel_io_perm_failure:
6561 		/* Permanent error, prepare for device removal */
6562 		return PCI_ERS_RESULT_DISCONNECT;
6563 	}
6564 
6565 	return PCI_ERS_RESULT_NEED_RESET;
6566 }
6567 
6568 /**
6569  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6570  * @pdev: pointer to PCI device
6571  */
6572 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6573 {
6574 
6575 	DRM_INFO("PCI error: mmio enabled callback!!\n");
6576 
6577 	/* TODO - dump whatever for debugging purposes */
6578 
6579 	/* This called only if amdgpu_pci_error_detected returns
6580 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6581 	 * works, no need to reset slot.
6582 	 */
6583 
6584 	return PCI_ERS_RESULT_RECOVERED;
6585 }
6586 
6587 /**
6588  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6589  * @pdev: PCI device struct
6590  *
6591  * Description: This routine is called by the pci error recovery
6592  * code after the PCI slot has been reset, just before we
6593  * should resume normal operations.
6594  */
6595 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6596 {
6597 	struct drm_device *dev = pci_get_drvdata(pdev);
6598 	struct amdgpu_device *adev = drm_to_adev(dev);
6599 	int r, i;
6600 	struct amdgpu_reset_context reset_context;
6601 	u32 memsize;
6602 	struct list_head device_list;
6603 
6604 	/* PCI error slot reset should be skipped During RAS recovery */
6605 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
6606 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) &&
6607 	    amdgpu_ras_in_recovery(adev))
6608 		return PCI_ERS_RESULT_RECOVERED;
6609 
6610 	DRM_INFO("PCI error: slot reset callback!!\n");
6611 
6612 	memset(&reset_context, 0, sizeof(reset_context));
6613 
6614 	INIT_LIST_HEAD(&device_list);
6615 	list_add_tail(&adev->reset_list, &device_list);
6616 
6617 	/* wait for asic to come out of reset */
6618 	msleep(500);
6619 
6620 	/* Restore PCI confspace */
6621 	amdgpu_device_load_pci_state(pdev);
6622 
6623 	/* confirm  ASIC came out of reset */
6624 	for (i = 0; i < adev->usec_timeout; i++) {
6625 		memsize = amdgpu_asic_get_config_memsize(adev);
6626 
6627 		if (memsize != 0xffffffff)
6628 			break;
6629 		udelay(1);
6630 	}
6631 	if (memsize == 0xffffffff) {
6632 		r = -ETIME;
6633 		goto out;
6634 	}
6635 
6636 	reset_context.method = AMD_RESET_METHOD_NONE;
6637 	reset_context.reset_req_dev = adev;
6638 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6639 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6640 
6641 	adev->no_hw_access = true;
6642 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6643 	adev->no_hw_access = false;
6644 	if (r)
6645 		goto out;
6646 
6647 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
6648 
6649 out:
6650 	if (!r) {
6651 		if (amdgpu_device_cache_pci_state(adev->pdev))
6652 			pci_restore_state(adev->pdev);
6653 
6654 		DRM_INFO("PCIe error recovery succeeded\n");
6655 	} else {
6656 		DRM_ERROR("PCIe error recovery failed, err:%d", r);
6657 		amdgpu_device_unset_mp1_state(adev);
6658 		amdgpu_device_unlock_reset_domain(adev->reset_domain);
6659 	}
6660 
6661 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6662 }
6663 
6664 /**
6665  * amdgpu_pci_resume() - resume normal ops after PCI reset
6666  * @pdev: pointer to PCI device
6667  *
6668  * Called when the error recovery driver tells us that its
6669  * OK to resume normal operation.
6670  */
6671 void amdgpu_pci_resume(struct pci_dev *pdev)
6672 {
6673 	struct drm_device *dev = pci_get_drvdata(pdev);
6674 	struct amdgpu_device *adev = drm_to_adev(dev);
6675 	int i;
6676 
6677 
6678 	DRM_INFO("PCI error: resume callback!!\n");
6679 
6680 	/* Only continue execution for the case of pci_channel_io_frozen */
6681 	if (adev->pci_channel_state != pci_channel_io_frozen)
6682 		return;
6683 
6684 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6685 		struct amdgpu_ring *ring = adev->rings[i];
6686 
6687 		if (!amdgpu_ring_sched_ready(ring))
6688 			continue;
6689 
6690 		drm_sched_start(&ring->sched, 0);
6691 	}
6692 
6693 	amdgpu_device_unset_mp1_state(adev);
6694 	amdgpu_device_unlock_reset_domain(adev->reset_domain);
6695 }
6696 
6697 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6698 {
6699 	struct drm_device *dev = pci_get_drvdata(pdev);
6700 	struct amdgpu_device *adev = drm_to_adev(dev);
6701 	int r;
6702 
6703 	if (amdgpu_sriov_vf(adev))
6704 		return false;
6705 
6706 	r = pci_save_state(pdev);
6707 	if (!r) {
6708 		kfree(adev->pci_state);
6709 
6710 		adev->pci_state = pci_store_saved_state(pdev);
6711 
6712 		if (!adev->pci_state) {
6713 			DRM_ERROR("Failed to store PCI saved state");
6714 			return false;
6715 		}
6716 	} else {
6717 		DRM_WARN("Failed to save PCI state, err:%d\n", r);
6718 		return false;
6719 	}
6720 
6721 	return true;
6722 }
6723 
6724 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6725 {
6726 	struct drm_device *dev = pci_get_drvdata(pdev);
6727 	struct amdgpu_device *adev = drm_to_adev(dev);
6728 	int r;
6729 
6730 	if (!adev->pci_state)
6731 		return false;
6732 
6733 	r = pci_load_saved_state(pdev, adev->pci_state);
6734 
6735 	if (!r) {
6736 		pci_restore_state(pdev);
6737 	} else {
6738 		DRM_WARN("Failed to load PCI state, err:%d\n", r);
6739 		return false;
6740 	}
6741 
6742 	return true;
6743 }
6744 
6745 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6746 		struct amdgpu_ring *ring)
6747 {
6748 #ifdef CONFIG_X86_64
6749 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6750 		return;
6751 #endif
6752 	if (adev->gmc.xgmi.connected_to_cpu)
6753 		return;
6754 
6755 	if (ring && ring->funcs->emit_hdp_flush)
6756 		amdgpu_ring_emit_hdp_flush(ring);
6757 	else
6758 		amdgpu_asic_flush_hdp(adev, ring);
6759 }
6760 
6761 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6762 		struct amdgpu_ring *ring)
6763 {
6764 #ifdef CONFIG_X86_64
6765 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6766 		return;
6767 #endif
6768 	if (adev->gmc.xgmi.connected_to_cpu)
6769 		return;
6770 
6771 	amdgpu_asic_invalidate_hdp(adev, ring);
6772 }
6773 
6774 int amdgpu_in_reset(struct amdgpu_device *adev)
6775 {
6776 	return atomic_read(&adev->reset_domain->in_gpu_reset);
6777 }
6778 
6779 /**
6780  * amdgpu_device_halt() - bring hardware to some kind of halt state
6781  *
6782  * @adev: amdgpu_device pointer
6783  *
6784  * Bring hardware to some kind of halt state so that no one can touch it
6785  * any more. It will help to maintain error context when error occurred.
6786  * Compare to a simple hang, the system will keep stable at least for SSH
6787  * access. Then it should be trivial to inspect the hardware state and
6788  * see what's going on. Implemented as following:
6789  *
6790  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6791  *    clears all CPU mappings to device, disallows remappings through page faults
6792  * 2. amdgpu_irq_disable_all() disables all interrupts
6793  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6794  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6795  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6796  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6797  *    flush any in flight DMA operations
6798  */
6799 void amdgpu_device_halt(struct amdgpu_device *adev)
6800 {
6801 	struct pci_dev *pdev = adev->pdev;
6802 	struct drm_device *ddev = adev_to_drm(adev);
6803 
6804 	amdgpu_xcp_dev_unplug(adev);
6805 	drm_dev_unplug(ddev);
6806 
6807 	amdgpu_irq_disable_all(adev);
6808 
6809 	amdgpu_fence_driver_hw_fini(adev);
6810 
6811 	adev->no_hw_access = true;
6812 
6813 	amdgpu_device_unmap_mmio(adev);
6814 
6815 	pci_disable_device(pdev);
6816 	pci_wait_for_pending_transaction(pdev);
6817 }
6818 
6819 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6820 				u32 reg)
6821 {
6822 	unsigned long flags, address, data;
6823 	u32 r;
6824 
6825 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6826 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6827 
6828 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6829 	WREG32(address, reg * 4);
6830 	(void)RREG32(address);
6831 	r = RREG32(data);
6832 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6833 	return r;
6834 }
6835 
6836 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6837 				u32 reg, u32 v)
6838 {
6839 	unsigned long flags, address, data;
6840 
6841 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6842 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6843 
6844 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6845 	WREG32(address, reg * 4);
6846 	(void)RREG32(address);
6847 	WREG32(data, v);
6848 	(void)RREG32(data);
6849 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6850 }
6851 
6852 /**
6853  * amdgpu_device_get_gang - return a reference to the current gang
6854  * @adev: amdgpu_device pointer
6855  *
6856  * Returns: A new reference to the current gang leader.
6857  */
6858 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)
6859 {
6860 	struct dma_fence *fence;
6861 
6862 	rcu_read_lock();
6863 	fence = dma_fence_get_rcu_safe(&adev->gang_submit);
6864 	rcu_read_unlock();
6865 	return fence;
6866 }
6867 
6868 /**
6869  * amdgpu_device_switch_gang - switch to a new gang
6870  * @adev: amdgpu_device pointer
6871  * @gang: the gang to switch to
6872  *
6873  * Try to switch to a new gang.
6874  * Returns: NULL if we switched to the new gang or a reference to the current
6875  * gang leader.
6876  */
6877 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6878 					    struct dma_fence *gang)
6879 {
6880 	struct dma_fence *old = NULL;
6881 
6882 	do {
6883 		dma_fence_put(old);
6884 		old = amdgpu_device_get_gang(adev);
6885 		if (old == gang)
6886 			break;
6887 
6888 		if (!dma_fence_is_signaled(old))
6889 			return old;
6890 
6891 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6892 			 old, gang) != old);
6893 
6894 	dma_fence_put(old);
6895 	return NULL;
6896 }
6897 
6898 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6899 {
6900 	switch (adev->asic_type) {
6901 #ifdef CONFIG_DRM_AMDGPU_SI
6902 	case CHIP_HAINAN:
6903 #endif
6904 	case CHIP_TOPAZ:
6905 		/* chips with no display hardware */
6906 		return false;
6907 #ifdef CONFIG_DRM_AMDGPU_SI
6908 	case CHIP_TAHITI:
6909 	case CHIP_PITCAIRN:
6910 	case CHIP_VERDE:
6911 	case CHIP_OLAND:
6912 #endif
6913 #ifdef CONFIG_DRM_AMDGPU_CIK
6914 	case CHIP_BONAIRE:
6915 	case CHIP_HAWAII:
6916 	case CHIP_KAVERI:
6917 	case CHIP_KABINI:
6918 	case CHIP_MULLINS:
6919 #endif
6920 	case CHIP_TONGA:
6921 	case CHIP_FIJI:
6922 	case CHIP_POLARIS10:
6923 	case CHIP_POLARIS11:
6924 	case CHIP_POLARIS12:
6925 	case CHIP_VEGAM:
6926 	case CHIP_CARRIZO:
6927 	case CHIP_STONEY:
6928 		/* chips with display hardware */
6929 		return true;
6930 	default:
6931 		/* IP discovery */
6932 		if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6933 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6934 			return false;
6935 		return true;
6936 	}
6937 }
6938 
6939 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6940 		uint32_t inst, uint32_t reg_addr, char reg_name[],
6941 		uint32_t expected_value, uint32_t mask)
6942 {
6943 	uint32_t ret = 0;
6944 	uint32_t old_ = 0;
6945 	uint32_t tmp_ = RREG32(reg_addr);
6946 	uint32_t loop = adev->usec_timeout;
6947 
6948 	while ((tmp_ & (mask)) != (expected_value)) {
6949 		if (old_ != tmp_) {
6950 			loop = adev->usec_timeout;
6951 			old_ = tmp_;
6952 		} else
6953 			udelay(1);
6954 		tmp_ = RREG32(reg_addr);
6955 		loop--;
6956 		if (!loop) {
6957 			DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6958 				  inst, reg_name, (uint32_t)expected_value,
6959 				  (uint32_t)(tmp_ & (mask)));
6960 			ret = -ETIMEDOUT;
6961 			break;
6962 		}
6963 	}
6964 	return ret;
6965 }
6966 
6967 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
6968 {
6969 	ssize_t size = 0;
6970 
6971 	if (!ring || !ring->adev)
6972 		return size;
6973 
6974 	if (amdgpu_device_should_recover_gpu(ring->adev))
6975 		size |= AMDGPU_RESET_TYPE_FULL;
6976 
6977 	if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
6978 	    !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
6979 		size |= AMDGPU_RESET_TYPE_SOFT_RESET;
6980 
6981 	return size;
6982 }
6983 
6984 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
6985 {
6986 	ssize_t size = 0;
6987 
6988 	if (supported_reset == 0) {
6989 		size += sysfs_emit_at(buf, size, "unsupported");
6990 		size += sysfs_emit_at(buf, size, "\n");
6991 		return size;
6992 
6993 	}
6994 
6995 	if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET)
6996 		size += sysfs_emit_at(buf, size, "soft ");
6997 
6998 	if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
6999 		size += sysfs_emit_at(buf, size, "queue ");
7000 
7001 	if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE)
7002 		size += sysfs_emit_at(buf, size, "pipe ");
7003 
7004 	if (supported_reset & AMDGPU_RESET_TYPE_FULL)
7005 		size += sysfs_emit_at(buf, size, "full ");
7006 
7007 	size += sysfs_emit_at(buf, size, "\n");
7008 	return size;
7009 }
7010