xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c (revision e619ac419174fdb6093b9e78b41bb5d0a97de9dd)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/aperture.h>
30 #include <linux/power_supply.h>
31 #include <linux/kthread.h>
32 #include <linux/module.h>
33 #include <linux/console.h>
34 #include <linux/slab.h>
35 #include <linux/iommu.h>
36 #include <linux/pci.h>
37 #include <linux/pci-p2pdma.h>
38 #include <linux/apple-gmux.h>
39 
40 #include <drm/drm_atomic_helper.h>
41 #include <drm/drm_client_event.h>
42 #include <drm/drm_crtc_helper.h>
43 #include <drm/drm_probe_helper.h>
44 #include <drm/amdgpu_drm.h>
45 #include <linux/device.h>
46 #include <linux/vgaarb.h>
47 #include <linux/vga_switcheroo.h>
48 #include <linux/efi.h>
49 #include "amdgpu.h"
50 #include "amdgpu_trace.h"
51 #include "amdgpu_i2c.h"
52 #include "atom.h"
53 #include "amdgpu_atombios.h"
54 #include "amdgpu_atomfirmware.h"
55 #include "amd_pcie.h"
56 #ifdef CONFIG_DRM_AMDGPU_SI
57 #include "si.h"
58 #endif
59 #ifdef CONFIG_DRM_AMDGPU_CIK
60 #include "cik.h"
61 #endif
62 #include "vi.h"
63 #include "soc15.h"
64 #include "nv.h"
65 #include "bif/bif_4_1_d.h"
66 #include <linux/firmware.h>
67 #include "amdgpu_vf_error.h"
68 
69 #include "amdgpu_amdkfd.h"
70 #include "amdgpu_pm.h"
71 
72 #include "amdgpu_xgmi.h"
73 #include "amdgpu_ras.h"
74 #include "amdgpu_pmu.h"
75 #include "amdgpu_fru_eeprom.h"
76 #include "amdgpu_reset.h"
77 #include "amdgpu_virt.h"
78 #include "amdgpu_dev_coredump.h"
79 
80 #include <linux/suspend.h>
81 #include <drm/task_barrier.h>
82 #include <linux/pm_runtime.h>
83 
84 #include <drm/drm_drv.h>
85 
86 #if IS_ENABLED(CONFIG_X86)
87 #include <asm/intel-family.h>
88 #endif
89 
90 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
91 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
92 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
93 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
94 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
95 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
96 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
97 
98 #define AMDGPU_RESUME_MS		2000
99 #define AMDGPU_MAX_RETRY_LIMIT		2
100 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
101 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
102 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
103 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
104 
105 #define AMDGPU_VBIOS_SKIP (1U << 0)
106 #define AMDGPU_VBIOS_OPTIONAL (1U << 1)
107 
108 static const struct drm_driver amdgpu_kms_driver;
109 
110 const char *amdgpu_asic_name[] = {
111 	"TAHITI",
112 	"PITCAIRN",
113 	"VERDE",
114 	"OLAND",
115 	"HAINAN",
116 	"BONAIRE",
117 	"KAVERI",
118 	"KABINI",
119 	"HAWAII",
120 	"MULLINS",
121 	"TOPAZ",
122 	"TONGA",
123 	"FIJI",
124 	"CARRIZO",
125 	"STONEY",
126 	"POLARIS10",
127 	"POLARIS11",
128 	"POLARIS12",
129 	"VEGAM",
130 	"VEGA10",
131 	"VEGA12",
132 	"VEGA20",
133 	"RAVEN",
134 	"ARCTURUS",
135 	"RENOIR",
136 	"ALDEBARAN",
137 	"NAVI10",
138 	"CYAN_SKILLFISH",
139 	"NAVI14",
140 	"NAVI12",
141 	"SIENNA_CICHLID",
142 	"NAVY_FLOUNDER",
143 	"VANGOGH",
144 	"DIMGREY_CAVEFISH",
145 	"BEIGE_GOBY",
146 	"YELLOW_CARP",
147 	"IP DISCOVERY",
148 	"LAST",
149 };
150 
151 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM  - 1, 0)
152 /*
153  * Default init level where all blocks are expected to be initialized. This is
154  * the level of initialization expected by default and also after a full reset
155  * of the device.
156  */
157 struct amdgpu_init_level amdgpu_init_default = {
158 	.level = AMDGPU_INIT_LEVEL_DEFAULT,
159 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
160 };
161 
162 struct amdgpu_init_level amdgpu_init_recovery = {
163 	.level = AMDGPU_INIT_LEVEL_RESET_RECOVERY,
164 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
165 };
166 
167 /*
168  * Minimal blocks needed to be initialized before a XGMI hive can be reset. This
169  * is used for cases like reset on initialization where the entire hive needs to
170  * be reset before first use.
171  */
172 struct amdgpu_init_level amdgpu_init_minimal_xgmi = {
173 	.level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI,
174 	.hwini_ip_block_mask =
175 		BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) |
176 		BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) |
177 		BIT(AMD_IP_BLOCK_TYPE_PSP)
178 };
179 
180 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,
181 					     enum amd_ip_block_type block)
182 {
183 	return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;
184 }
185 
186 void amdgpu_set_init_level(struct amdgpu_device *adev,
187 			   enum amdgpu_init_lvl_id lvl)
188 {
189 	switch (lvl) {
190 	case AMDGPU_INIT_LEVEL_MINIMAL_XGMI:
191 		adev->init_lvl = &amdgpu_init_minimal_xgmi;
192 		break;
193 	case AMDGPU_INIT_LEVEL_RESET_RECOVERY:
194 		adev->init_lvl = &amdgpu_init_recovery;
195 		break;
196 	case AMDGPU_INIT_LEVEL_DEFAULT:
197 		fallthrough;
198 	default:
199 		adev->init_lvl = &amdgpu_init_default;
200 		break;
201 	}
202 }
203 
204 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
205 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
206 				     void *data);
207 
208 /**
209  * DOC: pcie_replay_count
210  *
211  * The amdgpu driver provides a sysfs API for reporting the total number
212  * of PCIe replays (NAKs).
213  * The file pcie_replay_count is used for this and returns the total
214  * number of replays as a sum of the NAKs generated and NAKs received.
215  */
216 
217 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
218 		struct device_attribute *attr, char *buf)
219 {
220 	struct drm_device *ddev = dev_get_drvdata(dev);
221 	struct amdgpu_device *adev = drm_to_adev(ddev);
222 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
223 
224 	return sysfs_emit(buf, "%llu\n", cnt);
225 }
226 
227 static DEVICE_ATTR(pcie_replay_count, 0444,
228 		amdgpu_device_get_pcie_replay_count, NULL);
229 
230 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
231 					  struct bin_attribute *attr, char *buf,
232 					  loff_t ppos, size_t count)
233 {
234 	struct device *dev = kobj_to_dev(kobj);
235 	struct drm_device *ddev = dev_get_drvdata(dev);
236 	struct amdgpu_device *adev = drm_to_adev(ddev);
237 	ssize_t bytes_read;
238 
239 	switch (ppos) {
240 	case AMDGPU_SYS_REG_STATE_XGMI:
241 		bytes_read = amdgpu_asic_get_reg_state(
242 			adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
243 		break;
244 	case AMDGPU_SYS_REG_STATE_WAFL:
245 		bytes_read = amdgpu_asic_get_reg_state(
246 			adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
247 		break;
248 	case AMDGPU_SYS_REG_STATE_PCIE:
249 		bytes_read = amdgpu_asic_get_reg_state(
250 			adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
251 		break;
252 	case AMDGPU_SYS_REG_STATE_USR:
253 		bytes_read = amdgpu_asic_get_reg_state(
254 			adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
255 		break;
256 	case AMDGPU_SYS_REG_STATE_USR_1:
257 		bytes_read = amdgpu_asic_get_reg_state(
258 			adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
259 		break;
260 	default:
261 		return -EINVAL;
262 	}
263 
264 	return bytes_read;
265 }
266 
267 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
268 	 AMDGPU_SYS_REG_STATE_END);
269 
270 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
271 {
272 	int ret;
273 
274 	if (!amdgpu_asic_get_reg_state_supported(adev))
275 		return 0;
276 
277 	ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
278 
279 	return ret;
280 }
281 
282 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
283 {
284 	if (!amdgpu_asic_get_reg_state_supported(adev))
285 		return;
286 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
287 }
288 
289 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
290 {
291 	int r;
292 
293 	if (ip_block->version->funcs->suspend) {
294 		r = ip_block->version->funcs->suspend(ip_block);
295 		if (r) {
296 			dev_err(ip_block->adev->dev,
297 				"suspend of IP block <%s> failed %d\n",
298 				ip_block->version->funcs->name, r);
299 			return r;
300 		}
301 	}
302 
303 	ip_block->status.hw = false;
304 	return 0;
305 }
306 
307 int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block)
308 {
309 	int r;
310 
311 	if (ip_block->version->funcs->resume) {
312 		r = ip_block->version->funcs->resume(ip_block);
313 		if (r) {
314 			dev_err(ip_block->adev->dev,
315 				"resume of IP block <%s> failed %d\n",
316 				ip_block->version->funcs->name, r);
317 			return r;
318 		}
319 	}
320 
321 	ip_block->status.hw = true;
322 	return 0;
323 }
324 
325 /**
326  * DOC: board_info
327  *
328  * The amdgpu driver provides a sysfs API for giving board related information.
329  * It provides the form factor information in the format
330  *
331  *   type : form factor
332  *
333  * Possible form factor values
334  *
335  * - "cem"		- PCIE CEM card
336  * - "oam"		- Open Compute Accelerator Module
337  * - "unknown"	- Not known
338  *
339  */
340 
341 static ssize_t amdgpu_device_get_board_info(struct device *dev,
342 					    struct device_attribute *attr,
343 					    char *buf)
344 {
345 	struct drm_device *ddev = dev_get_drvdata(dev);
346 	struct amdgpu_device *adev = drm_to_adev(ddev);
347 	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
348 	const char *pkg;
349 
350 	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
351 		pkg_type = adev->smuio.funcs->get_pkg_type(adev);
352 
353 	switch (pkg_type) {
354 	case AMDGPU_PKG_TYPE_CEM:
355 		pkg = "cem";
356 		break;
357 	case AMDGPU_PKG_TYPE_OAM:
358 		pkg = "oam";
359 		break;
360 	default:
361 		pkg = "unknown";
362 		break;
363 	}
364 
365 	return sysfs_emit(buf, "%s : %s\n", "type", pkg);
366 }
367 
368 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
369 
370 static struct attribute *amdgpu_board_attrs[] = {
371 	&dev_attr_board_info.attr,
372 	NULL,
373 };
374 
375 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
376 					     struct attribute *attr, int n)
377 {
378 	struct device *dev = kobj_to_dev(kobj);
379 	struct drm_device *ddev = dev_get_drvdata(dev);
380 	struct amdgpu_device *adev = drm_to_adev(ddev);
381 
382 	if (adev->flags & AMD_IS_APU)
383 		return 0;
384 
385 	return attr->mode;
386 }
387 
388 static const struct attribute_group amdgpu_board_attrs_group = {
389 	.attrs = amdgpu_board_attrs,
390 	.is_visible = amdgpu_board_attrs_is_visible
391 };
392 
393 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
394 
395 
396 /**
397  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
398  *
399  * @dev: drm_device pointer
400  *
401  * Returns true if the device is a dGPU with ATPX power control,
402  * otherwise return false.
403  */
404 bool amdgpu_device_supports_px(struct drm_device *dev)
405 {
406 	struct amdgpu_device *adev = drm_to_adev(dev);
407 
408 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
409 		return true;
410 	return false;
411 }
412 
413 /**
414  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
415  *
416  * @dev: drm_device pointer
417  *
418  * Returns true if the device is a dGPU with ACPI power control,
419  * otherwise return false.
420  */
421 bool amdgpu_device_supports_boco(struct drm_device *dev)
422 {
423 	struct amdgpu_device *adev = drm_to_adev(dev);
424 
425 	if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
426 		return false;
427 
428 	if (adev->has_pr3 ||
429 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
430 		return true;
431 	return false;
432 }
433 
434 /**
435  * amdgpu_device_supports_baco - Does the device support BACO
436  *
437  * @dev: drm_device pointer
438  *
439  * Return:
440  * 1 if the device supports BACO;
441  * 3 if the device supports MACO (only works if BACO is supported)
442  * otherwise return 0.
443  */
444 int amdgpu_device_supports_baco(struct drm_device *dev)
445 {
446 	struct amdgpu_device *adev = drm_to_adev(dev);
447 
448 	return amdgpu_asic_supports_baco(adev);
449 }
450 
451 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
452 {
453 	struct drm_device *dev;
454 	int bamaco_support;
455 
456 	dev = adev_to_drm(adev);
457 
458 	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
459 	bamaco_support = amdgpu_device_supports_baco(dev);
460 
461 	switch (amdgpu_runtime_pm) {
462 	case 2:
463 		if (bamaco_support & MACO_SUPPORT) {
464 			adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
465 			dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
466 		} else if (bamaco_support == BACO_SUPPORT) {
467 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
468 			dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
469 		}
470 		break;
471 	case 1:
472 		if (bamaco_support & BACO_SUPPORT) {
473 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
474 			dev_info(adev->dev, "Forcing BACO for runtime pm\n");
475 		}
476 		break;
477 	case -1:
478 	case -2:
479 		if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */
480 			adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
481 			dev_info(adev->dev, "Using ATPX for runtime pm\n");
482 		} else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */
483 			adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
484 			dev_info(adev->dev, "Using BOCO for runtime pm\n");
485 		} else {
486 			if (!bamaco_support)
487 				goto no_runtime_pm;
488 
489 			switch (adev->asic_type) {
490 			case CHIP_VEGA20:
491 			case CHIP_ARCTURUS:
492 				/* BACO are not supported on vega20 and arctrus */
493 				break;
494 			case CHIP_VEGA10:
495 				/* enable BACO as runpm mode if noretry=0 */
496 				if (!adev->gmc.noretry)
497 					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
498 				break;
499 			default:
500 				/* enable BACO as runpm mode on CI+ */
501 				adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
502 				break;
503 			}
504 
505 			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
506 				if (bamaco_support & MACO_SUPPORT) {
507 					adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
508 					dev_info(adev->dev, "Using BAMACO for runtime pm\n");
509 				} else {
510 					dev_info(adev->dev, "Using BACO for runtime pm\n");
511 				}
512 			}
513 		}
514 		break;
515 	case 0:
516 		dev_info(adev->dev, "runtime pm is manually disabled\n");
517 		break;
518 	default:
519 		break;
520 	}
521 
522 no_runtime_pm:
523 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
524 		dev_info(adev->dev, "Runtime PM not available\n");
525 }
526 /**
527  * amdgpu_device_supports_smart_shift - Is the device dGPU with
528  * smart shift support
529  *
530  * @dev: drm_device pointer
531  *
532  * Returns true if the device is a dGPU with Smart Shift support,
533  * otherwise returns false.
534  */
535 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
536 {
537 	return (amdgpu_device_supports_boco(dev) &&
538 		amdgpu_acpi_is_power_shift_control_supported());
539 }
540 
541 /*
542  * VRAM access helper functions
543  */
544 
545 /**
546  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
547  *
548  * @adev: amdgpu_device pointer
549  * @pos: offset of the buffer in vram
550  * @buf: virtual address of the buffer in system memory
551  * @size: read/write size, sizeof(@buf) must > @size
552  * @write: true - write to vram, otherwise - read from vram
553  */
554 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
555 			     void *buf, size_t size, bool write)
556 {
557 	unsigned long flags;
558 	uint32_t hi = ~0, tmp = 0;
559 	uint32_t *data = buf;
560 	uint64_t last;
561 	int idx;
562 
563 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
564 		return;
565 
566 	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
567 
568 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
569 	for (last = pos + size; pos < last; pos += 4) {
570 		tmp = pos >> 31;
571 
572 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
573 		if (tmp != hi) {
574 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
575 			hi = tmp;
576 		}
577 		if (write)
578 			WREG32_NO_KIQ(mmMM_DATA, *data++);
579 		else
580 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
581 	}
582 
583 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
584 	drm_dev_exit(idx);
585 }
586 
587 /**
588  * amdgpu_device_aper_access - access vram by vram aperture
589  *
590  * @adev: amdgpu_device pointer
591  * @pos: offset of the buffer in vram
592  * @buf: virtual address of the buffer in system memory
593  * @size: read/write size, sizeof(@buf) must > @size
594  * @write: true - write to vram, otherwise - read from vram
595  *
596  * The return value means how many bytes have been transferred.
597  */
598 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
599 				 void *buf, size_t size, bool write)
600 {
601 #ifdef CONFIG_64BIT
602 	void __iomem *addr;
603 	size_t count = 0;
604 	uint64_t last;
605 
606 	if (!adev->mman.aper_base_kaddr)
607 		return 0;
608 
609 	last = min(pos + size, adev->gmc.visible_vram_size);
610 	if (last > pos) {
611 		addr = adev->mman.aper_base_kaddr + pos;
612 		count = last - pos;
613 
614 		if (write) {
615 			memcpy_toio(addr, buf, count);
616 			/* Make sure HDP write cache flush happens without any reordering
617 			 * after the system memory contents are sent over PCIe device
618 			 */
619 			mb();
620 			amdgpu_device_flush_hdp(adev, NULL);
621 		} else {
622 			amdgpu_device_invalidate_hdp(adev, NULL);
623 			/* Make sure HDP read cache is invalidated before issuing a read
624 			 * to the PCIe device
625 			 */
626 			mb();
627 			memcpy_fromio(buf, addr, count);
628 		}
629 
630 	}
631 
632 	return count;
633 #else
634 	return 0;
635 #endif
636 }
637 
638 /**
639  * amdgpu_device_vram_access - read/write a buffer in vram
640  *
641  * @adev: amdgpu_device pointer
642  * @pos: offset of the buffer in vram
643  * @buf: virtual address of the buffer in system memory
644  * @size: read/write size, sizeof(@buf) must > @size
645  * @write: true - write to vram, otherwise - read from vram
646  */
647 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
648 			       void *buf, size_t size, bool write)
649 {
650 	size_t count;
651 
652 	/* try to using vram apreature to access vram first */
653 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
654 	size -= count;
655 	if (size) {
656 		/* using MM to access rest vram */
657 		pos += count;
658 		buf += count;
659 		amdgpu_device_mm_access(adev, pos, buf, size, write);
660 	}
661 }
662 
663 /*
664  * register access helper functions.
665  */
666 
667 /* Check if hw access should be skipped because of hotplug or device error */
668 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
669 {
670 	if (adev->no_hw_access)
671 		return true;
672 
673 #ifdef CONFIG_LOCKDEP
674 	/*
675 	 * This is a bit complicated to understand, so worth a comment. What we assert
676 	 * here is that the GPU reset is not running on another thread in parallel.
677 	 *
678 	 * For this we trylock the read side of the reset semaphore, if that succeeds
679 	 * we know that the reset is not running in parallel.
680 	 *
681 	 * If the trylock fails we assert that we are either already holding the read
682 	 * side of the lock or are the reset thread itself and hold the write side of
683 	 * the lock.
684 	 */
685 	if (in_task()) {
686 		if (down_read_trylock(&adev->reset_domain->sem))
687 			up_read(&adev->reset_domain->sem);
688 		else
689 			lockdep_assert_held(&adev->reset_domain->sem);
690 	}
691 #endif
692 	return false;
693 }
694 
695 /**
696  * amdgpu_device_rreg - read a memory mapped IO or indirect register
697  *
698  * @adev: amdgpu_device pointer
699  * @reg: dword aligned register offset
700  * @acc_flags: access flags which require special behavior
701  *
702  * Returns the 32 bit value from the offset specified.
703  */
704 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
705 			    uint32_t reg, uint32_t acc_flags)
706 {
707 	uint32_t ret;
708 
709 	if (amdgpu_device_skip_hw_access(adev))
710 		return 0;
711 
712 	if ((reg * 4) < adev->rmmio_size) {
713 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
714 		    amdgpu_sriov_runtime(adev) &&
715 		    down_read_trylock(&adev->reset_domain->sem)) {
716 			ret = amdgpu_kiq_rreg(adev, reg, 0);
717 			up_read(&adev->reset_domain->sem);
718 		} else {
719 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
720 		}
721 	} else {
722 		ret = adev->pcie_rreg(adev, reg * 4);
723 	}
724 
725 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
726 
727 	return ret;
728 }
729 
730 /*
731  * MMIO register read with bytes helper functions
732  * @offset:bytes offset from MMIO start
733  */
734 
735 /**
736  * amdgpu_mm_rreg8 - read a memory mapped IO register
737  *
738  * @adev: amdgpu_device pointer
739  * @offset: byte aligned register offset
740  *
741  * Returns the 8 bit value from the offset specified.
742  */
743 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
744 {
745 	if (amdgpu_device_skip_hw_access(adev))
746 		return 0;
747 
748 	if (offset < adev->rmmio_size)
749 		return (readb(adev->rmmio + offset));
750 	BUG();
751 }
752 
753 
754 /**
755  * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
756  *
757  * @adev: amdgpu_device pointer
758  * @reg: dword aligned register offset
759  * @acc_flags: access flags which require special behavior
760  * @xcc_id: xcc accelerated compute core id
761  *
762  * Returns the 32 bit value from the offset specified.
763  */
764 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
765 				uint32_t reg, uint32_t acc_flags,
766 				uint32_t xcc_id)
767 {
768 	uint32_t ret, rlcg_flag;
769 
770 	if (amdgpu_device_skip_hw_access(adev))
771 		return 0;
772 
773 	if ((reg * 4) < adev->rmmio_size) {
774 		if (amdgpu_sriov_vf(adev) &&
775 		    !amdgpu_sriov_runtime(adev) &&
776 		    adev->gfx.rlc.rlcg_reg_access_supported &&
777 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
778 							 GC_HWIP, false,
779 							 &rlcg_flag)) {
780 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id));
781 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
782 		    amdgpu_sriov_runtime(adev) &&
783 		    down_read_trylock(&adev->reset_domain->sem)) {
784 			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
785 			up_read(&adev->reset_domain->sem);
786 		} else {
787 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
788 		}
789 	} else {
790 		ret = adev->pcie_rreg(adev, reg * 4);
791 	}
792 
793 	return ret;
794 }
795 
796 /*
797  * MMIO register write with bytes helper functions
798  * @offset:bytes offset from MMIO start
799  * @value: the value want to be written to the register
800  */
801 
802 /**
803  * amdgpu_mm_wreg8 - read a memory mapped IO register
804  *
805  * @adev: amdgpu_device pointer
806  * @offset: byte aligned register offset
807  * @value: 8 bit value to write
808  *
809  * Writes the value specified to the offset specified.
810  */
811 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
812 {
813 	if (amdgpu_device_skip_hw_access(adev))
814 		return;
815 
816 	if (offset < adev->rmmio_size)
817 		writeb(value, adev->rmmio + offset);
818 	else
819 		BUG();
820 }
821 
822 /**
823  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
824  *
825  * @adev: amdgpu_device pointer
826  * @reg: dword aligned register offset
827  * @v: 32 bit value to write to the register
828  * @acc_flags: access flags which require special behavior
829  *
830  * Writes the value specified to the offset specified.
831  */
832 void amdgpu_device_wreg(struct amdgpu_device *adev,
833 			uint32_t reg, uint32_t v,
834 			uint32_t acc_flags)
835 {
836 	if (amdgpu_device_skip_hw_access(adev))
837 		return;
838 
839 	if ((reg * 4) < adev->rmmio_size) {
840 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
841 		    amdgpu_sriov_runtime(adev) &&
842 		    down_read_trylock(&adev->reset_domain->sem)) {
843 			amdgpu_kiq_wreg(adev, reg, v, 0);
844 			up_read(&adev->reset_domain->sem);
845 		} else {
846 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
847 		}
848 	} else {
849 		adev->pcie_wreg(adev, reg * 4, v);
850 	}
851 
852 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
853 }
854 
855 /**
856  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
857  *
858  * @adev: amdgpu_device pointer
859  * @reg: mmio/rlc register
860  * @v: value to write
861  * @xcc_id: xcc accelerated compute core id
862  *
863  * this function is invoked only for the debugfs register access
864  */
865 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
866 			     uint32_t reg, uint32_t v,
867 			     uint32_t xcc_id)
868 {
869 	if (amdgpu_device_skip_hw_access(adev))
870 		return;
871 
872 	if (amdgpu_sriov_fullaccess(adev) &&
873 	    adev->gfx.rlc.funcs &&
874 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
875 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
876 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
877 	} else if ((reg * 4) >= adev->rmmio_size) {
878 		adev->pcie_wreg(adev, reg * 4, v);
879 	} else {
880 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
881 	}
882 }
883 
884 /**
885  * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
886  *
887  * @adev: amdgpu_device pointer
888  * @reg: dword aligned register offset
889  * @v: 32 bit value to write to the register
890  * @acc_flags: access flags which require special behavior
891  * @xcc_id: xcc accelerated compute core id
892  *
893  * Writes the value specified to the offset specified.
894  */
895 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
896 			uint32_t reg, uint32_t v,
897 			uint32_t acc_flags, uint32_t xcc_id)
898 {
899 	uint32_t rlcg_flag;
900 
901 	if (amdgpu_device_skip_hw_access(adev))
902 		return;
903 
904 	if ((reg * 4) < adev->rmmio_size) {
905 		if (amdgpu_sriov_vf(adev) &&
906 		    !amdgpu_sriov_runtime(adev) &&
907 		    adev->gfx.rlc.rlcg_reg_access_supported &&
908 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
909 							 GC_HWIP, true,
910 							 &rlcg_flag)) {
911 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id));
912 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
913 		    amdgpu_sriov_runtime(adev) &&
914 		    down_read_trylock(&adev->reset_domain->sem)) {
915 			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
916 			up_read(&adev->reset_domain->sem);
917 		} else {
918 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
919 		}
920 	} else {
921 		adev->pcie_wreg(adev, reg * 4, v);
922 	}
923 }
924 
925 /**
926  * amdgpu_device_indirect_rreg - read an indirect register
927  *
928  * @adev: amdgpu_device pointer
929  * @reg_addr: indirect register address to read from
930  *
931  * Returns the value of indirect register @reg_addr
932  */
933 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
934 				u32 reg_addr)
935 {
936 	unsigned long flags, pcie_index, pcie_data;
937 	void __iomem *pcie_index_offset;
938 	void __iomem *pcie_data_offset;
939 	u32 r;
940 
941 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
942 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
943 
944 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
945 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
946 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
947 
948 	writel(reg_addr, pcie_index_offset);
949 	readl(pcie_index_offset);
950 	r = readl(pcie_data_offset);
951 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
952 
953 	return r;
954 }
955 
956 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
957 				    u64 reg_addr)
958 {
959 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
960 	u32 r;
961 	void __iomem *pcie_index_offset;
962 	void __iomem *pcie_index_hi_offset;
963 	void __iomem *pcie_data_offset;
964 
965 	if (unlikely(!adev->nbio.funcs)) {
966 		pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
967 		pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
968 	} else {
969 		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
970 		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
971 	}
972 
973 	if (reg_addr >> 32) {
974 		if (unlikely(!adev->nbio.funcs))
975 			pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
976 		else
977 			pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
978 	} else {
979 		pcie_index_hi = 0;
980 	}
981 
982 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
983 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
984 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
985 	if (pcie_index_hi != 0)
986 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
987 				pcie_index_hi * 4;
988 
989 	writel(reg_addr, pcie_index_offset);
990 	readl(pcie_index_offset);
991 	if (pcie_index_hi != 0) {
992 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
993 		readl(pcie_index_hi_offset);
994 	}
995 	r = readl(pcie_data_offset);
996 
997 	/* clear the high bits */
998 	if (pcie_index_hi != 0) {
999 		writel(0, pcie_index_hi_offset);
1000 		readl(pcie_index_hi_offset);
1001 	}
1002 
1003 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1004 
1005 	return r;
1006 }
1007 
1008 /**
1009  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
1010  *
1011  * @adev: amdgpu_device pointer
1012  * @reg_addr: indirect register address to read from
1013  *
1014  * Returns the value of indirect register @reg_addr
1015  */
1016 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
1017 				  u32 reg_addr)
1018 {
1019 	unsigned long flags, pcie_index, pcie_data;
1020 	void __iomem *pcie_index_offset;
1021 	void __iomem *pcie_data_offset;
1022 	u64 r;
1023 
1024 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1025 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1026 
1027 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1028 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1029 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1030 
1031 	/* read low 32 bits */
1032 	writel(reg_addr, pcie_index_offset);
1033 	readl(pcie_index_offset);
1034 	r = readl(pcie_data_offset);
1035 	/* read high 32 bits */
1036 	writel(reg_addr + 4, pcie_index_offset);
1037 	readl(pcie_index_offset);
1038 	r |= ((u64)readl(pcie_data_offset) << 32);
1039 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1040 
1041 	return r;
1042 }
1043 
1044 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
1045 				  u64 reg_addr)
1046 {
1047 	unsigned long flags, pcie_index, pcie_data;
1048 	unsigned long pcie_index_hi = 0;
1049 	void __iomem *pcie_index_offset;
1050 	void __iomem *pcie_index_hi_offset;
1051 	void __iomem *pcie_data_offset;
1052 	u64 r;
1053 
1054 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1055 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1056 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1057 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1058 
1059 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1060 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1061 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1062 	if (pcie_index_hi != 0)
1063 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1064 			pcie_index_hi * 4;
1065 
1066 	/* read low 32 bits */
1067 	writel(reg_addr, pcie_index_offset);
1068 	readl(pcie_index_offset);
1069 	if (pcie_index_hi != 0) {
1070 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1071 		readl(pcie_index_hi_offset);
1072 	}
1073 	r = readl(pcie_data_offset);
1074 	/* read high 32 bits */
1075 	writel(reg_addr + 4, pcie_index_offset);
1076 	readl(pcie_index_offset);
1077 	if (pcie_index_hi != 0) {
1078 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1079 		readl(pcie_index_hi_offset);
1080 	}
1081 	r |= ((u64)readl(pcie_data_offset) << 32);
1082 
1083 	/* clear the high bits */
1084 	if (pcie_index_hi != 0) {
1085 		writel(0, pcie_index_hi_offset);
1086 		readl(pcie_index_hi_offset);
1087 	}
1088 
1089 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1090 
1091 	return r;
1092 }
1093 
1094 /**
1095  * amdgpu_device_indirect_wreg - write an indirect register address
1096  *
1097  * @adev: amdgpu_device pointer
1098  * @reg_addr: indirect register offset
1099  * @reg_data: indirect register data
1100  *
1101  */
1102 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
1103 				 u32 reg_addr, u32 reg_data)
1104 {
1105 	unsigned long flags, pcie_index, pcie_data;
1106 	void __iomem *pcie_index_offset;
1107 	void __iomem *pcie_data_offset;
1108 
1109 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1110 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1111 
1112 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1113 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1114 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1115 
1116 	writel(reg_addr, pcie_index_offset);
1117 	readl(pcie_index_offset);
1118 	writel(reg_data, pcie_data_offset);
1119 	readl(pcie_data_offset);
1120 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1121 }
1122 
1123 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
1124 				     u64 reg_addr, u32 reg_data)
1125 {
1126 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
1127 	void __iomem *pcie_index_offset;
1128 	void __iomem *pcie_index_hi_offset;
1129 	void __iomem *pcie_data_offset;
1130 
1131 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1132 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1133 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1134 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1135 	else
1136 		pcie_index_hi = 0;
1137 
1138 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1139 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1140 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1141 	if (pcie_index_hi != 0)
1142 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1143 				pcie_index_hi * 4;
1144 
1145 	writel(reg_addr, pcie_index_offset);
1146 	readl(pcie_index_offset);
1147 	if (pcie_index_hi != 0) {
1148 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1149 		readl(pcie_index_hi_offset);
1150 	}
1151 	writel(reg_data, pcie_data_offset);
1152 	readl(pcie_data_offset);
1153 
1154 	/* clear the high bits */
1155 	if (pcie_index_hi != 0) {
1156 		writel(0, pcie_index_hi_offset);
1157 		readl(pcie_index_hi_offset);
1158 	}
1159 
1160 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1161 }
1162 
1163 /**
1164  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
1165  *
1166  * @adev: amdgpu_device pointer
1167  * @reg_addr: indirect register offset
1168  * @reg_data: indirect register data
1169  *
1170  */
1171 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
1172 				   u32 reg_addr, u64 reg_data)
1173 {
1174 	unsigned long flags, pcie_index, pcie_data;
1175 	void __iomem *pcie_index_offset;
1176 	void __iomem *pcie_data_offset;
1177 
1178 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1179 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1180 
1181 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1182 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1183 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1184 
1185 	/* write low 32 bits */
1186 	writel(reg_addr, pcie_index_offset);
1187 	readl(pcie_index_offset);
1188 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1189 	readl(pcie_data_offset);
1190 	/* write high 32 bits */
1191 	writel(reg_addr + 4, pcie_index_offset);
1192 	readl(pcie_index_offset);
1193 	writel((u32)(reg_data >> 32), pcie_data_offset);
1194 	readl(pcie_data_offset);
1195 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1196 }
1197 
1198 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1199 				   u64 reg_addr, u64 reg_data)
1200 {
1201 	unsigned long flags, pcie_index, pcie_data;
1202 	unsigned long pcie_index_hi = 0;
1203 	void __iomem *pcie_index_offset;
1204 	void __iomem *pcie_index_hi_offset;
1205 	void __iomem *pcie_data_offset;
1206 
1207 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1208 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1209 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1210 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1211 
1212 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1213 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1214 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1215 	if (pcie_index_hi != 0)
1216 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1217 				pcie_index_hi * 4;
1218 
1219 	/* write low 32 bits */
1220 	writel(reg_addr, pcie_index_offset);
1221 	readl(pcie_index_offset);
1222 	if (pcie_index_hi != 0) {
1223 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1224 		readl(pcie_index_hi_offset);
1225 	}
1226 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1227 	readl(pcie_data_offset);
1228 	/* write high 32 bits */
1229 	writel(reg_addr + 4, pcie_index_offset);
1230 	readl(pcie_index_offset);
1231 	if (pcie_index_hi != 0) {
1232 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1233 		readl(pcie_index_hi_offset);
1234 	}
1235 	writel((u32)(reg_data >> 32), pcie_data_offset);
1236 	readl(pcie_data_offset);
1237 
1238 	/* clear the high bits */
1239 	if (pcie_index_hi != 0) {
1240 		writel(0, pcie_index_hi_offset);
1241 		readl(pcie_index_hi_offset);
1242 	}
1243 
1244 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1245 }
1246 
1247 /**
1248  * amdgpu_device_get_rev_id - query device rev_id
1249  *
1250  * @adev: amdgpu_device pointer
1251  *
1252  * Return device rev_id
1253  */
1254 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1255 {
1256 	return adev->nbio.funcs->get_rev_id(adev);
1257 }
1258 
1259 /**
1260  * amdgpu_invalid_rreg - dummy reg read function
1261  *
1262  * @adev: amdgpu_device pointer
1263  * @reg: offset of register
1264  *
1265  * Dummy register read function.  Used for register blocks
1266  * that certain asics don't have (all asics).
1267  * Returns the value in the register.
1268  */
1269 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1270 {
1271 	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1272 	BUG();
1273 	return 0;
1274 }
1275 
1276 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1277 {
1278 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1279 	BUG();
1280 	return 0;
1281 }
1282 
1283 /**
1284  * amdgpu_invalid_wreg - dummy reg write function
1285  *
1286  * @adev: amdgpu_device pointer
1287  * @reg: offset of register
1288  * @v: value to write to the register
1289  *
1290  * Dummy register read function.  Used for register blocks
1291  * that certain asics don't have (all asics).
1292  */
1293 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1294 {
1295 	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1296 		  reg, v);
1297 	BUG();
1298 }
1299 
1300 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1301 {
1302 	DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1303 		  reg, v);
1304 	BUG();
1305 }
1306 
1307 /**
1308  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1309  *
1310  * @adev: amdgpu_device pointer
1311  * @reg: offset of register
1312  *
1313  * Dummy register read function.  Used for register blocks
1314  * that certain asics don't have (all asics).
1315  * Returns the value in the register.
1316  */
1317 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1318 {
1319 	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1320 	BUG();
1321 	return 0;
1322 }
1323 
1324 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1325 {
1326 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1327 	BUG();
1328 	return 0;
1329 }
1330 
1331 /**
1332  * amdgpu_invalid_wreg64 - dummy reg write function
1333  *
1334  * @adev: amdgpu_device pointer
1335  * @reg: offset of register
1336  * @v: value to write to the register
1337  *
1338  * Dummy register read function.  Used for register blocks
1339  * that certain asics don't have (all asics).
1340  */
1341 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1342 {
1343 	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1344 		  reg, v);
1345 	BUG();
1346 }
1347 
1348 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1349 {
1350 	DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1351 		  reg, v);
1352 	BUG();
1353 }
1354 
1355 /**
1356  * amdgpu_block_invalid_rreg - dummy reg read function
1357  *
1358  * @adev: amdgpu_device pointer
1359  * @block: offset of instance
1360  * @reg: offset of register
1361  *
1362  * Dummy register read function.  Used for register blocks
1363  * that certain asics don't have (all asics).
1364  * Returns the value in the register.
1365  */
1366 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1367 					  uint32_t block, uint32_t reg)
1368 {
1369 	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1370 		  reg, block);
1371 	BUG();
1372 	return 0;
1373 }
1374 
1375 /**
1376  * amdgpu_block_invalid_wreg - dummy reg write function
1377  *
1378  * @adev: amdgpu_device pointer
1379  * @block: offset of instance
1380  * @reg: offset of register
1381  * @v: value to write to the register
1382  *
1383  * Dummy register read function.  Used for register blocks
1384  * that certain asics don't have (all asics).
1385  */
1386 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1387 				      uint32_t block,
1388 				      uint32_t reg, uint32_t v)
1389 {
1390 	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1391 		  reg, block, v);
1392 	BUG();
1393 }
1394 
1395 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)
1396 {
1397 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1398 		return AMDGPU_VBIOS_SKIP;
1399 
1400 	if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))
1401 		return AMDGPU_VBIOS_OPTIONAL;
1402 
1403 	return 0;
1404 }
1405 
1406 /**
1407  * amdgpu_device_asic_init - Wrapper for atom asic_init
1408  *
1409  * @adev: amdgpu_device pointer
1410  *
1411  * Does any asic specific work and then calls atom asic init.
1412  */
1413 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1414 {
1415 	uint32_t flags;
1416 	bool optional;
1417 	int ret;
1418 
1419 	amdgpu_asic_pre_asic_init(adev);
1420 	flags = amdgpu_device_get_vbios_flags(adev);
1421 	optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP));
1422 
1423 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1424 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
1425 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
1426 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1427 		amdgpu_psp_wait_for_bootloader(adev);
1428 		if (optional && !adev->bios)
1429 			return 0;
1430 
1431 		ret = amdgpu_atomfirmware_asic_init(adev, true);
1432 		return ret;
1433 	} else {
1434 		if (optional && !adev->bios)
1435 			return 0;
1436 
1437 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1438 	}
1439 
1440 	return 0;
1441 }
1442 
1443 /**
1444  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1445  *
1446  * @adev: amdgpu_device pointer
1447  *
1448  * Allocates a scratch page of VRAM for use by various things in the
1449  * driver.
1450  */
1451 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1452 {
1453 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1454 				       AMDGPU_GEM_DOMAIN_VRAM |
1455 				       AMDGPU_GEM_DOMAIN_GTT,
1456 				       &adev->mem_scratch.robj,
1457 				       &adev->mem_scratch.gpu_addr,
1458 				       (void **)&adev->mem_scratch.ptr);
1459 }
1460 
1461 /**
1462  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1463  *
1464  * @adev: amdgpu_device pointer
1465  *
1466  * Frees the VRAM scratch page.
1467  */
1468 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1469 {
1470 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1471 }
1472 
1473 /**
1474  * amdgpu_device_program_register_sequence - program an array of registers.
1475  *
1476  * @adev: amdgpu_device pointer
1477  * @registers: pointer to the register array
1478  * @array_size: size of the register array
1479  *
1480  * Programs an array or registers with and or masks.
1481  * This is a helper for setting golden registers.
1482  */
1483 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1484 					     const u32 *registers,
1485 					     const u32 array_size)
1486 {
1487 	u32 tmp, reg, and_mask, or_mask;
1488 	int i;
1489 
1490 	if (array_size % 3)
1491 		return;
1492 
1493 	for (i = 0; i < array_size; i += 3) {
1494 		reg = registers[i + 0];
1495 		and_mask = registers[i + 1];
1496 		or_mask = registers[i + 2];
1497 
1498 		if (and_mask == 0xffffffff) {
1499 			tmp = or_mask;
1500 		} else {
1501 			tmp = RREG32(reg);
1502 			tmp &= ~and_mask;
1503 			if (adev->family >= AMDGPU_FAMILY_AI)
1504 				tmp |= (or_mask & and_mask);
1505 			else
1506 				tmp |= or_mask;
1507 		}
1508 		WREG32(reg, tmp);
1509 	}
1510 }
1511 
1512 /**
1513  * amdgpu_device_pci_config_reset - reset the GPU
1514  *
1515  * @adev: amdgpu_device pointer
1516  *
1517  * Resets the GPU using the pci config reset sequence.
1518  * Only applicable to asics prior to vega10.
1519  */
1520 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1521 {
1522 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1523 }
1524 
1525 /**
1526  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1527  *
1528  * @adev: amdgpu_device pointer
1529  *
1530  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1531  */
1532 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1533 {
1534 	return pci_reset_function(adev->pdev);
1535 }
1536 
1537 /*
1538  * amdgpu_device_wb_*()
1539  * Writeback is the method by which the GPU updates special pages in memory
1540  * with the status of certain GPU events (fences, ring pointers,etc.).
1541  */
1542 
1543 /**
1544  * amdgpu_device_wb_fini - Disable Writeback and free memory
1545  *
1546  * @adev: amdgpu_device pointer
1547  *
1548  * Disables Writeback and frees the Writeback memory (all asics).
1549  * Used at driver shutdown.
1550  */
1551 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1552 {
1553 	if (adev->wb.wb_obj) {
1554 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1555 				      &adev->wb.gpu_addr,
1556 				      (void **)&adev->wb.wb);
1557 		adev->wb.wb_obj = NULL;
1558 	}
1559 }
1560 
1561 /**
1562  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1563  *
1564  * @adev: amdgpu_device pointer
1565  *
1566  * Initializes writeback and allocates writeback memory (all asics).
1567  * Used at driver startup.
1568  * Returns 0 on success or an -error on failure.
1569  */
1570 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1571 {
1572 	int r;
1573 
1574 	if (adev->wb.wb_obj == NULL) {
1575 		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1576 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1577 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1578 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
1579 					    (void **)&adev->wb.wb);
1580 		if (r) {
1581 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1582 			return r;
1583 		}
1584 
1585 		adev->wb.num_wb = AMDGPU_MAX_WB;
1586 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1587 
1588 		/* clear wb memory */
1589 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1590 	}
1591 
1592 	return 0;
1593 }
1594 
1595 /**
1596  * amdgpu_device_wb_get - Allocate a wb entry
1597  *
1598  * @adev: amdgpu_device pointer
1599  * @wb: wb index
1600  *
1601  * Allocate a wb slot for use by the driver (all asics).
1602  * Returns 0 on success or -EINVAL on failure.
1603  */
1604 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1605 {
1606 	unsigned long flags, offset;
1607 
1608 	spin_lock_irqsave(&adev->wb.lock, flags);
1609 	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1610 	if (offset < adev->wb.num_wb) {
1611 		__set_bit(offset, adev->wb.used);
1612 		spin_unlock_irqrestore(&adev->wb.lock, flags);
1613 		*wb = offset << 3; /* convert to dw offset */
1614 		return 0;
1615 	} else {
1616 		spin_unlock_irqrestore(&adev->wb.lock, flags);
1617 		return -EINVAL;
1618 	}
1619 }
1620 
1621 /**
1622  * amdgpu_device_wb_free - Free a wb entry
1623  *
1624  * @adev: amdgpu_device pointer
1625  * @wb: wb index
1626  *
1627  * Free a wb slot allocated for use by the driver (all asics)
1628  */
1629 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1630 {
1631 	unsigned long flags;
1632 
1633 	wb >>= 3;
1634 	spin_lock_irqsave(&adev->wb.lock, flags);
1635 	if (wb < adev->wb.num_wb)
1636 		__clear_bit(wb, adev->wb.used);
1637 	spin_unlock_irqrestore(&adev->wb.lock, flags);
1638 }
1639 
1640 /**
1641  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1642  *
1643  * @adev: amdgpu_device pointer
1644  *
1645  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1646  * to fail, but if any of the BARs is not accessible after the size we abort
1647  * driver loading by returning -ENODEV.
1648  */
1649 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1650 {
1651 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1652 	struct pci_bus *root;
1653 	struct resource *res;
1654 	unsigned int i;
1655 	u16 cmd;
1656 	int r;
1657 
1658 	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1659 		return 0;
1660 
1661 	/* Bypass for VF */
1662 	if (amdgpu_sriov_vf(adev))
1663 		return 0;
1664 
1665 	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1666 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1667 		DRM_WARN("System can't access extended configuration space, please check!!\n");
1668 
1669 	/* skip if the bios has already enabled large BAR */
1670 	if (adev->gmc.real_vram_size &&
1671 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1672 		return 0;
1673 
1674 	/* Check if the root BUS has 64bit memory resources */
1675 	root = adev->pdev->bus;
1676 	while (root->parent)
1677 		root = root->parent;
1678 
1679 	pci_bus_for_each_resource(root, res, i) {
1680 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1681 		    res->start > 0x100000000ull)
1682 			break;
1683 	}
1684 
1685 	/* Trying to resize is pointless without a root hub window above 4GB */
1686 	if (!res)
1687 		return 0;
1688 
1689 	/* Limit the BAR size to what is available */
1690 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1691 			rbar_size);
1692 
1693 	/* Disable memory decoding while we change the BAR addresses and size */
1694 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1695 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1696 			      cmd & ~PCI_COMMAND_MEMORY);
1697 
1698 	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1699 	amdgpu_doorbell_fini(adev);
1700 	if (adev->asic_type >= CHIP_BONAIRE)
1701 		pci_release_resource(adev->pdev, 2);
1702 
1703 	pci_release_resource(adev->pdev, 0);
1704 
1705 	r = pci_resize_resource(adev->pdev, 0, rbar_size);
1706 	if (r == -ENOSPC)
1707 		DRM_INFO("Not enough PCI address space for a large BAR.");
1708 	else if (r && r != -ENOTSUPP)
1709 		DRM_ERROR("Problem resizing BAR0 (%d).", r);
1710 
1711 	pci_assign_unassigned_bus_resources(adev->pdev->bus);
1712 
1713 	/* When the doorbell or fb BAR isn't available we have no chance of
1714 	 * using the device.
1715 	 */
1716 	r = amdgpu_doorbell_init(adev);
1717 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1718 		return -ENODEV;
1719 
1720 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1721 
1722 	return 0;
1723 }
1724 
1725 /*
1726  * GPU helpers function.
1727  */
1728 /**
1729  * amdgpu_device_need_post - check if the hw need post or not
1730  *
1731  * @adev: amdgpu_device pointer
1732  *
1733  * Check if the asic has been initialized (all asics) at driver startup
1734  * or post is needed if  hw reset is performed.
1735  * Returns true if need or false if not.
1736  */
1737 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1738 {
1739 	uint32_t reg, flags;
1740 
1741 	if (amdgpu_sriov_vf(adev))
1742 		return false;
1743 
1744 	flags = amdgpu_device_get_vbios_flags(adev);
1745 	if (flags & AMDGPU_VBIOS_SKIP)
1746 		return false;
1747 	if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)
1748 		return false;
1749 
1750 	if (amdgpu_passthrough(adev)) {
1751 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1752 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1753 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1754 		 * vpost executed for smc version below 22.15
1755 		 */
1756 		if (adev->asic_type == CHIP_FIJI) {
1757 			int err;
1758 			uint32_t fw_ver;
1759 
1760 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1761 			/* force vPost if error occurred */
1762 			if (err)
1763 				return true;
1764 
1765 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1766 			release_firmware(adev->pm.fw);
1767 			if (fw_ver < 0x00160e00)
1768 				return true;
1769 		}
1770 	}
1771 
1772 	/* Don't post if we need to reset whole hive on init */
1773 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
1774 		return false;
1775 
1776 	if (adev->has_hw_reset) {
1777 		adev->has_hw_reset = false;
1778 		return true;
1779 	}
1780 
1781 	/* bios scratch used on CIK+ */
1782 	if (adev->asic_type >= CHIP_BONAIRE)
1783 		return amdgpu_atombios_scratch_need_asic_init(adev);
1784 
1785 	/* check MEM_SIZE for older asics */
1786 	reg = amdgpu_asic_get_config_memsize(adev);
1787 
1788 	if ((reg != 0) && (reg != 0xffffffff))
1789 		return false;
1790 
1791 	return true;
1792 }
1793 
1794 /*
1795  * Check whether seamless boot is supported.
1796  *
1797  * So far we only support seamless boot on DCE 3.0 or later.
1798  * If users report that it works on older ASICS as well, we may
1799  * loosen this.
1800  */
1801 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1802 {
1803 	switch (amdgpu_seamless) {
1804 	case -1:
1805 		break;
1806 	case 1:
1807 		return true;
1808 	case 0:
1809 		return false;
1810 	default:
1811 		DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1812 			  amdgpu_seamless);
1813 		return false;
1814 	}
1815 
1816 	if (!(adev->flags & AMD_IS_APU))
1817 		return false;
1818 
1819 	if (adev->mman.keep_stolen_vga_memory)
1820 		return false;
1821 
1822 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1823 }
1824 
1825 /*
1826  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1827  * don't support dynamic speed switching. Until we have confirmation from Intel
1828  * that a specific host supports it, it's safer that we keep it disabled for all.
1829  *
1830  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1831  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1832  */
1833 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1834 {
1835 #if IS_ENABLED(CONFIG_X86)
1836 	struct cpuinfo_x86 *c = &cpu_data(0);
1837 
1838 	/* eGPU change speeds based on USB4 fabric conditions */
1839 	if (dev_is_removable(adev->dev))
1840 		return true;
1841 
1842 	if (c->x86_vendor == X86_VENDOR_INTEL)
1843 		return false;
1844 #endif
1845 	return true;
1846 }
1847 
1848 /**
1849  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1850  *
1851  * @adev: amdgpu_device pointer
1852  *
1853  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1854  * be set for this device.
1855  *
1856  * Returns true if it should be used or false if not.
1857  */
1858 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1859 {
1860 	switch (amdgpu_aspm) {
1861 	case -1:
1862 		break;
1863 	case 0:
1864 		return false;
1865 	case 1:
1866 		return true;
1867 	default:
1868 		return false;
1869 	}
1870 	if (adev->flags & AMD_IS_APU)
1871 		return false;
1872 	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1873 		return false;
1874 	return pcie_aspm_enabled(adev->pdev);
1875 }
1876 
1877 /* if we get transitioned to only one device, take VGA back */
1878 /**
1879  * amdgpu_device_vga_set_decode - enable/disable vga decode
1880  *
1881  * @pdev: PCI device pointer
1882  * @state: enable/disable vga decode
1883  *
1884  * Enable/disable vga decode (all asics).
1885  * Returns VGA resource flags.
1886  */
1887 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1888 		bool state)
1889 {
1890 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1891 
1892 	amdgpu_asic_set_vga_state(adev, state);
1893 	if (state)
1894 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1895 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1896 	else
1897 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1898 }
1899 
1900 /**
1901  * amdgpu_device_check_block_size - validate the vm block size
1902  *
1903  * @adev: amdgpu_device pointer
1904  *
1905  * Validates the vm block size specified via module parameter.
1906  * The vm block size defines number of bits in page table versus page directory,
1907  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1908  * page table and the remaining bits are in the page directory.
1909  */
1910 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1911 {
1912 	/* defines number of bits in page table versus page directory,
1913 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1914 	 * page table and the remaining bits are in the page directory
1915 	 */
1916 	if (amdgpu_vm_block_size == -1)
1917 		return;
1918 
1919 	if (amdgpu_vm_block_size < 9) {
1920 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1921 			 amdgpu_vm_block_size);
1922 		amdgpu_vm_block_size = -1;
1923 	}
1924 }
1925 
1926 /**
1927  * amdgpu_device_check_vm_size - validate the vm size
1928  *
1929  * @adev: amdgpu_device pointer
1930  *
1931  * Validates the vm size in GB specified via module parameter.
1932  * The VM size is the size of the GPU virtual memory space in GB.
1933  */
1934 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1935 {
1936 	/* no need to check the default value */
1937 	if (amdgpu_vm_size == -1)
1938 		return;
1939 
1940 	if (amdgpu_vm_size < 1) {
1941 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1942 			 amdgpu_vm_size);
1943 		amdgpu_vm_size = -1;
1944 	}
1945 }
1946 
1947 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1948 {
1949 	struct sysinfo si;
1950 	bool is_os_64 = (sizeof(void *) == 8);
1951 	uint64_t total_memory;
1952 	uint64_t dram_size_seven_GB = 0x1B8000000;
1953 	uint64_t dram_size_three_GB = 0xB8000000;
1954 
1955 	if (amdgpu_smu_memory_pool_size == 0)
1956 		return;
1957 
1958 	if (!is_os_64) {
1959 		DRM_WARN("Not 64-bit OS, feature not supported\n");
1960 		goto def_value;
1961 	}
1962 	si_meminfo(&si);
1963 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1964 
1965 	if ((amdgpu_smu_memory_pool_size == 1) ||
1966 		(amdgpu_smu_memory_pool_size == 2)) {
1967 		if (total_memory < dram_size_three_GB)
1968 			goto def_value1;
1969 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1970 		(amdgpu_smu_memory_pool_size == 8)) {
1971 		if (total_memory < dram_size_seven_GB)
1972 			goto def_value1;
1973 	} else {
1974 		DRM_WARN("Smu memory pool size not supported\n");
1975 		goto def_value;
1976 	}
1977 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1978 
1979 	return;
1980 
1981 def_value1:
1982 	DRM_WARN("No enough system memory\n");
1983 def_value:
1984 	adev->pm.smu_prv_buffer_size = 0;
1985 }
1986 
1987 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1988 {
1989 	if (!(adev->flags & AMD_IS_APU) ||
1990 	    adev->asic_type < CHIP_RAVEN)
1991 		return 0;
1992 
1993 	switch (adev->asic_type) {
1994 	case CHIP_RAVEN:
1995 		if (adev->pdev->device == 0x15dd)
1996 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1997 		if (adev->pdev->device == 0x15d8)
1998 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1999 		break;
2000 	case CHIP_RENOIR:
2001 		if ((adev->pdev->device == 0x1636) ||
2002 		    (adev->pdev->device == 0x164c))
2003 			adev->apu_flags |= AMD_APU_IS_RENOIR;
2004 		else
2005 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
2006 		break;
2007 	case CHIP_VANGOGH:
2008 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
2009 		break;
2010 	case CHIP_YELLOW_CARP:
2011 		break;
2012 	case CHIP_CYAN_SKILLFISH:
2013 		if ((adev->pdev->device == 0x13FE) ||
2014 		    (adev->pdev->device == 0x143F))
2015 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
2016 		break;
2017 	default:
2018 		break;
2019 	}
2020 
2021 	return 0;
2022 }
2023 
2024 /**
2025  * amdgpu_device_check_arguments - validate module params
2026  *
2027  * @adev: amdgpu_device pointer
2028  *
2029  * Validates certain module parameters and updates
2030  * the associated values used by the driver (all asics).
2031  */
2032 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
2033 {
2034 	int i;
2035 
2036 	if (amdgpu_sched_jobs < 4) {
2037 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
2038 			 amdgpu_sched_jobs);
2039 		amdgpu_sched_jobs = 4;
2040 	} else if (!is_power_of_2(amdgpu_sched_jobs)) {
2041 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
2042 			 amdgpu_sched_jobs);
2043 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
2044 	}
2045 
2046 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
2047 		/* gart size must be greater or equal to 32M */
2048 		dev_warn(adev->dev, "gart size (%d) too small\n",
2049 			 amdgpu_gart_size);
2050 		amdgpu_gart_size = -1;
2051 	}
2052 
2053 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
2054 		/* gtt size must be greater or equal to 32M */
2055 		dev_warn(adev->dev, "gtt size (%d) too small\n",
2056 				 amdgpu_gtt_size);
2057 		amdgpu_gtt_size = -1;
2058 	}
2059 
2060 	/* valid range is between 4 and 9 inclusive */
2061 	if (amdgpu_vm_fragment_size != -1 &&
2062 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
2063 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
2064 		amdgpu_vm_fragment_size = -1;
2065 	}
2066 
2067 	if (amdgpu_sched_hw_submission < 2) {
2068 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
2069 			 amdgpu_sched_hw_submission);
2070 		amdgpu_sched_hw_submission = 2;
2071 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
2072 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
2073 			 amdgpu_sched_hw_submission);
2074 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
2075 	}
2076 
2077 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
2078 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
2079 		amdgpu_reset_method = -1;
2080 	}
2081 
2082 	amdgpu_device_check_smu_prv_buffer_size(adev);
2083 
2084 	amdgpu_device_check_vm_size(adev);
2085 
2086 	amdgpu_device_check_block_size(adev);
2087 
2088 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
2089 
2090 	for (i = 0; i < MAX_XCP; i++)
2091 		adev->enforce_isolation[i] = !!enforce_isolation;
2092 
2093 	return 0;
2094 }
2095 
2096 /**
2097  * amdgpu_switcheroo_set_state - set switcheroo state
2098  *
2099  * @pdev: pci dev pointer
2100  * @state: vga_switcheroo state
2101  *
2102  * Callback for the switcheroo driver.  Suspends or resumes
2103  * the asics before or after it is powered up using ACPI methods.
2104  */
2105 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
2106 					enum vga_switcheroo_state state)
2107 {
2108 	struct drm_device *dev = pci_get_drvdata(pdev);
2109 	int r;
2110 
2111 	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
2112 		return;
2113 
2114 	if (state == VGA_SWITCHEROO_ON) {
2115 		pr_info("switched on\n");
2116 		/* don't suspend or resume card normally */
2117 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2118 
2119 		pci_set_power_state(pdev, PCI_D0);
2120 		amdgpu_device_load_pci_state(pdev);
2121 		r = pci_enable_device(pdev);
2122 		if (r)
2123 			DRM_WARN("pci_enable_device failed (%d)\n", r);
2124 		amdgpu_device_resume(dev, true);
2125 
2126 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
2127 	} else {
2128 		pr_info("switched off\n");
2129 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
2130 		amdgpu_device_prepare(dev);
2131 		amdgpu_device_suspend(dev, true);
2132 		amdgpu_device_cache_pci_state(pdev);
2133 		/* Shut down the device */
2134 		pci_disable_device(pdev);
2135 		pci_set_power_state(pdev, PCI_D3cold);
2136 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
2137 	}
2138 }
2139 
2140 /**
2141  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
2142  *
2143  * @pdev: pci dev pointer
2144  *
2145  * Callback for the switcheroo driver.  Check of the switcheroo
2146  * state can be changed.
2147  * Returns true if the state can be changed, false if not.
2148  */
2149 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
2150 {
2151 	struct drm_device *dev = pci_get_drvdata(pdev);
2152 
2153        /*
2154 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
2155 	* locking inversion with the driver load path. And the access here is
2156 	* completely racy anyway. So don't bother with locking for now.
2157 	*/
2158 	return atomic_read(&dev->open_count) == 0;
2159 }
2160 
2161 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
2162 	.set_gpu_state = amdgpu_switcheroo_set_state,
2163 	.reprobe = NULL,
2164 	.can_switch = amdgpu_switcheroo_can_switch,
2165 };
2166 
2167 /**
2168  * amdgpu_device_ip_set_clockgating_state - set the CG state
2169  *
2170  * @dev: amdgpu_device pointer
2171  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2172  * @state: clockgating state (gate or ungate)
2173  *
2174  * Sets the requested clockgating state for all instances of
2175  * the hardware IP specified.
2176  * Returns the error code from the last instance.
2177  */
2178 int amdgpu_device_ip_set_clockgating_state(void *dev,
2179 					   enum amd_ip_block_type block_type,
2180 					   enum amd_clockgating_state state)
2181 {
2182 	struct amdgpu_device *adev = dev;
2183 	int i, r = 0;
2184 
2185 	for (i = 0; i < adev->num_ip_blocks; i++) {
2186 		if (!adev->ip_blocks[i].status.valid)
2187 			continue;
2188 		if (adev->ip_blocks[i].version->type != block_type)
2189 			continue;
2190 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
2191 			continue;
2192 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
2193 			&adev->ip_blocks[i], state);
2194 		if (r)
2195 			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
2196 				  adev->ip_blocks[i].version->funcs->name, r);
2197 	}
2198 	return r;
2199 }
2200 
2201 /**
2202  * amdgpu_device_ip_set_powergating_state - set the PG state
2203  *
2204  * @dev: amdgpu_device pointer
2205  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2206  * @state: powergating state (gate or ungate)
2207  *
2208  * Sets the requested powergating state for all instances of
2209  * the hardware IP specified.
2210  * Returns the error code from the last instance.
2211  */
2212 int amdgpu_device_ip_set_powergating_state(void *dev,
2213 					   enum amd_ip_block_type block_type,
2214 					   enum amd_powergating_state state)
2215 {
2216 	struct amdgpu_device *adev = dev;
2217 	int i, r = 0;
2218 
2219 	for (i = 0; i < adev->num_ip_blocks; i++) {
2220 		if (!adev->ip_blocks[i].status.valid)
2221 			continue;
2222 		if (adev->ip_blocks[i].version->type != block_type)
2223 			continue;
2224 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2225 			continue;
2226 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2227 			&adev->ip_blocks[i], state);
2228 		if (r)
2229 			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2230 				  adev->ip_blocks[i].version->funcs->name, r);
2231 	}
2232 	return r;
2233 }
2234 
2235 /**
2236  * amdgpu_device_ip_get_clockgating_state - get the CG state
2237  *
2238  * @adev: amdgpu_device pointer
2239  * @flags: clockgating feature flags
2240  *
2241  * Walks the list of IPs on the device and updates the clockgating
2242  * flags for each IP.
2243  * Updates @flags with the feature flags for each hardware IP where
2244  * clockgating is enabled.
2245  */
2246 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2247 					    u64 *flags)
2248 {
2249 	int i;
2250 
2251 	for (i = 0; i < adev->num_ip_blocks; i++) {
2252 		if (!adev->ip_blocks[i].status.valid)
2253 			continue;
2254 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2255 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2256 	}
2257 }
2258 
2259 /**
2260  * amdgpu_device_ip_wait_for_idle - wait for idle
2261  *
2262  * @adev: amdgpu_device pointer
2263  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2264  *
2265  * Waits for the request hardware IP to be idle.
2266  * Returns 0 for success or a negative error code on failure.
2267  */
2268 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2269 				   enum amd_ip_block_type block_type)
2270 {
2271 	int i, r;
2272 
2273 	for (i = 0; i < adev->num_ip_blocks; i++) {
2274 		if (!adev->ip_blocks[i].status.valid)
2275 			continue;
2276 		if (adev->ip_blocks[i].version->type == block_type) {
2277 			if (adev->ip_blocks[i].version->funcs->wait_for_idle) {
2278 				r = adev->ip_blocks[i].version->funcs->wait_for_idle(
2279 								&adev->ip_blocks[i]);
2280 				if (r)
2281 					return r;
2282 			}
2283 			break;
2284 		}
2285 	}
2286 	return 0;
2287 
2288 }
2289 
2290 /**
2291  * amdgpu_device_ip_is_valid - is the hardware IP enabled
2292  *
2293  * @adev: amdgpu_device pointer
2294  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2295  *
2296  * Check if the hardware IP is enable or not.
2297  * Returns true if it the IP is enable, false if not.
2298  */
2299 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
2300 			       enum amd_ip_block_type block_type)
2301 {
2302 	int i;
2303 
2304 	for (i = 0; i < adev->num_ip_blocks; i++) {
2305 		if (adev->ip_blocks[i].version->type == block_type)
2306 			return adev->ip_blocks[i].status.valid;
2307 	}
2308 	return false;
2309 
2310 }
2311 
2312 /**
2313  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2314  *
2315  * @adev: amdgpu_device pointer
2316  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2317  *
2318  * Returns a pointer to the hardware IP block structure
2319  * if it exists for the asic, otherwise NULL.
2320  */
2321 struct amdgpu_ip_block *
2322 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2323 			      enum amd_ip_block_type type)
2324 {
2325 	int i;
2326 
2327 	for (i = 0; i < adev->num_ip_blocks; i++)
2328 		if (adev->ip_blocks[i].version->type == type)
2329 			return &adev->ip_blocks[i];
2330 
2331 	return NULL;
2332 }
2333 
2334 /**
2335  * amdgpu_device_ip_block_version_cmp
2336  *
2337  * @adev: amdgpu_device pointer
2338  * @type: enum amd_ip_block_type
2339  * @major: major version
2340  * @minor: minor version
2341  *
2342  * return 0 if equal or greater
2343  * return 1 if smaller or the ip_block doesn't exist
2344  */
2345 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2346 				       enum amd_ip_block_type type,
2347 				       u32 major, u32 minor)
2348 {
2349 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2350 
2351 	if (ip_block && ((ip_block->version->major > major) ||
2352 			((ip_block->version->major == major) &&
2353 			(ip_block->version->minor >= minor))))
2354 		return 0;
2355 
2356 	return 1;
2357 }
2358 
2359 /**
2360  * amdgpu_device_ip_block_add
2361  *
2362  * @adev: amdgpu_device pointer
2363  * @ip_block_version: pointer to the IP to add
2364  *
2365  * Adds the IP block driver information to the collection of IPs
2366  * on the asic.
2367  */
2368 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2369 			       const struct amdgpu_ip_block_version *ip_block_version)
2370 {
2371 	if (!ip_block_version)
2372 		return -EINVAL;
2373 
2374 	switch (ip_block_version->type) {
2375 	case AMD_IP_BLOCK_TYPE_VCN:
2376 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2377 			return 0;
2378 		break;
2379 	case AMD_IP_BLOCK_TYPE_JPEG:
2380 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2381 			return 0;
2382 		break;
2383 	default:
2384 		break;
2385 	}
2386 
2387 	dev_info(adev->dev, "detected ip block number %d <%s>\n",
2388 		 adev->num_ip_blocks, ip_block_version->funcs->name);
2389 
2390 	adev->ip_blocks[adev->num_ip_blocks].adev = adev;
2391 
2392 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2393 
2394 	return 0;
2395 }
2396 
2397 /**
2398  * amdgpu_device_enable_virtual_display - enable virtual display feature
2399  *
2400  * @adev: amdgpu_device pointer
2401  *
2402  * Enabled the virtual display feature if the user has enabled it via
2403  * the module parameter virtual_display.  This feature provides a virtual
2404  * display hardware on headless boards or in virtualized environments.
2405  * This function parses and validates the configuration string specified by
2406  * the user and configures the virtual display configuration (number of
2407  * virtual connectors, crtcs, etc.) specified.
2408  */
2409 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2410 {
2411 	adev->enable_virtual_display = false;
2412 
2413 	if (amdgpu_virtual_display) {
2414 		const char *pci_address_name = pci_name(adev->pdev);
2415 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2416 
2417 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2418 		pciaddstr_tmp = pciaddstr;
2419 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2420 			pciaddname = strsep(&pciaddname_tmp, ",");
2421 			if (!strcmp("all", pciaddname)
2422 			    || !strcmp(pci_address_name, pciaddname)) {
2423 				long num_crtc;
2424 				int res = -1;
2425 
2426 				adev->enable_virtual_display = true;
2427 
2428 				if (pciaddname_tmp)
2429 					res = kstrtol(pciaddname_tmp, 10,
2430 						      &num_crtc);
2431 
2432 				if (!res) {
2433 					if (num_crtc < 1)
2434 						num_crtc = 1;
2435 					if (num_crtc > 6)
2436 						num_crtc = 6;
2437 					adev->mode_info.num_crtc = num_crtc;
2438 				} else {
2439 					adev->mode_info.num_crtc = 1;
2440 				}
2441 				break;
2442 			}
2443 		}
2444 
2445 		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2446 			 amdgpu_virtual_display, pci_address_name,
2447 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2448 
2449 		kfree(pciaddstr);
2450 	}
2451 }
2452 
2453 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2454 {
2455 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2456 		adev->mode_info.num_crtc = 1;
2457 		adev->enable_virtual_display = true;
2458 		DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2459 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2460 	}
2461 }
2462 
2463 /**
2464  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2465  *
2466  * @adev: amdgpu_device pointer
2467  *
2468  * Parses the asic configuration parameters specified in the gpu info
2469  * firmware and makes them available to the driver for use in configuring
2470  * the asic.
2471  * Returns 0 on success, -EINVAL on failure.
2472  */
2473 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2474 {
2475 	const char *chip_name;
2476 	int err;
2477 	const struct gpu_info_firmware_header_v1_0 *hdr;
2478 
2479 	adev->firmware.gpu_info_fw = NULL;
2480 
2481 	if (adev->mman.discovery_bin)
2482 		return 0;
2483 
2484 	switch (adev->asic_type) {
2485 	default:
2486 		return 0;
2487 	case CHIP_VEGA10:
2488 		chip_name = "vega10";
2489 		break;
2490 	case CHIP_VEGA12:
2491 		chip_name = "vega12";
2492 		break;
2493 	case CHIP_RAVEN:
2494 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2495 			chip_name = "raven2";
2496 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2497 			chip_name = "picasso";
2498 		else
2499 			chip_name = "raven";
2500 		break;
2501 	case CHIP_ARCTURUS:
2502 		chip_name = "arcturus";
2503 		break;
2504 	case CHIP_NAVI12:
2505 		chip_name = "navi12";
2506 		break;
2507 	}
2508 
2509 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,
2510 				   AMDGPU_UCODE_OPTIONAL,
2511 				   "amdgpu/%s_gpu_info.bin", chip_name);
2512 	if (err) {
2513 		dev_err(adev->dev,
2514 			"Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n",
2515 			chip_name);
2516 		goto out;
2517 	}
2518 
2519 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2520 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2521 
2522 	switch (hdr->version_major) {
2523 	case 1:
2524 	{
2525 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2526 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2527 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2528 
2529 		/*
2530 		 * Should be dropped when DAL no longer needs it.
2531 		 */
2532 		if (adev->asic_type == CHIP_NAVI12)
2533 			goto parse_soc_bounding_box;
2534 
2535 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2536 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2537 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2538 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2539 		adev->gfx.config.max_texture_channel_caches =
2540 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
2541 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2542 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2543 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2544 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2545 		adev->gfx.config.double_offchip_lds_buf =
2546 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2547 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2548 		adev->gfx.cu_info.max_waves_per_simd =
2549 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2550 		adev->gfx.cu_info.max_scratch_slots_per_cu =
2551 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2552 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2553 		if (hdr->version_minor >= 1) {
2554 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2555 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2556 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2557 			adev->gfx.config.num_sc_per_sh =
2558 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2559 			adev->gfx.config.num_packer_per_sc =
2560 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2561 		}
2562 
2563 parse_soc_bounding_box:
2564 		/*
2565 		 * soc bounding box info is not integrated in disocovery table,
2566 		 * we always need to parse it from gpu info firmware if needed.
2567 		 */
2568 		if (hdr->version_minor == 2) {
2569 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2570 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2571 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2572 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2573 		}
2574 		break;
2575 	}
2576 	default:
2577 		dev_err(adev->dev,
2578 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2579 		err = -EINVAL;
2580 		goto out;
2581 	}
2582 out:
2583 	return err;
2584 }
2585 
2586 /**
2587  * amdgpu_device_ip_early_init - run early init for hardware IPs
2588  *
2589  * @adev: amdgpu_device pointer
2590  *
2591  * Early initialization pass for hardware IPs.  The hardware IPs that make
2592  * up each asic are discovered each IP's early_init callback is run.  This
2593  * is the first stage in initializing the asic.
2594  * Returns 0 on success, negative error code on failure.
2595  */
2596 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2597 {
2598 	struct amdgpu_ip_block *ip_block;
2599 	struct pci_dev *parent;
2600 	bool total, skip_bios;
2601 	uint32_t bios_flags;
2602 	int i, r;
2603 
2604 	amdgpu_device_enable_virtual_display(adev);
2605 
2606 	if (amdgpu_sriov_vf(adev)) {
2607 		r = amdgpu_virt_request_full_gpu(adev, true);
2608 		if (r)
2609 			return r;
2610 	}
2611 
2612 	switch (adev->asic_type) {
2613 #ifdef CONFIG_DRM_AMDGPU_SI
2614 	case CHIP_VERDE:
2615 	case CHIP_TAHITI:
2616 	case CHIP_PITCAIRN:
2617 	case CHIP_OLAND:
2618 	case CHIP_HAINAN:
2619 		adev->family = AMDGPU_FAMILY_SI;
2620 		r = si_set_ip_blocks(adev);
2621 		if (r)
2622 			return r;
2623 		break;
2624 #endif
2625 #ifdef CONFIG_DRM_AMDGPU_CIK
2626 	case CHIP_BONAIRE:
2627 	case CHIP_HAWAII:
2628 	case CHIP_KAVERI:
2629 	case CHIP_KABINI:
2630 	case CHIP_MULLINS:
2631 		if (adev->flags & AMD_IS_APU)
2632 			adev->family = AMDGPU_FAMILY_KV;
2633 		else
2634 			adev->family = AMDGPU_FAMILY_CI;
2635 
2636 		r = cik_set_ip_blocks(adev);
2637 		if (r)
2638 			return r;
2639 		break;
2640 #endif
2641 	case CHIP_TOPAZ:
2642 	case CHIP_TONGA:
2643 	case CHIP_FIJI:
2644 	case CHIP_POLARIS10:
2645 	case CHIP_POLARIS11:
2646 	case CHIP_POLARIS12:
2647 	case CHIP_VEGAM:
2648 	case CHIP_CARRIZO:
2649 	case CHIP_STONEY:
2650 		if (adev->flags & AMD_IS_APU)
2651 			adev->family = AMDGPU_FAMILY_CZ;
2652 		else
2653 			adev->family = AMDGPU_FAMILY_VI;
2654 
2655 		r = vi_set_ip_blocks(adev);
2656 		if (r)
2657 			return r;
2658 		break;
2659 	default:
2660 		r = amdgpu_discovery_set_ip_blocks(adev);
2661 		if (r)
2662 			return r;
2663 		break;
2664 	}
2665 
2666 	if (amdgpu_has_atpx() &&
2667 	    (amdgpu_is_atpx_hybrid() ||
2668 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2669 	    ((adev->flags & AMD_IS_APU) == 0) &&
2670 	    !dev_is_removable(&adev->pdev->dev))
2671 		adev->flags |= AMD_IS_PX;
2672 
2673 	if (!(adev->flags & AMD_IS_APU)) {
2674 		parent = pcie_find_root_port(adev->pdev);
2675 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2676 	}
2677 
2678 
2679 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2680 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2681 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2682 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2683 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2684 	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2685 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2686 
2687 	total = true;
2688 	for (i = 0; i < adev->num_ip_blocks; i++) {
2689 		ip_block = &adev->ip_blocks[i];
2690 
2691 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2692 			DRM_WARN("disabled ip block: %d <%s>\n",
2693 				  i, adev->ip_blocks[i].version->funcs->name);
2694 			adev->ip_blocks[i].status.valid = false;
2695 		} else if (ip_block->version->funcs->early_init) {
2696 			r = ip_block->version->funcs->early_init(ip_block);
2697 			if (r == -ENOENT) {
2698 				adev->ip_blocks[i].status.valid = false;
2699 			} else if (r) {
2700 				DRM_ERROR("early_init of IP block <%s> failed %d\n",
2701 					  adev->ip_blocks[i].version->funcs->name, r);
2702 				total = false;
2703 			} else {
2704 				adev->ip_blocks[i].status.valid = true;
2705 			}
2706 		} else {
2707 			adev->ip_blocks[i].status.valid = true;
2708 		}
2709 		/* get the vbios after the asic_funcs are set up */
2710 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2711 			r = amdgpu_device_parse_gpu_info_fw(adev);
2712 			if (r)
2713 				return r;
2714 
2715 			bios_flags = amdgpu_device_get_vbios_flags(adev);
2716 			skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP);
2717 			/* Read BIOS */
2718 			if (!skip_bios) {
2719 				bool optional =
2720 					!!(bios_flags & AMDGPU_VBIOS_OPTIONAL);
2721 				if (!amdgpu_get_bios(adev) && !optional)
2722 					return -EINVAL;
2723 
2724 				if (optional && !adev->bios)
2725 					dev_info(
2726 						adev->dev,
2727 						"VBIOS image optional, proceeding without VBIOS image");
2728 
2729 				if (adev->bios) {
2730 					r = amdgpu_atombios_init(adev);
2731 					if (r) {
2732 						dev_err(adev->dev,
2733 							"amdgpu_atombios_init failed\n");
2734 						amdgpu_vf_error_put(
2735 							adev,
2736 							AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL,
2737 							0, 0);
2738 						return r;
2739 					}
2740 				}
2741 			}
2742 
2743 			/*get pf2vf msg info at it's earliest time*/
2744 			if (amdgpu_sriov_vf(adev))
2745 				amdgpu_virt_init_data_exchange(adev);
2746 
2747 		}
2748 	}
2749 	if (!total)
2750 		return -ENODEV;
2751 
2752 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
2753 	if (ip_block->status.valid != false)
2754 		amdgpu_amdkfd_device_probe(adev);
2755 
2756 	adev->cg_flags &= amdgpu_cg_mask;
2757 	adev->pg_flags &= amdgpu_pg_mask;
2758 
2759 	return 0;
2760 }
2761 
2762 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2763 {
2764 	int i, r;
2765 
2766 	for (i = 0; i < adev->num_ip_blocks; i++) {
2767 		if (!adev->ip_blocks[i].status.sw)
2768 			continue;
2769 		if (adev->ip_blocks[i].status.hw)
2770 			continue;
2771 		if (!amdgpu_ip_member_of_hwini(
2772 			    adev, adev->ip_blocks[i].version->type))
2773 			continue;
2774 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2775 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2776 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2777 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2778 			if (r) {
2779 				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2780 					  adev->ip_blocks[i].version->funcs->name, r);
2781 				return r;
2782 			}
2783 			adev->ip_blocks[i].status.hw = true;
2784 		}
2785 	}
2786 
2787 	return 0;
2788 }
2789 
2790 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2791 {
2792 	int i, r;
2793 
2794 	for (i = 0; i < adev->num_ip_blocks; i++) {
2795 		if (!adev->ip_blocks[i].status.sw)
2796 			continue;
2797 		if (adev->ip_blocks[i].status.hw)
2798 			continue;
2799 		if (!amdgpu_ip_member_of_hwini(
2800 			    adev, adev->ip_blocks[i].version->type))
2801 			continue;
2802 		r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2803 		if (r) {
2804 			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2805 				  adev->ip_blocks[i].version->funcs->name, r);
2806 			return r;
2807 		}
2808 		adev->ip_blocks[i].status.hw = true;
2809 	}
2810 
2811 	return 0;
2812 }
2813 
2814 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2815 {
2816 	int r = 0;
2817 	int i;
2818 	uint32_t smu_version;
2819 
2820 	if (adev->asic_type >= CHIP_VEGA10) {
2821 		for (i = 0; i < adev->num_ip_blocks; i++) {
2822 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2823 				continue;
2824 
2825 			if (!amdgpu_ip_member_of_hwini(adev,
2826 						       AMD_IP_BLOCK_TYPE_PSP))
2827 				break;
2828 
2829 			if (!adev->ip_blocks[i].status.sw)
2830 				continue;
2831 
2832 			/* no need to do the fw loading again if already done*/
2833 			if (adev->ip_blocks[i].status.hw == true)
2834 				break;
2835 
2836 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2837 				r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
2838 				if (r)
2839 					return r;
2840 			} else {
2841 				r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2842 				if (r) {
2843 					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2844 							  adev->ip_blocks[i].version->funcs->name, r);
2845 					return r;
2846 				}
2847 				adev->ip_blocks[i].status.hw = true;
2848 			}
2849 			break;
2850 		}
2851 	}
2852 
2853 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2854 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2855 
2856 	return r;
2857 }
2858 
2859 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2860 {
2861 	long timeout;
2862 	int r, i;
2863 
2864 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2865 		struct amdgpu_ring *ring = adev->rings[i];
2866 
2867 		/* No need to setup the GPU scheduler for rings that don't need it */
2868 		if (!ring || ring->no_scheduler)
2869 			continue;
2870 
2871 		switch (ring->funcs->type) {
2872 		case AMDGPU_RING_TYPE_GFX:
2873 			timeout = adev->gfx_timeout;
2874 			break;
2875 		case AMDGPU_RING_TYPE_COMPUTE:
2876 			timeout = adev->compute_timeout;
2877 			break;
2878 		case AMDGPU_RING_TYPE_SDMA:
2879 			timeout = adev->sdma_timeout;
2880 			break;
2881 		default:
2882 			timeout = adev->video_timeout;
2883 			break;
2884 		}
2885 
2886 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2887 				   DRM_SCHED_PRIORITY_COUNT,
2888 				   ring->num_hw_submission, 0,
2889 				   timeout, adev->reset_domain->wq,
2890 				   ring->sched_score, ring->name,
2891 				   adev->dev);
2892 		if (r) {
2893 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
2894 				  ring->name);
2895 			return r;
2896 		}
2897 		r = amdgpu_uvd_entity_init(adev, ring);
2898 		if (r) {
2899 			DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2900 				  ring->name);
2901 			return r;
2902 		}
2903 		r = amdgpu_vce_entity_init(adev, ring);
2904 		if (r) {
2905 			DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2906 				  ring->name);
2907 			return r;
2908 		}
2909 	}
2910 
2911 	amdgpu_xcp_update_partition_sched_list(adev);
2912 
2913 	return 0;
2914 }
2915 
2916 
2917 /**
2918  * amdgpu_device_ip_init - run init for hardware IPs
2919  *
2920  * @adev: amdgpu_device pointer
2921  *
2922  * Main initialization pass for hardware IPs.  The list of all the hardware
2923  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2924  * are run.  sw_init initializes the software state associated with each IP
2925  * and hw_init initializes the hardware associated with each IP.
2926  * Returns 0 on success, negative error code on failure.
2927  */
2928 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2929 {
2930 	bool init_badpage;
2931 	int i, r;
2932 
2933 	r = amdgpu_ras_init(adev);
2934 	if (r)
2935 		return r;
2936 
2937 	for (i = 0; i < adev->num_ip_blocks; i++) {
2938 		if (!adev->ip_blocks[i].status.valid)
2939 			continue;
2940 		if (adev->ip_blocks[i].version->funcs->sw_init) {
2941 			r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);
2942 			if (r) {
2943 				DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2944 					  adev->ip_blocks[i].version->funcs->name, r);
2945 				goto init_failed;
2946 			}
2947 		}
2948 		adev->ip_blocks[i].status.sw = true;
2949 
2950 		if (!amdgpu_ip_member_of_hwini(
2951 			    adev, adev->ip_blocks[i].version->type))
2952 			continue;
2953 
2954 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2955 			/* need to do common hw init early so everything is set up for gmc */
2956 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2957 			if (r) {
2958 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2959 				goto init_failed;
2960 			}
2961 			adev->ip_blocks[i].status.hw = true;
2962 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2963 			/* need to do gmc hw init early so we can allocate gpu mem */
2964 			/* Try to reserve bad pages early */
2965 			if (amdgpu_sriov_vf(adev))
2966 				amdgpu_virt_exchange_data(adev);
2967 
2968 			r = amdgpu_device_mem_scratch_init(adev);
2969 			if (r) {
2970 				DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2971 				goto init_failed;
2972 			}
2973 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2974 			if (r) {
2975 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2976 				goto init_failed;
2977 			}
2978 			r = amdgpu_device_wb_init(adev);
2979 			if (r) {
2980 				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2981 				goto init_failed;
2982 			}
2983 			adev->ip_blocks[i].status.hw = true;
2984 
2985 			/* right after GMC hw init, we create CSA */
2986 			if (adev->gfx.mcbp) {
2987 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2988 							       AMDGPU_GEM_DOMAIN_VRAM |
2989 							       AMDGPU_GEM_DOMAIN_GTT,
2990 							       AMDGPU_CSA_SIZE);
2991 				if (r) {
2992 					DRM_ERROR("allocate CSA failed %d\n", r);
2993 					goto init_failed;
2994 				}
2995 			}
2996 
2997 			r = amdgpu_seq64_init(adev);
2998 			if (r) {
2999 				DRM_ERROR("allocate seq64 failed %d\n", r);
3000 				goto init_failed;
3001 			}
3002 		}
3003 	}
3004 
3005 	if (amdgpu_sriov_vf(adev))
3006 		amdgpu_virt_init_data_exchange(adev);
3007 
3008 	r = amdgpu_ib_pool_init(adev);
3009 	if (r) {
3010 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
3011 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
3012 		goto init_failed;
3013 	}
3014 
3015 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
3016 	if (r)
3017 		goto init_failed;
3018 
3019 	r = amdgpu_device_ip_hw_init_phase1(adev);
3020 	if (r)
3021 		goto init_failed;
3022 
3023 	r = amdgpu_device_fw_loading(adev);
3024 	if (r)
3025 		goto init_failed;
3026 
3027 	r = amdgpu_device_ip_hw_init_phase2(adev);
3028 	if (r)
3029 		goto init_failed;
3030 
3031 	/*
3032 	 * retired pages will be loaded from eeprom and reserved here,
3033 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
3034 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
3035 	 * for I2C communication which only true at this point.
3036 	 *
3037 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
3038 	 * failure from bad gpu situation and stop amdgpu init process
3039 	 * accordingly. For other failed cases, it will still release all
3040 	 * the resource and print error message, rather than returning one
3041 	 * negative value to upper level.
3042 	 *
3043 	 * Note: theoretically, this should be called before all vram allocations
3044 	 * to protect retired page from abusing
3045 	 */
3046 	init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
3047 	r = amdgpu_ras_recovery_init(adev, init_badpage);
3048 	if (r)
3049 		goto init_failed;
3050 
3051 	/**
3052 	 * In case of XGMI grab extra reference for reset domain for this device
3053 	 */
3054 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3055 		if (amdgpu_xgmi_add_device(adev) == 0) {
3056 			if (!amdgpu_sriov_vf(adev)) {
3057 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3058 
3059 				if (WARN_ON(!hive)) {
3060 					r = -ENOENT;
3061 					goto init_failed;
3062 				}
3063 
3064 				if (!hive->reset_domain ||
3065 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
3066 					r = -ENOENT;
3067 					amdgpu_put_xgmi_hive(hive);
3068 					goto init_failed;
3069 				}
3070 
3071 				/* Drop the early temporary reset domain we created for device */
3072 				amdgpu_reset_put_reset_domain(adev->reset_domain);
3073 				adev->reset_domain = hive->reset_domain;
3074 				amdgpu_put_xgmi_hive(hive);
3075 			}
3076 		}
3077 	}
3078 
3079 	r = amdgpu_device_init_schedulers(adev);
3080 	if (r)
3081 		goto init_failed;
3082 
3083 	if (adev->mman.buffer_funcs_ring->sched.ready)
3084 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
3085 
3086 	/* Don't init kfd if whole hive need to be reset during init */
3087 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
3088 		kgd2kfd_init_zone_device(adev);
3089 		amdgpu_amdkfd_device_init(adev);
3090 	}
3091 
3092 	amdgpu_fru_get_product_info(adev);
3093 
3094 	if (amdgpu_aca_is_enabled(adev))
3095 		r = amdgpu_cper_init(adev);
3096 
3097 init_failed:
3098 
3099 	return r;
3100 }
3101 
3102 /**
3103  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
3104  *
3105  * @adev: amdgpu_device pointer
3106  *
3107  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
3108  * this function before a GPU reset.  If the value is retained after a
3109  * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents.
3110  */
3111 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
3112 {
3113 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
3114 }
3115 
3116 /**
3117  * amdgpu_device_check_vram_lost - check if vram is valid
3118  *
3119  * @adev: amdgpu_device pointer
3120  *
3121  * Checks the reset magic value written to the gart pointer in VRAM.
3122  * The driver calls this after a GPU reset to see if the contents of
3123  * VRAM is lost or now.
3124  * returns true if vram is lost, false if not.
3125  */
3126 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
3127 {
3128 	if (memcmp(adev->gart.ptr, adev->reset_magic,
3129 			AMDGPU_RESET_MAGIC_NUM))
3130 		return true;
3131 
3132 	if (!amdgpu_in_reset(adev))
3133 		return false;
3134 
3135 	/*
3136 	 * For all ASICs with baco/mode1 reset, the VRAM is
3137 	 * always assumed to be lost.
3138 	 */
3139 	switch (amdgpu_asic_reset_method(adev)) {
3140 	case AMD_RESET_METHOD_BACO:
3141 	case AMD_RESET_METHOD_MODE1:
3142 		return true;
3143 	default:
3144 		return false;
3145 	}
3146 }
3147 
3148 /**
3149  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
3150  *
3151  * @adev: amdgpu_device pointer
3152  * @state: clockgating state (gate or ungate)
3153  *
3154  * The list of all the hardware IPs that make up the asic is walked and the
3155  * set_clockgating_state callbacks are run.
3156  * Late initialization pass enabling clockgating for hardware IPs.
3157  * Fini or suspend, pass disabling clockgating for hardware IPs.
3158  * Returns 0 on success, negative error code on failure.
3159  */
3160 
3161 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
3162 			       enum amd_clockgating_state state)
3163 {
3164 	int i, j, r;
3165 
3166 	if (amdgpu_emu_mode == 1)
3167 		return 0;
3168 
3169 	for (j = 0; j < adev->num_ip_blocks; j++) {
3170 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3171 		if (!adev->ip_blocks[i].status.late_initialized)
3172 			continue;
3173 		/* skip CG for GFX, SDMA on S0ix */
3174 		if (adev->in_s0ix &&
3175 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3176 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3177 			continue;
3178 		/* skip CG for VCE/UVD, it's handled specially */
3179 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3180 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3181 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3182 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3183 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
3184 			/* enable clockgating to save power */
3185 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i],
3186 										     state);
3187 			if (r) {
3188 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
3189 					  adev->ip_blocks[i].version->funcs->name, r);
3190 				return r;
3191 			}
3192 		}
3193 	}
3194 
3195 	return 0;
3196 }
3197 
3198 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
3199 			       enum amd_powergating_state state)
3200 {
3201 	int i, j, r;
3202 
3203 	if (amdgpu_emu_mode == 1)
3204 		return 0;
3205 
3206 	for (j = 0; j < adev->num_ip_blocks; j++) {
3207 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
3208 		if (!adev->ip_blocks[i].status.late_initialized)
3209 			continue;
3210 		/* skip PG for GFX, SDMA on S0ix */
3211 		if (adev->in_s0ix &&
3212 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3213 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3214 			continue;
3215 		/* skip CG for VCE/UVD, it's handled specially */
3216 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
3217 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
3218 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
3219 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
3220 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
3221 			/* enable powergating to save power */
3222 			r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i],
3223 											state);
3224 			if (r) {
3225 				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
3226 					  adev->ip_blocks[i].version->funcs->name, r);
3227 				return r;
3228 			}
3229 		}
3230 	}
3231 	return 0;
3232 }
3233 
3234 static int amdgpu_device_enable_mgpu_fan_boost(void)
3235 {
3236 	struct amdgpu_gpu_instance *gpu_ins;
3237 	struct amdgpu_device *adev;
3238 	int i, ret = 0;
3239 
3240 	mutex_lock(&mgpu_info.mutex);
3241 
3242 	/*
3243 	 * MGPU fan boost feature should be enabled
3244 	 * only when there are two or more dGPUs in
3245 	 * the system
3246 	 */
3247 	if (mgpu_info.num_dgpu < 2)
3248 		goto out;
3249 
3250 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
3251 		gpu_ins = &(mgpu_info.gpu_ins[i]);
3252 		adev = gpu_ins->adev;
3253 		if (!(adev->flags & AMD_IS_APU) &&
3254 		    !gpu_ins->mgpu_fan_enabled) {
3255 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3256 			if (ret)
3257 				break;
3258 
3259 			gpu_ins->mgpu_fan_enabled = 1;
3260 		}
3261 	}
3262 
3263 out:
3264 	mutex_unlock(&mgpu_info.mutex);
3265 
3266 	return ret;
3267 }
3268 
3269 /**
3270  * amdgpu_device_ip_late_init - run late init for hardware IPs
3271  *
3272  * @adev: amdgpu_device pointer
3273  *
3274  * Late initialization pass for hardware IPs.  The list of all the hardware
3275  * IPs that make up the asic is walked and the late_init callbacks are run.
3276  * late_init covers any special initialization that an IP requires
3277  * after all of the have been initialized or something that needs to happen
3278  * late in the init process.
3279  * Returns 0 on success, negative error code on failure.
3280  */
3281 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3282 {
3283 	struct amdgpu_gpu_instance *gpu_instance;
3284 	int i = 0, r;
3285 
3286 	for (i = 0; i < adev->num_ip_blocks; i++) {
3287 		if (!adev->ip_blocks[i].status.hw)
3288 			continue;
3289 		if (adev->ip_blocks[i].version->funcs->late_init) {
3290 			r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
3291 			if (r) {
3292 				DRM_ERROR("late_init of IP block <%s> failed %d\n",
3293 					  adev->ip_blocks[i].version->funcs->name, r);
3294 				return r;
3295 			}
3296 		}
3297 		adev->ip_blocks[i].status.late_initialized = true;
3298 	}
3299 
3300 	r = amdgpu_ras_late_init(adev);
3301 	if (r) {
3302 		DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3303 		return r;
3304 	}
3305 
3306 	if (!amdgpu_reset_in_recovery(adev))
3307 		amdgpu_ras_set_error_query_ready(adev, true);
3308 
3309 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3310 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3311 
3312 	amdgpu_device_fill_reset_magic(adev);
3313 
3314 	r = amdgpu_device_enable_mgpu_fan_boost();
3315 	if (r)
3316 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3317 
3318 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3319 	if (amdgpu_passthrough(adev) &&
3320 	    ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3321 	     adev->asic_type == CHIP_ALDEBARAN))
3322 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
3323 
3324 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3325 		mutex_lock(&mgpu_info.mutex);
3326 
3327 		/*
3328 		 * Reset device p-state to low as this was booted with high.
3329 		 *
3330 		 * This should be performed only after all devices from the same
3331 		 * hive get initialized.
3332 		 *
3333 		 * However, it's unknown how many device in the hive in advance.
3334 		 * As this is counted one by one during devices initializations.
3335 		 *
3336 		 * So, we wait for all XGMI interlinked devices initialized.
3337 		 * This may bring some delays as those devices may come from
3338 		 * different hives. But that should be OK.
3339 		 */
3340 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3341 			for (i = 0; i < mgpu_info.num_gpu; i++) {
3342 				gpu_instance = &(mgpu_info.gpu_ins[i]);
3343 				if (gpu_instance->adev->flags & AMD_IS_APU)
3344 					continue;
3345 
3346 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3347 						AMDGPU_XGMI_PSTATE_MIN);
3348 				if (r) {
3349 					DRM_ERROR("pstate setting failed (%d).\n", r);
3350 					break;
3351 				}
3352 			}
3353 		}
3354 
3355 		mutex_unlock(&mgpu_info.mutex);
3356 	}
3357 
3358 	return 0;
3359 }
3360 
3361 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block)
3362 {
3363 	int r;
3364 
3365 	if (!ip_block->version->funcs->hw_fini) {
3366 		DRM_ERROR("hw_fini of IP block <%s> not defined\n",
3367 			  ip_block->version->funcs->name);
3368 	} else {
3369 		r = ip_block->version->funcs->hw_fini(ip_block);
3370 		/* XXX handle errors */
3371 		if (r) {
3372 			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3373 				  ip_block->version->funcs->name, r);
3374 		}
3375 	}
3376 
3377 	ip_block->status.hw = false;
3378 }
3379 
3380 /**
3381  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3382  *
3383  * @adev: amdgpu_device pointer
3384  *
3385  * For ASICs need to disable SMC first
3386  */
3387 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3388 {
3389 	int i;
3390 
3391 	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3392 		return;
3393 
3394 	for (i = 0; i < adev->num_ip_blocks; i++) {
3395 		if (!adev->ip_blocks[i].status.hw)
3396 			continue;
3397 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3398 			amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3399 			break;
3400 		}
3401 	}
3402 }
3403 
3404 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3405 {
3406 	int i, r;
3407 
3408 	for (i = 0; i < adev->num_ip_blocks; i++) {
3409 		if (!adev->ip_blocks[i].version->funcs->early_fini)
3410 			continue;
3411 
3412 		r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);
3413 		if (r) {
3414 			DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3415 				  adev->ip_blocks[i].version->funcs->name, r);
3416 		}
3417 	}
3418 
3419 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3420 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3421 
3422 	amdgpu_amdkfd_suspend(adev, false);
3423 
3424 	/* Workaround for ASICs need to disable SMC first */
3425 	amdgpu_device_smu_fini_early(adev);
3426 
3427 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3428 		if (!adev->ip_blocks[i].status.hw)
3429 			continue;
3430 
3431 		amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
3432 	}
3433 
3434 	if (amdgpu_sriov_vf(adev)) {
3435 		if (amdgpu_virt_release_full_gpu(adev, false))
3436 			DRM_ERROR("failed to release exclusive mode on fini\n");
3437 	}
3438 
3439 	return 0;
3440 }
3441 
3442 /**
3443  * amdgpu_device_ip_fini - run fini for hardware IPs
3444  *
3445  * @adev: amdgpu_device pointer
3446  *
3447  * Main teardown pass for hardware IPs.  The list of all the hardware
3448  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3449  * are run.  hw_fini tears down the hardware associated with each IP
3450  * and sw_fini tears down any software state associated with each IP.
3451  * Returns 0 on success, negative error code on failure.
3452  */
3453 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3454 {
3455 	int i, r;
3456 
3457 	if (amdgpu_aca_is_enabled(adev))
3458 		amdgpu_cper_fini(adev);
3459 
3460 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3461 		amdgpu_virt_release_ras_err_handler_data(adev);
3462 
3463 	if (adev->gmc.xgmi.num_physical_nodes > 1)
3464 		amdgpu_xgmi_remove_device(adev);
3465 
3466 	amdgpu_amdkfd_device_fini_sw(adev);
3467 
3468 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3469 		if (!adev->ip_blocks[i].status.sw)
3470 			continue;
3471 
3472 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3473 			amdgpu_ucode_free_bo(adev);
3474 			amdgpu_free_static_csa(&adev->virt.csa_obj);
3475 			amdgpu_device_wb_fini(adev);
3476 			amdgpu_device_mem_scratch_fini(adev);
3477 			amdgpu_ib_pool_fini(adev);
3478 			amdgpu_seq64_fini(adev);
3479 		}
3480 		if (adev->ip_blocks[i].version->funcs->sw_fini) {
3481 			r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);
3482 			/* XXX handle errors */
3483 			if (r) {
3484 				DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3485 					  adev->ip_blocks[i].version->funcs->name, r);
3486 			}
3487 		}
3488 		adev->ip_blocks[i].status.sw = false;
3489 		adev->ip_blocks[i].status.valid = false;
3490 	}
3491 
3492 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3493 		if (!adev->ip_blocks[i].status.late_initialized)
3494 			continue;
3495 		if (adev->ip_blocks[i].version->funcs->late_fini)
3496 			adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);
3497 		adev->ip_blocks[i].status.late_initialized = false;
3498 	}
3499 
3500 	amdgpu_ras_fini(adev);
3501 
3502 	return 0;
3503 }
3504 
3505 /**
3506  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3507  *
3508  * @work: work_struct.
3509  */
3510 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3511 {
3512 	struct amdgpu_device *adev =
3513 		container_of(work, struct amdgpu_device, delayed_init_work.work);
3514 	int r;
3515 
3516 	r = amdgpu_ib_ring_tests(adev);
3517 	if (r)
3518 		DRM_ERROR("ib ring test failed (%d).\n", r);
3519 }
3520 
3521 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3522 {
3523 	struct amdgpu_device *adev =
3524 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3525 
3526 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3527 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3528 
3529 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
3530 		adev->gfx.gfx_off_state = true;
3531 }
3532 
3533 /**
3534  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3535  *
3536  * @adev: amdgpu_device pointer
3537  *
3538  * Main suspend function for hardware IPs.  The list of all the hardware
3539  * IPs that make up the asic is walked, clockgating is disabled and the
3540  * suspend callbacks are run.  suspend puts the hardware and software state
3541  * in each IP into a state suitable for suspend.
3542  * Returns 0 on success, negative error code on failure.
3543  */
3544 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3545 {
3546 	int i, r;
3547 
3548 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3549 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3550 
3551 	/*
3552 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
3553 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3554 	 * scenario. Add the missing df cstate disablement here.
3555 	 */
3556 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3557 		dev_warn(adev->dev, "Failed to disallow df cstate");
3558 
3559 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3560 		if (!adev->ip_blocks[i].status.valid)
3561 			continue;
3562 
3563 		/* displays are handled separately */
3564 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3565 			continue;
3566 
3567 		/* XXX handle errors */
3568 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3569 		if (r)
3570 			return r;
3571 	}
3572 
3573 	return 0;
3574 }
3575 
3576 /**
3577  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3578  *
3579  * @adev: amdgpu_device pointer
3580  *
3581  * Main suspend function for hardware IPs.  The list of all the hardware
3582  * IPs that make up the asic is walked, clockgating is disabled and the
3583  * suspend callbacks are run.  suspend puts the hardware and software state
3584  * in each IP into a state suitable for suspend.
3585  * Returns 0 on success, negative error code on failure.
3586  */
3587 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3588 {
3589 	int i, r;
3590 
3591 	if (adev->in_s0ix)
3592 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3593 
3594 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3595 		if (!adev->ip_blocks[i].status.valid)
3596 			continue;
3597 		/* displays are handled in phase1 */
3598 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3599 			continue;
3600 		/* PSP lost connection when err_event_athub occurs */
3601 		if (amdgpu_ras_intr_triggered() &&
3602 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3603 			adev->ip_blocks[i].status.hw = false;
3604 			continue;
3605 		}
3606 
3607 		/* skip unnecessary suspend if we do not initialize them yet */
3608 		if (!amdgpu_ip_member_of_hwini(
3609 			    adev, adev->ip_blocks[i].version->type))
3610 			continue;
3611 
3612 		/* skip suspend of gfx/mes and psp for S0ix
3613 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3614 		 * like at runtime. PSP is also part of the always on hardware
3615 		 * so no need to suspend it.
3616 		 */
3617 		if (adev->in_s0ix &&
3618 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3619 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3620 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3621 			continue;
3622 
3623 		/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3624 		if (adev->in_s0ix &&
3625 		    (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3626 		     IP_VERSION(5, 0, 0)) &&
3627 		    (adev->ip_blocks[i].version->type ==
3628 		     AMD_IP_BLOCK_TYPE_SDMA))
3629 			continue;
3630 
3631 		/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3632 		 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3633 		 * from this location and RLC Autoload automatically also gets loaded
3634 		 * from here based on PMFW -> PSP message during re-init sequence.
3635 		 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3636 		 * the TMR and reload FWs again for IMU enabled APU ASICs.
3637 		 */
3638 		if (amdgpu_in_reset(adev) &&
3639 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3640 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3641 			continue;
3642 
3643 		/* XXX handle errors */
3644 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3645 		adev->ip_blocks[i].status.hw = false;
3646 
3647 		/* handle putting the SMC in the appropriate state */
3648 		if (!amdgpu_sriov_vf(adev)) {
3649 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3650 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3651 				if (r) {
3652 					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3653 							adev->mp1_state, r);
3654 					return r;
3655 				}
3656 			}
3657 		}
3658 	}
3659 
3660 	return 0;
3661 }
3662 
3663 /**
3664  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3665  *
3666  * @adev: amdgpu_device pointer
3667  *
3668  * Main suspend function for hardware IPs.  The list of all the hardware
3669  * IPs that make up the asic is walked, clockgating is disabled and the
3670  * suspend callbacks are run.  suspend puts the hardware and software state
3671  * in each IP into a state suitable for suspend.
3672  * Returns 0 on success, negative error code on failure.
3673  */
3674 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3675 {
3676 	int r;
3677 
3678 	if (amdgpu_sriov_vf(adev)) {
3679 		amdgpu_virt_fini_data_exchange(adev);
3680 		amdgpu_virt_request_full_gpu(adev, false);
3681 	}
3682 
3683 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
3684 
3685 	r = amdgpu_device_ip_suspend_phase1(adev);
3686 	if (r)
3687 		return r;
3688 	r = amdgpu_device_ip_suspend_phase2(adev);
3689 
3690 	if (amdgpu_sriov_vf(adev))
3691 		amdgpu_virt_release_full_gpu(adev, false);
3692 
3693 	return r;
3694 }
3695 
3696 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3697 {
3698 	int i, r;
3699 
3700 	static enum amd_ip_block_type ip_order[] = {
3701 		AMD_IP_BLOCK_TYPE_COMMON,
3702 		AMD_IP_BLOCK_TYPE_GMC,
3703 		AMD_IP_BLOCK_TYPE_PSP,
3704 		AMD_IP_BLOCK_TYPE_IH,
3705 	};
3706 
3707 	for (i = 0; i < adev->num_ip_blocks; i++) {
3708 		int j;
3709 		struct amdgpu_ip_block *block;
3710 
3711 		block = &adev->ip_blocks[i];
3712 		block->status.hw = false;
3713 
3714 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3715 
3716 			if (block->version->type != ip_order[j] ||
3717 				!block->status.valid)
3718 				continue;
3719 
3720 			r = block->version->funcs->hw_init(&adev->ip_blocks[i]);
3721 			if (r) {
3722 				dev_err(adev->dev, "RE-INIT-early: %s failed\n",
3723 					 block->version->funcs->name);
3724 				return r;
3725 			}
3726 			block->status.hw = true;
3727 		}
3728 	}
3729 
3730 	return 0;
3731 }
3732 
3733 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3734 {
3735 	struct amdgpu_ip_block *block;
3736 	int i, r = 0;
3737 
3738 	static enum amd_ip_block_type ip_order[] = {
3739 		AMD_IP_BLOCK_TYPE_SMC,
3740 		AMD_IP_BLOCK_TYPE_DCE,
3741 		AMD_IP_BLOCK_TYPE_GFX,
3742 		AMD_IP_BLOCK_TYPE_SDMA,
3743 		AMD_IP_BLOCK_TYPE_MES,
3744 		AMD_IP_BLOCK_TYPE_UVD,
3745 		AMD_IP_BLOCK_TYPE_VCE,
3746 		AMD_IP_BLOCK_TYPE_VCN,
3747 		AMD_IP_BLOCK_TYPE_JPEG
3748 	};
3749 
3750 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3751 		block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);
3752 
3753 		if (!block)
3754 			continue;
3755 
3756 		if (block->status.valid && !block->status.hw) {
3757 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) {
3758 				r = amdgpu_ip_block_resume(block);
3759 			} else {
3760 				r = block->version->funcs->hw_init(block);
3761 			}
3762 
3763 			if (r) {
3764 				dev_err(adev->dev, "RE-INIT-late: %s failed\n",
3765 					 block->version->funcs->name);
3766 				break;
3767 			}
3768 			block->status.hw = true;
3769 		}
3770 	}
3771 
3772 	return r;
3773 }
3774 
3775 /**
3776  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3777  *
3778  * @adev: amdgpu_device pointer
3779  *
3780  * First resume function for hardware IPs.  The list of all the hardware
3781  * IPs that make up the asic is walked and the resume callbacks are run for
3782  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3783  * after a suspend and updates the software state as necessary.  This
3784  * function is also used for restoring the GPU after a GPU reset.
3785  * Returns 0 on success, negative error code on failure.
3786  */
3787 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3788 {
3789 	int i, r;
3790 
3791 	for (i = 0; i < adev->num_ip_blocks; i++) {
3792 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3793 			continue;
3794 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3795 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3796 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3797 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3798 
3799 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3800 			if (r)
3801 				return r;
3802 		}
3803 	}
3804 
3805 	return 0;
3806 }
3807 
3808 /**
3809  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3810  *
3811  * @adev: amdgpu_device pointer
3812  *
3813  * Second resume function for hardware IPs.  The list of all the hardware
3814  * IPs that make up the asic is walked and the resume callbacks are run for
3815  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3816  * functional state after a suspend and updates the software state as
3817  * necessary.  This function is also used for restoring the GPU after a GPU
3818  * reset.
3819  * Returns 0 on success, negative error code on failure.
3820  */
3821 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3822 {
3823 	int i, r;
3824 
3825 	for (i = 0; i < adev->num_ip_blocks; i++) {
3826 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3827 			continue;
3828 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3829 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3830 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3831 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||
3832 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3833 			continue;
3834 		r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3835 		if (r)
3836 			return r;
3837 	}
3838 
3839 	return 0;
3840 }
3841 
3842 /**
3843  * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3844  *
3845  * @adev: amdgpu_device pointer
3846  *
3847  * Third resume function for hardware IPs.  The list of all the hardware
3848  * IPs that make up the asic is walked and the resume callbacks are run for
3849  * all DCE.  resume puts the hardware into a functional state after a suspend
3850  * and updates the software state as necessary.  This function is also used
3851  * for restoring the GPU after a GPU reset.
3852  *
3853  * Returns 0 on success, negative error code on failure.
3854  */
3855 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)
3856 {
3857 	int i, r;
3858 
3859 	for (i = 0; i < adev->num_ip_blocks; i++) {
3860 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3861 			continue;
3862 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
3863 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3864 			if (r)
3865 				return r;
3866 		}
3867 	}
3868 
3869 	return 0;
3870 }
3871 
3872 /**
3873  * amdgpu_device_ip_resume - run resume for hardware IPs
3874  *
3875  * @adev: amdgpu_device pointer
3876  *
3877  * Main resume function for hardware IPs.  The hardware IPs
3878  * are split into two resume functions because they are
3879  * also used in recovering from a GPU reset and some additional
3880  * steps need to be take between them.  In this case (S3/S4) they are
3881  * run sequentially.
3882  * Returns 0 on success, negative error code on failure.
3883  */
3884 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3885 {
3886 	int r;
3887 
3888 	r = amdgpu_device_ip_resume_phase1(adev);
3889 	if (r)
3890 		return r;
3891 
3892 	r = amdgpu_device_fw_loading(adev);
3893 	if (r)
3894 		return r;
3895 
3896 	r = amdgpu_device_ip_resume_phase2(adev);
3897 
3898 	if (adev->mman.buffer_funcs_ring->sched.ready)
3899 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
3900 
3901 	if (r)
3902 		return r;
3903 
3904 	amdgpu_fence_driver_hw_init(adev);
3905 
3906 	r = amdgpu_device_ip_resume_phase3(adev);
3907 
3908 	return r;
3909 }
3910 
3911 /**
3912  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3913  *
3914  * @adev: amdgpu_device pointer
3915  *
3916  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3917  */
3918 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3919 {
3920 	if (amdgpu_sriov_vf(adev)) {
3921 		if (adev->is_atom_fw) {
3922 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3923 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3924 		} else {
3925 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3926 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3927 		}
3928 
3929 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3930 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3931 	}
3932 }
3933 
3934 /**
3935  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3936  *
3937  * @asic_type: AMD asic type
3938  *
3939  * Check if there is DC (new modesetting infrastructre) support for an asic.
3940  * returns true if DC has support, false if not.
3941  */
3942 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3943 {
3944 	switch (asic_type) {
3945 #ifdef CONFIG_DRM_AMDGPU_SI
3946 	case CHIP_HAINAN:
3947 #endif
3948 	case CHIP_TOPAZ:
3949 		/* chips with no display hardware */
3950 		return false;
3951 #if defined(CONFIG_DRM_AMD_DC)
3952 	case CHIP_TAHITI:
3953 	case CHIP_PITCAIRN:
3954 	case CHIP_VERDE:
3955 	case CHIP_OLAND:
3956 		/*
3957 		 * We have systems in the wild with these ASICs that require
3958 		 * LVDS and VGA support which is not supported with DC.
3959 		 *
3960 		 * Fallback to the non-DC driver here by default so as not to
3961 		 * cause regressions.
3962 		 */
3963 #if defined(CONFIG_DRM_AMD_DC_SI)
3964 		return amdgpu_dc > 0;
3965 #else
3966 		return false;
3967 #endif
3968 	case CHIP_BONAIRE:
3969 	case CHIP_KAVERI:
3970 	case CHIP_KABINI:
3971 	case CHIP_MULLINS:
3972 		/*
3973 		 * We have systems in the wild with these ASICs that require
3974 		 * VGA support which is not supported with DC.
3975 		 *
3976 		 * Fallback to the non-DC driver here by default so as not to
3977 		 * cause regressions.
3978 		 */
3979 		return amdgpu_dc > 0;
3980 	default:
3981 		return amdgpu_dc != 0;
3982 #else
3983 	default:
3984 		if (amdgpu_dc > 0)
3985 			DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3986 		return false;
3987 #endif
3988 	}
3989 }
3990 
3991 /**
3992  * amdgpu_device_has_dc_support - check if dc is supported
3993  *
3994  * @adev: amdgpu_device pointer
3995  *
3996  * Returns true for supported, false for not supported
3997  */
3998 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3999 {
4000 	if (adev->enable_virtual_display ||
4001 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
4002 		return false;
4003 
4004 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
4005 }
4006 
4007 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
4008 {
4009 	struct amdgpu_device *adev =
4010 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
4011 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4012 
4013 	/* It's a bug to not have a hive within this function */
4014 	if (WARN_ON(!hive))
4015 		return;
4016 
4017 	/*
4018 	 * Use task barrier to synchronize all xgmi reset works across the
4019 	 * hive. task_barrier_enter and task_barrier_exit will block
4020 	 * until all the threads running the xgmi reset works reach
4021 	 * those points. task_barrier_full will do both blocks.
4022 	 */
4023 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
4024 
4025 		task_barrier_enter(&hive->tb);
4026 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
4027 
4028 		if (adev->asic_reset_res)
4029 			goto fail;
4030 
4031 		task_barrier_exit(&hive->tb);
4032 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
4033 
4034 		if (adev->asic_reset_res)
4035 			goto fail;
4036 
4037 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
4038 	} else {
4039 
4040 		task_barrier_full(&hive->tb);
4041 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
4042 	}
4043 
4044 fail:
4045 	if (adev->asic_reset_res)
4046 		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
4047 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
4048 	amdgpu_put_xgmi_hive(hive);
4049 }
4050 
4051 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
4052 {
4053 	char *input = amdgpu_lockup_timeout;
4054 	char *timeout_setting = NULL;
4055 	int index = 0;
4056 	long timeout;
4057 	int ret = 0;
4058 
4059 	/*
4060 	 * By default timeout for non compute jobs is 10000
4061 	 * and 60000 for compute jobs.
4062 	 * In SR-IOV or passthrough mode, timeout for compute
4063 	 * jobs are 60000 by default.
4064 	 */
4065 	adev->gfx_timeout = msecs_to_jiffies(10000);
4066 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4067 	if (amdgpu_sriov_vf(adev))
4068 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
4069 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
4070 	else
4071 		adev->compute_timeout =  msecs_to_jiffies(60000);
4072 
4073 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4074 		while ((timeout_setting = strsep(&input, ",")) &&
4075 				strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
4076 			ret = kstrtol(timeout_setting, 0, &timeout);
4077 			if (ret)
4078 				return ret;
4079 
4080 			if (timeout == 0) {
4081 				index++;
4082 				continue;
4083 			} else if (timeout < 0) {
4084 				timeout = MAX_SCHEDULE_TIMEOUT;
4085 				dev_warn(adev->dev, "lockup timeout disabled");
4086 				add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
4087 			} else {
4088 				timeout = msecs_to_jiffies(timeout);
4089 			}
4090 
4091 			switch (index++) {
4092 			case 0:
4093 				adev->gfx_timeout = timeout;
4094 				break;
4095 			case 1:
4096 				adev->compute_timeout = timeout;
4097 				break;
4098 			case 2:
4099 				adev->sdma_timeout = timeout;
4100 				break;
4101 			case 3:
4102 				adev->video_timeout = timeout;
4103 				break;
4104 			default:
4105 				break;
4106 			}
4107 		}
4108 		/*
4109 		 * There is only one value specified and
4110 		 * it should apply to all non-compute jobs.
4111 		 */
4112 		if (index == 1) {
4113 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
4114 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
4115 				adev->compute_timeout = adev->gfx_timeout;
4116 		}
4117 	}
4118 
4119 	return ret;
4120 }
4121 
4122 /**
4123  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
4124  *
4125  * @adev: amdgpu_device pointer
4126  *
4127  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
4128  */
4129 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
4130 {
4131 	struct iommu_domain *domain;
4132 
4133 	domain = iommu_get_domain_for_dev(adev->dev);
4134 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
4135 		adev->ram_is_direct_mapped = true;
4136 }
4137 
4138 #if defined(CONFIG_HSA_AMD_P2P)
4139 /**
4140  * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
4141  *
4142  * @adev: amdgpu_device pointer
4143  *
4144  * return if IOMMU remapping bar address
4145  */
4146 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)
4147 {
4148 	struct iommu_domain *domain;
4149 
4150 	domain = iommu_get_domain_for_dev(adev->dev);
4151 	if (domain && (domain->type == IOMMU_DOMAIN_DMA ||
4152 		domain->type ==	IOMMU_DOMAIN_DMA_FQ))
4153 		return true;
4154 
4155 	return false;
4156 }
4157 #endif
4158 
4159 static const struct attribute *amdgpu_dev_attributes[] = {
4160 	&dev_attr_pcie_replay_count.attr,
4161 	NULL
4162 };
4163 
4164 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
4165 {
4166 	if (amdgpu_mcbp == 1)
4167 		adev->gfx.mcbp = true;
4168 	else if (amdgpu_mcbp == 0)
4169 		adev->gfx.mcbp = false;
4170 
4171 	if (amdgpu_sriov_vf(adev))
4172 		adev->gfx.mcbp = true;
4173 
4174 	if (adev->gfx.mcbp)
4175 		DRM_INFO("MCBP is enabled\n");
4176 }
4177 
4178 /**
4179  * amdgpu_device_init - initialize the driver
4180  *
4181  * @adev: amdgpu_device pointer
4182  * @flags: driver flags
4183  *
4184  * Initializes the driver info and hw (all asics).
4185  * Returns 0 for success or an error on failure.
4186  * Called at driver startup.
4187  */
4188 int amdgpu_device_init(struct amdgpu_device *adev,
4189 		       uint32_t flags)
4190 {
4191 	struct drm_device *ddev = adev_to_drm(adev);
4192 	struct pci_dev *pdev = adev->pdev;
4193 	int r, i;
4194 	bool px = false;
4195 	u32 max_MBps;
4196 	int tmp;
4197 
4198 	adev->shutdown = false;
4199 	adev->flags = flags;
4200 
4201 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
4202 		adev->asic_type = amdgpu_force_asic_type;
4203 	else
4204 		adev->asic_type = flags & AMD_ASIC_MASK;
4205 
4206 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
4207 	if (amdgpu_emu_mode == 1)
4208 		adev->usec_timeout *= 10;
4209 	adev->gmc.gart_size = 512 * 1024 * 1024;
4210 	adev->accel_working = false;
4211 	adev->num_rings = 0;
4212 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
4213 	adev->mman.buffer_funcs = NULL;
4214 	adev->mman.buffer_funcs_ring = NULL;
4215 	adev->vm_manager.vm_pte_funcs = NULL;
4216 	adev->vm_manager.vm_pte_num_scheds = 0;
4217 	adev->gmc.gmc_funcs = NULL;
4218 	adev->harvest_ip_mask = 0x0;
4219 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
4220 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
4221 
4222 	adev->smc_rreg = &amdgpu_invalid_rreg;
4223 	adev->smc_wreg = &amdgpu_invalid_wreg;
4224 	adev->pcie_rreg = &amdgpu_invalid_rreg;
4225 	adev->pcie_wreg = &amdgpu_invalid_wreg;
4226 	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
4227 	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
4228 	adev->pciep_rreg = &amdgpu_invalid_rreg;
4229 	adev->pciep_wreg = &amdgpu_invalid_wreg;
4230 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
4231 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
4232 	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
4233 	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
4234 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
4235 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
4236 	adev->didt_rreg = &amdgpu_invalid_rreg;
4237 	adev->didt_wreg = &amdgpu_invalid_wreg;
4238 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
4239 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
4240 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
4241 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
4242 
4243 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
4244 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
4245 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
4246 
4247 	/* mutex initialization are all done here so we
4248 	 * can recall function without having locking issues
4249 	 */
4250 	mutex_init(&adev->firmware.mutex);
4251 	mutex_init(&adev->pm.mutex);
4252 	mutex_init(&adev->gfx.gpu_clock_mutex);
4253 	mutex_init(&adev->srbm_mutex);
4254 	mutex_init(&adev->gfx.pipe_reserve_mutex);
4255 	mutex_init(&adev->gfx.gfx_off_mutex);
4256 	mutex_init(&adev->gfx.partition_mutex);
4257 	mutex_init(&adev->grbm_idx_mutex);
4258 	mutex_init(&adev->mn_lock);
4259 	mutex_init(&adev->virt.vf_errors.lock);
4260 	mutex_init(&adev->virt.rlcg_reg_lock);
4261 	hash_init(adev->mn_hash);
4262 	mutex_init(&adev->psp.mutex);
4263 	mutex_init(&adev->notifier_lock);
4264 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
4265 	mutex_init(&adev->benchmark_mutex);
4266 	mutex_init(&adev->gfx.reset_sem_mutex);
4267 	/* Initialize the mutex for cleaner shader isolation between GFX and compute processes */
4268 	mutex_init(&adev->enforce_isolation_mutex);
4269 	mutex_init(&adev->gfx.kfd_sch_mutex);
4270 
4271 	amdgpu_device_init_apu_flags(adev);
4272 
4273 	r = amdgpu_device_check_arguments(adev);
4274 	if (r)
4275 		return r;
4276 
4277 	spin_lock_init(&adev->mmio_idx_lock);
4278 	spin_lock_init(&adev->smc_idx_lock);
4279 	spin_lock_init(&adev->pcie_idx_lock);
4280 	spin_lock_init(&adev->uvd_ctx_idx_lock);
4281 	spin_lock_init(&adev->didt_idx_lock);
4282 	spin_lock_init(&adev->gc_cac_idx_lock);
4283 	spin_lock_init(&adev->se_cac_idx_lock);
4284 	spin_lock_init(&adev->audio_endpt_idx_lock);
4285 	spin_lock_init(&adev->mm_stats.lock);
4286 	spin_lock_init(&adev->wb.lock);
4287 
4288 	INIT_LIST_HEAD(&adev->reset_list);
4289 
4290 	INIT_LIST_HEAD(&adev->ras_list);
4291 
4292 	INIT_LIST_HEAD(&adev->pm.od_kobj_list);
4293 
4294 	INIT_DELAYED_WORK(&adev->delayed_init_work,
4295 			  amdgpu_device_delayed_init_work_handler);
4296 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
4297 			  amdgpu_device_delay_enable_gfx_off);
4298 	/*
4299 	 * Initialize the enforce_isolation work structures for each XCP
4300 	 * partition.  This work handler is responsible for enforcing shader
4301 	 * isolation on AMD GPUs.  It counts the number of emitted fences for
4302 	 * each GFX and compute ring.  If there are any fences, it schedules
4303 	 * the `enforce_isolation_work` to be run after a delay.  If there are
4304 	 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the
4305 	 * runqueue.
4306 	 */
4307 	for (i = 0; i < MAX_XCP; i++) {
4308 		INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,
4309 				  amdgpu_gfx_enforce_isolation_handler);
4310 		adev->gfx.enforce_isolation[i].adev = adev;
4311 		adev->gfx.enforce_isolation[i].xcp_id = i;
4312 	}
4313 
4314 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4315 
4316 	adev->gfx.gfx_off_req_count = 1;
4317 	adev->gfx.gfx_off_residency = 0;
4318 	adev->gfx.gfx_off_entrycount = 0;
4319 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4320 
4321 	atomic_set(&adev->throttling_logging_enabled, 1);
4322 	/*
4323 	 * If throttling continues, logging will be performed every minute
4324 	 * to avoid log flooding. "-1" is subtracted since the thermal
4325 	 * throttling interrupt comes every second. Thus, the total logging
4326 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
4327 	 * for throttling interrupt) = 60 seconds.
4328 	 */
4329 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4330 	ratelimit_state_init(&adev->virt.ras_telemetry_rs, 5 * HZ, 1);
4331 
4332 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4333 	ratelimit_set_flags(&adev->virt.ras_telemetry_rs, RATELIMIT_MSG_ON_RELEASE);
4334 
4335 	/* Registers mapping */
4336 	/* TODO: block userspace mapping of io register */
4337 	if (adev->asic_type >= CHIP_BONAIRE) {
4338 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4339 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4340 	} else {
4341 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4342 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4343 	}
4344 
4345 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4346 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4347 
4348 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4349 	if (!adev->rmmio)
4350 		return -ENOMEM;
4351 
4352 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4353 	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4354 
4355 	/*
4356 	 * Reset domain needs to be present early, before XGMI hive discovered
4357 	 * (if any) and initialized to use reset sem and in_gpu reset flag
4358 	 * early on during init and before calling to RREG32.
4359 	 */
4360 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4361 	if (!adev->reset_domain)
4362 		return -ENOMEM;
4363 
4364 	/* detect hw virtualization here */
4365 	amdgpu_detect_virtualization(adev);
4366 
4367 	amdgpu_device_get_pcie_info(adev);
4368 
4369 	r = amdgpu_device_get_job_timeout_settings(adev);
4370 	if (r) {
4371 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4372 		return r;
4373 	}
4374 
4375 	amdgpu_device_set_mcbp(adev);
4376 
4377 	/*
4378 	 * By default, use default mode where all blocks are expected to be
4379 	 * initialized. At present a 'swinit' of blocks is required to be
4380 	 * completed before the need for a different level is detected.
4381 	 */
4382 	amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
4383 	/* early init functions */
4384 	r = amdgpu_device_ip_early_init(adev);
4385 	if (r)
4386 		return r;
4387 
4388 	/* Get rid of things like offb */
4389 	r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);
4390 	if (r)
4391 		return r;
4392 
4393 	/* Enable TMZ based on IP_VERSION */
4394 	amdgpu_gmc_tmz_set(adev);
4395 
4396 	if (amdgpu_sriov_vf(adev) &&
4397 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
4398 		/* VF MMIO access (except mailbox range) from CPU
4399 		 * will be blocked during sriov runtime
4400 		 */
4401 		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
4402 
4403 	amdgpu_gmc_noretry_set(adev);
4404 	/* Need to get xgmi info early to decide the reset behavior*/
4405 	if (adev->gmc.xgmi.supported) {
4406 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
4407 		if (r)
4408 			return r;
4409 	}
4410 
4411 	/* enable PCIE atomic ops */
4412 	if (amdgpu_sriov_vf(adev)) {
4413 		if (adev->virt.fw_reserve.p_pf2vf)
4414 			adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4415 						      adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4416 				(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4417 	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4418 	 * internal path natively support atomics, set have_atomics_support to true.
4419 	 */
4420 	} else if ((adev->flags & AMD_IS_APU) &&
4421 		   (amdgpu_ip_version(adev, GC_HWIP, 0) >
4422 		    IP_VERSION(9, 0, 0))) {
4423 		adev->have_atomics_support = true;
4424 	} else {
4425 		adev->have_atomics_support =
4426 			!pci_enable_atomic_ops_to_root(adev->pdev,
4427 					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4428 					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4429 	}
4430 
4431 	if (!adev->have_atomics_support)
4432 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4433 
4434 	/* doorbell bar mapping and doorbell index init*/
4435 	amdgpu_doorbell_init(adev);
4436 
4437 	if (amdgpu_emu_mode == 1) {
4438 		/* post the asic on emulation mode */
4439 		emu_soc_asic_init(adev);
4440 		goto fence_driver_init;
4441 	}
4442 
4443 	amdgpu_reset_init(adev);
4444 
4445 	/* detect if we are with an SRIOV vbios */
4446 	if (adev->bios)
4447 		amdgpu_device_detect_sriov_bios(adev);
4448 
4449 	/* check if we need to reset the asic
4450 	 *  E.g., driver was not cleanly unloaded previously, etc.
4451 	 */
4452 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4453 		if (adev->gmc.xgmi.num_physical_nodes) {
4454 			dev_info(adev->dev, "Pending hive reset.\n");
4455 			amdgpu_set_init_level(adev,
4456 					      AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
4457 		} else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) &&
4458 				   !amdgpu_device_has_display_hardware(adev)) {
4459 					r = psp_gpu_reset(adev);
4460 		} else {
4461 				tmp = amdgpu_reset_method;
4462 				/* It should do a default reset when loading or reloading the driver,
4463 				 * regardless of the module parameter reset_method.
4464 				 */
4465 				amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4466 				r = amdgpu_asic_reset(adev);
4467 				amdgpu_reset_method = tmp;
4468 		}
4469 
4470 		if (r) {
4471 		  dev_err(adev->dev, "asic reset on init failed\n");
4472 		  goto failed;
4473 		}
4474 	}
4475 
4476 	/* Post card if necessary */
4477 	if (amdgpu_device_need_post(adev)) {
4478 		if (!adev->bios) {
4479 			dev_err(adev->dev, "no vBIOS found\n");
4480 			r = -EINVAL;
4481 			goto failed;
4482 		}
4483 		DRM_INFO("GPU posting now...\n");
4484 		r = amdgpu_device_asic_init(adev);
4485 		if (r) {
4486 			dev_err(adev->dev, "gpu post error!\n");
4487 			goto failed;
4488 		}
4489 	}
4490 
4491 	if (adev->bios) {
4492 		if (adev->is_atom_fw) {
4493 			/* Initialize clocks */
4494 			r = amdgpu_atomfirmware_get_clock_info(adev);
4495 			if (r) {
4496 				dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4497 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4498 				goto failed;
4499 			}
4500 		} else {
4501 			/* Initialize clocks */
4502 			r = amdgpu_atombios_get_clock_info(adev);
4503 			if (r) {
4504 				dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4505 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4506 				goto failed;
4507 			}
4508 			/* init i2c buses */
4509 			amdgpu_i2c_init(adev);
4510 		}
4511 	}
4512 
4513 fence_driver_init:
4514 	/* Fence driver */
4515 	r = amdgpu_fence_driver_sw_init(adev);
4516 	if (r) {
4517 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4518 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4519 		goto failed;
4520 	}
4521 
4522 	/* init the mode config */
4523 	drm_mode_config_init(adev_to_drm(adev));
4524 
4525 	r = amdgpu_device_ip_init(adev);
4526 	if (r) {
4527 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4528 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4529 		goto release_ras_con;
4530 	}
4531 
4532 	amdgpu_fence_driver_hw_init(adev);
4533 
4534 	dev_info(adev->dev,
4535 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4536 			adev->gfx.config.max_shader_engines,
4537 			adev->gfx.config.max_sh_per_se,
4538 			adev->gfx.config.max_cu_per_sh,
4539 			adev->gfx.cu_info.number);
4540 
4541 	adev->accel_working = true;
4542 
4543 	amdgpu_vm_check_compute_bug(adev);
4544 
4545 	/* Initialize the buffer migration limit. */
4546 	if (amdgpu_moverate >= 0)
4547 		max_MBps = amdgpu_moverate;
4548 	else
4549 		max_MBps = 8; /* Allow 8 MB/s. */
4550 	/* Get a log2 for easy divisions. */
4551 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4552 
4553 	/*
4554 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4555 	 * Otherwise the mgpu fan boost feature will be skipped due to the
4556 	 * gpu instance is counted less.
4557 	 */
4558 	amdgpu_register_gpu_instance(adev);
4559 
4560 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4561 	 * explicit gating rather than handling it automatically.
4562 	 */
4563 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4564 		r = amdgpu_device_ip_late_init(adev);
4565 		if (r) {
4566 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4567 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4568 			goto release_ras_con;
4569 		}
4570 		/* must succeed. */
4571 		amdgpu_ras_resume(adev);
4572 		queue_delayed_work(system_wq, &adev->delayed_init_work,
4573 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
4574 	}
4575 
4576 	if (amdgpu_sriov_vf(adev)) {
4577 		amdgpu_virt_release_full_gpu(adev, true);
4578 		flush_delayed_work(&adev->delayed_init_work);
4579 	}
4580 
4581 	/*
4582 	 * Place those sysfs registering after `late_init`. As some of those
4583 	 * operations performed in `late_init` might affect the sysfs
4584 	 * interfaces creating.
4585 	 */
4586 	r = amdgpu_atombios_sysfs_init(adev);
4587 	if (r)
4588 		drm_err(&adev->ddev,
4589 			"registering atombios sysfs failed (%d).\n", r);
4590 
4591 	r = amdgpu_pm_sysfs_init(adev);
4592 	if (r)
4593 		DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4594 
4595 	r = amdgpu_ucode_sysfs_init(adev);
4596 	if (r) {
4597 		adev->ucode_sysfs_en = false;
4598 		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4599 	} else
4600 		adev->ucode_sysfs_en = true;
4601 
4602 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4603 	if (r)
4604 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
4605 
4606 	r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4607 	if (r)
4608 		dev_err(adev->dev,
4609 			"Could not create amdgpu board attributes\n");
4610 
4611 	amdgpu_fru_sysfs_init(adev);
4612 	amdgpu_reg_state_sysfs_init(adev);
4613 	amdgpu_xcp_cfg_sysfs_init(adev);
4614 
4615 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4616 		r = amdgpu_pmu_init(adev);
4617 	if (r)
4618 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4619 
4620 	/* Have stored pci confspace at hand for restore in sudden PCI error */
4621 	if (amdgpu_device_cache_pci_state(adev->pdev))
4622 		pci_restore_state(pdev);
4623 
4624 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4625 	/* this will fail for cards that aren't VGA class devices, just
4626 	 * ignore it
4627 	 */
4628 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4629 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4630 
4631 	px = amdgpu_device_supports_px(ddev);
4632 
4633 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4634 				apple_gmux_detect(NULL, NULL)))
4635 		vga_switcheroo_register_client(adev->pdev,
4636 					       &amdgpu_switcheroo_ops, px);
4637 
4638 	if (px)
4639 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4640 
4641 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
4642 		amdgpu_xgmi_reset_on_init(adev);
4643 
4644 	amdgpu_device_check_iommu_direct_map(adev);
4645 
4646 	adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
4647 	r = register_pm_notifier(&adev->pm_nb);
4648 	if (r)
4649 		goto failed;
4650 
4651 	return 0;
4652 
4653 release_ras_con:
4654 	if (amdgpu_sriov_vf(adev))
4655 		amdgpu_virt_release_full_gpu(adev, true);
4656 
4657 	/* failed in exclusive mode due to timeout */
4658 	if (amdgpu_sriov_vf(adev) &&
4659 		!amdgpu_sriov_runtime(adev) &&
4660 		amdgpu_virt_mmio_blocked(adev) &&
4661 		!amdgpu_virt_wait_reset(adev)) {
4662 		dev_err(adev->dev, "VF exclusive mode timeout\n");
4663 		/* Don't send request since VF is inactive. */
4664 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4665 		adev->virt.ops = NULL;
4666 		r = -EAGAIN;
4667 	}
4668 	amdgpu_release_ras_context(adev);
4669 
4670 failed:
4671 	amdgpu_vf_error_trans_all(adev);
4672 
4673 	return r;
4674 }
4675 
4676 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4677 {
4678 
4679 	/* Clear all CPU mappings pointing to this device */
4680 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4681 
4682 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4683 	amdgpu_doorbell_fini(adev);
4684 
4685 	iounmap(adev->rmmio);
4686 	adev->rmmio = NULL;
4687 	if (adev->mman.aper_base_kaddr)
4688 		iounmap(adev->mman.aper_base_kaddr);
4689 	adev->mman.aper_base_kaddr = NULL;
4690 
4691 	/* Memory manager related */
4692 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4693 		arch_phys_wc_del(adev->gmc.vram_mtrr);
4694 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4695 	}
4696 }
4697 
4698 /**
4699  * amdgpu_device_fini_hw - tear down the driver
4700  *
4701  * @adev: amdgpu_device pointer
4702  *
4703  * Tear down the driver info (all asics).
4704  * Called at driver shutdown.
4705  */
4706 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4707 {
4708 	dev_info(adev->dev, "amdgpu: finishing device.\n");
4709 	flush_delayed_work(&adev->delayed_init_work);
4710 
4711 	if (adev->mman.initialized)
4712 		drain_workqueue(adev->mman.bdev.wq);
4713 	adev->shutdown = true;
4714 
4715 	unregister_pm_notifier(&adev->pm_nb);
4716 
4717 	/* make sure IB test finished before entering exclusive mode
4718 	 * to avoid preemption on IB test
4719 	 */
4720 	if (amdgpu_sriov_vf(adev)) {
4721 		amdgpu_virt_request_full_gpu(adev, false);
4722 		amdgpu_virt_fini_data_exchange(adev);
4723 	}
4724 
4725 	/* disable all interrupts */
4726 	amdgpu_irq_disable_all(adev);
4727 	if (adev->mode_info.mode_config_initialized) {
4728 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4729 			drm_helper_force_disable_all(adev_to_drm(adev));
4730 		else
4731 			drm_atomic_helper_shutdown(adev_to_drm(adev));
4732 	}
4733 	amdgpu_fence_driver_hw_fini(adev);
4734 
4735 	if (adev->pm.sysfs_initialized)
4736 		amdgpu_pm_sysfs_fini(adev);
4737 	if (adev->ucode_sysfs_en)
4738 		amdgpu_ucode_sysfs_fini(adev);
4739 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4740 	amdgpu_fru_sysfs_fini(adev);
4741 
4742 	amdgpu_reg_state_sysfs_fini(adev);
4743 	amdgpu_xcp_cfg_sysfs_fini(adev);
4744 
4745 	/* disable ras feature must before hw fini */
4746 	amdgpu_ras_pre_fini(adev);
4747 
4748 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4749 
4750 	amdgpu_device_ip_fini_early(adev);
4751 
4752 	amdgpu_irq_fini_hw(adev);
4753 
4754 	if (adev->mman.initialized)
4755 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4756 
4757 	amdgpu_gart_dummy_page_fini(adev);
4758 
4759 	if (drm_dev_is_unplugged(adev_to_drm(adev)))
4760 		amdgpu_device_unmap_mmio(adev);
4761 
4762 }
4763 
4764 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4765 {
4766 	int idx;
4767 	bool px;
4768 
4769 	amdgpu_device_ip_fini(adev);
4770 	amdgpu_fence_driver_sw_fini(adev);
4771 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4772 	adev->accel_working = false;
4773 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4774 
4775 	amdgpu_reset_fini(adev);
4776 
4777 	/* free i2c buses */
4778 	amdgpu_i2c_fini(adev);
4779 
4780 	if (adev->bios) {
4781 		if (amdgpu_emu_mode != 1)
4782 			amdgpu_atombios_fini(adev);
4783 		amdgpu_bios_release(adev);
4784 	}
4785 
4786 	kfree(adev->fru_info);
4787 	adev->fru_info = NULL;
4788 
4789 	px = amdgpu_device_supports_px(adev_to_drm(adev));
4790 
4791 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4792 				apple_gmux_detect(NULL, NULL)))
4793 		vga_switcheroo_unregister_client(adev->pdev);
4794 
4795 	if (px)
4796 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4797 
4798 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4799 		vga_client_unregister(adev->pdev);
4800 
4801 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4802 
4803 		iounmap(adev->rmmio);
4804 		adev->rmmio = NULL;
4805 		amdgpu_doorbell_fini(adev);
4806 		drm_dev_exit(idx);
4807 	}
4808 
4809 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4810 		amdgpu_pmu_fini(adev);
4811 	if (adev->mman.discovery_bin)
4812 		amdgpu_discovery_fini(adev);
4813 
4814 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4815 	adev->reset_domain = NULL;
4816 
4817 	kfree(adev->pci_state);
4818 
4819 }
4820 
4821 /**
4822  * amdgpu_device_evict_resources - evict device resources
4823  * @adev: amdgpu device object
4824  *
4825  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4826  * of the vram memory type. Mainly used for evicting device resources
4827  * at suspend time.
4828  *
4829  */
4830 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4831 {
4832 	int ret;
4833 
4834 	/* No need to evict vram on APUs unless going to S4 */
4835 	if (!adev->in_s4 && (adev->flags & AMD_IS_APU))
4836 		return 0;
4837 
4838 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4839 	if (ret)
4840 		DRM_WARN("evicting device resources failed\n");
4841 	return ret;
4842 }
4843 
4844 /*
4845  * Suspend & resume.
4846  */
4847 /**
4848  * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events
4849  * @nb: notifier block
4850  * @mode: suspend mode
4851  * @data: data
4852  *
4853  * This function is called when the system is about to suspend or hibernate.
4854  * It is used to evict resources from the device before the system goes to
4855  * sleep while there is still access to swap.
4856  */
4857 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
4858 				     void *data)
4859 {
4860 	struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);
4861 	int r;
4862 
4863 	switch (mode) {
4864 	case PM_HIBERNATION_PREPARE:
4865 		adev->in_s4 = true;
4866 		fallthrough;
4867 	case PM_SUSPEND_PREPARE:
4868 		r = amdgpu_device_evict_resources(adev);
4869 		/*
4870 		 * This is considered non-fatal at this time because
4871 		 * amdgpu_device_prepare() will also fatally evict resources.
4872 		 * See https://gitlab.freedesktop.org/drm/amd/-/issues/3781
4873 		 */
4874 		if (r)
4875 			drm_warn(adev_to_drm(adev), "Failed to evict resources, freeze active processes if problems occur: %d\n", r);
4876 		break;
4877 	}
4878 
4879 	return NOTIFY_DONE;
4880 }
4881 
4882 /**
4883  * amdgpu_device_prepare - prepare for device suspend
4884  *
4885  * @dev: drm dev pointer
4886  *
4887  * Prepare to put the hw in the suspend state (all asics).
4888  * Returns 0 for success or an error on failure.
4889  * Called at driver suspend.
4890  */
4891 int amdgpu_device_prepare(struct drm_device *dev)
4892 {
4893 	struct amdgpu_device *adev = drm_to_adev(dev);
4894 	int i, r;
4895 
4896 	amdgpu_choose_low_power_state(adev);
4897 
4898 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4899 		return 0;
4900 
4901 	/* Evict the majority of BOs before starting suspend sequence */
4902 	r = amdgpu_device_evict_resources(adev);
4903 	if (r)
4904 		goto unprepare;
4905 
4906 	flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4907 
4908 	for (i = 0; i < adev->num_ip_blocks; i++) {
4909 		if (!adev->ip_blocks[i].status.valid)
4910 			continue;
4911 		if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4912 			continue;
4913 		r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);
4914 		if (r)
4915 			goto unprepare;
4916 	}
4917 
4918 	return 0;
4919 
4920 unprepare:
4921 	adev->in_s0ix = adev->in_s3 = adev->in_s4 = false;
4922 
4923 	return r;
4924 }
4925 
4926 /**
4927  * amdgpu_device_suspend - initiate device suspend
4928  *
4929  * @dev: drm dev pointer
4930  * @notify_clients: notify in-kernel DRM clients
4931  *
4932  * Puts the hw in the suspend state (all asics).
4933  * Returns 0 for success or an error on failure.
4934  * Called at driver suspend.
4935  */
4936 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
4937 {
4938 	struct amdgpu_device *adev = drm_to_adev(dev);
4939 	int r = 0;
4940 
4941 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4942 		return 0;
4943 
4944 	adev->in_suspend = true;
4945 
4946 	if (amdgpu_sriov_vf(adev)) {
4947 		amdgpu_virt_fini_data_exchange(adev);
4948 		r = amdgpu_virt_request_full_gpu(adev, false);
4949 		if (r)
4950 			return r;
4951 	}
4952 
4953 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4954 		DRM_WARN("smart shift update failed\n");
4955 
4956 	if (notify_clients)
4957 		drm_client_dev_suspend(adev_to_drm(adev), false);
4958 
4959 	cancel_delayed_work_sync(&adev->delayed_init_work);
4960 
4961 	amdgpu_ras_suspend(adev);
4962 
4963 	amdgpu_device_ip_suspend_phase1(adev);
4964 
4965 	if (!adev->in_s0ix)
4966 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4967 
4968 	r = amdgpu_device_evict_resources(adev);
4969 	if (r)
4970 		return r;
4971 
4972 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4973 
4974 	amdgpu_fence_driver_hw_fini(adev);
4975 
4976 	amdgpu_device_ip_suspend_phase2(adev);
4977 
4978 	if (amdgpu_sriov_vf(adev))
4979 		amdgpu_virt_release_full_gpu(adev, false);
4980 
4981 	r = amdgpu_dpm_notify_rlc_state(adev, false);
4982 	if (r)
4983 		return r;
4984 
4985 	return 0;
4986 }
4987 
4988 /**
4989  * amdgpu_device_resume - initiate device resume
4990  *
4991  * @dev: drm dev pointer
4992  * @notify_clients: notify in-kernel DRM clients
4993  *
4994  * Bring the hw back to operating state (all asics).
4995  * Returns 0 for success or an error on failure.
4996  * Called at driver resume.
4997  */
4998 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
4999 {
5000 	struct amdgpu_device *adev = drm_to_adev(dev);
5001 	int r = 0;
5002 
5003 	if (amdgpu_sriov_vf(adev)) {
5004 		r = amdgpu_virt_request_full_gpu(adev, true);
5005 		if (r)
5006 			return r;
5007 	}
5008 
5009 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
5010 		return 0;
5011 
5012 	if (adev->in_s0ix)
5013 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
5014 
5015 	/* post card */
5016 	if (amdgpu_device_need_post(adev)) {
5017 		r = amdgpu_device_asic_init(adev);
5018 		if (r)
5019 			dev_err(adev->dev, "amdgpu asic init failed\n");
5020 	}
5021 
5022 	r = amdgpu_device_ip_resume(adev);
5023 
5024 	if (r) {
5025 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
5026 		goto exit;
5027 	}
5028 
5029 	if (!adev->in_s0ix) {
5030 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
5031 		if (r)
5032 			goto exit;
5033 	}
5034 
5035 	r = amdgpu_device_ip_late_init(adev);
5036 	if (r)
5037 		goto exit;
5038 
5039 	queue_delayed_work(system_wq, &adev->delayed_init_work,
5040 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
5041 exit:
5042 	if (amdgpu_sriov_vf(adev)) {
5043 		amdgpu_virt_init_data_exchange(adev);
5044 		amdgpu_virt_release_full_gpu(adev, true);
5045 	}
5046 
5047 	if (r)
5048 		return r;
5049 
5050 	/* Make sure IB tests flushed */
5051 	flush_delayed_work(&adev->delayed_init_work);
5052 
5053 	if (notify_clients)
5054 		drm_client_dev_resume(adev_to_drm(adev), false);
5055 
5056 	amdgpu_ras_resume(adev);
5057 
5058 	if (adev->mode_info.num_crtc) {
5059 		/*
5060 		 * Most of the connector probing functions try to acquire runtime pm
5061 		 * refs to ensure that the GPU is powered on when connector polling is
5062 		 * performed. Since we're calling this from a runtime PM callback,
5063 		 * trying to acquire rpm refs will cause us to deadlock.
5064 		 *
5065 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
5066 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
5067 		 */
5068 #ifdef CONFIG_PM
5069 		dev->dev->power.disable_depth++;
5070 #endif
5071 		if (!adev->dc_enabled)
5072 			drm_helper_hpd_irq_event(dev);
5073 		else
5074 			drm_kms_helper_hotplug_event(dev);
5075 #ifdef CONFIG_PM
5076 		dev->dev->power.disable_depth--;
5077 #endif
5078 	}
5079 	adev->in_suspend = false;
5080 
5081 	if (adev->enable_mes)
5082 		amdgpu_mes_self_test(adev);
5083 
5084 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
5085 		DRM_WARN("smart shift update failed\n");
5086 
5087 	return 0;
5088 }
5089 
5090 /**
5091  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
5092  *
5093  * @adev: amdgpu_device pointer
5094  *
5095  * The list of all the hardware IPs that make up the asic is walked and
5096  * the check_soft_reset callbacks are run.  check_soft_reset determines
5097  * if the asic is still hung or not.
5098  * Returns true if any of the IPs are still in a hung state, false if not.
5099  */
5100 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
5101 {
5102 	int i;
5103 	bool asic_hang = false;
5104 
5105 	if (amdgpu_sriov_vf(adev))
5106 		return true;
5107 
5108 	if (amdgpu_asic_need_full_reset(adev))
5109 		return true;
5110 
5111 	for (i = 0; i < adev->num_ip_blocks; i++) {
5112 		if (!adev->ip_blocks[i].status.valid)
5113 			continue;
5114 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
5115 			adev->ip_blocks[i].status.hang =
5116 				adev->ip_blocks[i].version->funcs->check_soft_reset(
5117 					&adev->ip_blocks[i]);
5118 		if (adev->ip_blocks[i].status.hang) {
5119 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
5120 			asic_hang = true;
5121 		}
5122 	}
5123 	return asic_hang;
5124 }
5125 
5126 /**
5127  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
5128  *
5129  * @adev: amdgpu_device pointer
5130  *
5131  * The list of all the hardware IPs that make up the asic is walked and the
5132  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
5133  * handles any IP specific hardware or software state changes that are
5134  * necessary for a soft reset to succeed.
5135  * Returns 0 on success, negative error code on failure.
5136  */
5137 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
5138 {
5139 	int i, r = 0;
5140 
5141 	for (i = 0; i < adev->num_ip_blocks; i++) {
5142 		if (!adev->ip_blocks[i].status.valid)
5143 			continue;
5144 		if (adev->ip_blocks[i].status.hang &&
5145 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
5146 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]);
5147 			if (r)
5148 				return r;
5149 		}
5150 	}
5151 
5152 	return 0;
5153 }
5154 
5155 /**
5156  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
5157  *
5158  * @adev: amdgpu_device pointer
5159  *
5160  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
5161  * reset is necessary to recover.
5162  * Returns true if a full asic reset is required, false if not.
5163  */
5164 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
5165 {
5166 	int i;
5167 
5168 	if (amdgpu_asic_need_full_reset(adev))
5169 		return true;
5170 
5171 	for (i = 0; i < adev->num_ip_blocks; i++) {
5172 		if (!adev->ip_blocks[i].status.valid)
5173 			continue;
5174 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
5175 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
5176 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
5177 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
5178 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
5179 			if (adev->ip_blocks[i].status.hang) {
5180 				dev_info(adev->dev, "Some block need full reset!\n");
5181 				return true;
5182 			}
5183 		}
5184 	}
5185 	return false;
5186 }
5187 
5188 /**
5189  * amdgpu_device_ip_soft_reset - do a soft reset
5190  *
5191  * @adev: amdgpu_device pointer
5192  *
5193  * The list of all the hardware IPs that make up the asic is walked and the
5194  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
5195  * IP specific hardware or software state changes that are necessary to soft
5196  * reset the IP.
5197  * Returns 0 on success, negative error code on failure.
5198  */
5199 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
5200 {
5201 	int i, r = 0;
5202 
5203 	for (i = 0; i < adev->num_ip_blocks; i++) {
5204 		if (!adev->ip_blocks[i].status.valid)
5205 			continue;
5206 		if (adev->ip_blocks[i].status.hang &&
5207 		    adev->ip_blocks[i].version->funcs->soft_reset) {
5208 			r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]);
5209 			if (r)
5210 				return r;
5211 		}
5212 	}
5213 
5214 	return 0;
5215 }
5216 
5217 /**
5218  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
5219  *
5220  * @adev: amdgpu_device pointer
5221  *
5222  * The list of all the hardware IPs that make up the asic is walked and the
5223  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
5224  * handles any IP specific hardware or software state changes that are
5225  * necessary after the IP has been soft reset.
5226  * Returns 0 on success, negative error code on failure.
5227  */
5228 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
5229 {
5230 	int i, r = 0;
5231 
5232 	for (i = 0; i < adev->num_ip_blocks; i++) {
5233 		if (!adev->ip_blocks[i].status.valid)
5234 			continue;
5235 		if (adev->ip_blocks[i].status.hang &&
5236 		    adev->ip_blocks[i].version->funcs->post_soft_reset)
5237 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]);
5238 		if (r)
5239 			return r;
5240 	}
5241 
5242 	return 0;
5243 }
5244 
5245 /**
5246  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
5247  *
5248  * @adev: amdgpu_device pointer
5249  * @reset_context: amdgpu reset context pointer
5250  *
5251  * do VF FLR and reinitialize Asic
5252  * return 0 means succeeded otherwise failed
5253  */
5254 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
5255 				     struct amdgpu_reset_context *reset_context)
5256 {
5257 	int r;
5258 	struct amdgpu_hive_info *hive = NULL;
5259 
5260 	if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) {
5261 		if (!amdgpu_ras_get_fed_status(adev))
5262 			amdgpu_virt_ready_to_reset(adev);
5263 		amdgpu_virt_wait_reset(adev);
5264 		clear_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5265 		r = amdgpu_virt_request_full_gpu(adev, true);
5266 	} else {
5267 		r = amdgpu_virt_reset_gpu(adev);
5268 	}
5269 	if (r)
5270 		return r;
5271 
5272 	amdgpu_ras_clear_err_state(adev);
5273 	amdgpu_irq_gpu_reset_resume_helper(adev);
5274 
5275 	/* some sw clean up VF needs to do before recover */
5276 	amdgpu_virt_post_reset(adev);
5277 
5278 	/* Resume IP prior to SMC */
5279 	r = amdgpu_device_ip_reinit_early_sriov(adev);
5280 	if (r)
5281 		return r;
5282 
5283 	amdgpu_virt_init_data_exchange(adev);
5284 
5285 	r = amdgpu_device_fw_loading(adev);
5286 	if (r)
5287 		return r;
5288 
5289 	/* now we are okay to resume SMC/CP/SDMA */
5290 	r = amdgpu_device_ip_reinit_late_sriov(adev);
5291 	if (r)
5292 		return r;
5293 
5294 	hive = amdgpu_get_xgmi_hive(adev);
5295 	/* Update PSP FW topology after reset */
5296 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5297 		r = amdgpu_xgmi_update_topology(hive, adev);
5298 	if (hive)
5299 		amdgpu_put_xgmi_hive(hive);
5300 	if (r)
5301 		return r;
5302 
5303 	r = amdgpu_ib_ring_tests(adev);
5304 	if (r)
5305 		return r;
5306 
5307 	if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)
5308 		amdgpu_inc_vram_lost(adev);
5309 
5310 	/* need to be called during full access so we can't do it later like
5311 	 * bare-metal does.
5312 	 */
5313 	amdgpu_amdkfd_post_reset(adev);
5314 	amdgpu_virt_release_full_gpu(adev, true);
5315 
5316 	/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5317 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
5318 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
5319 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
5320 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5321 		amdgpu_ras_resume(adev);
5322 
5323 	amdgpu_virt_ras_telemetry_post_reset(adev);
5324 
5325 	return 0;
5326 }
5327 
5328 /**
5329  * amdgpu_device_has_job_running - check if there is any unfinished job
5330  *
5331  * @adev: amdgpu_device pointer
5332  *
5333  * check if there is any job running on the device when guest driver receives
5334  * FLR notification from host driver. If there are still jobs running, then
5335  * the guest driver will not respond the FLR reset. Instead, let the job hit
5336  * the timeout and guest driver then issue the reset request.
5337  */
5338 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5339 {
5340 	int i;
5341 
5342 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5343 		struct amdgpu_ring *ring = adev->rings[i];
5344 
5345 		if (!amdgpu_ring_sched_ready(ring))
5346 			continue;
5347 
5348 		if (amdgpu_fence_count_emitted(ring))
5349 			return true;
5350 	}
5351 	return false;
5352 }
5353 
5354 /**
5355  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5356  *
5357  * @adev: amdgpu_device pointer
5358  *
5359  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5360  * a hung GPU.
5361  */
5362 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5363 {
5364 
5365 	if (amdgpu_gpu_recovery == 0)
5366 		goto disabled;
5367 
5368 	/* Skip soft reset check in fatal error mode */
5369 	if (!amdgpu_ras_is_poison_mode_supported(adev))
5370 		return true;
5371 
5372 	if (amdgpu_sriov_vf(adev))
5373 		return true;
5374 
5375 	if (amdgpu_gpu_recovery == -1) {
5376 		switch (adev->asic_type) {
5377 #ifdef CONFIG_DRM_AMDGPU_SI
5378 		case CHIP_VERDE:
5379 		case CHIP_TAHITI:
5380 		case CHIP_PITCAIRN:
5381 		case CHIP_OLAND:
5382 		case CHIP_HAINAN:
5383 #endif
5384 #ifdef CONFIG_DRM_AMDGPU_CIK
5385 		case CHIP_KAVERI:
5386 		case CHIP_KABINI:
5387 		case CHIP_MULLINS:
5388 #endif
5389 		case CHIP_CARRIZO:
5390 		case CHIP_STONEY:
5391 		case CHIP_CYAN_SKILLFISH:
5392 			goto disabled;
5393 		default:
5394 			break;
5395 		}
5396 	}
5397 
5398 	return true;
5399 
5400 disabled:
5401 		dev_info(adev->dev, "GPU recovery disabled.\n");
5402 		return false;
5403 }
5404 
5405 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5406 {
5407 	u32 i;
5408 	int ret = 0;
5409 
5410 	if (adev->bios)
5411 		amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5412 
5413 	dev_info(adev->dev, "GPU mode1 reset\n");
5414 
5415 	/* Cache the state before bus master disable. The saved config space
5416 	 * values are used in other cases like restore after mode-2 reset.
5417 	 */
5418 	amdgpu_device_cache_pci_state(adev->pdev);
5419 
5420 	/* disable BM */
5421 	pci_clear_master(adev->pdev);
5422 
5423 	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5424 		dev_info(adev->dev, "GPU smu mode1 reset\n");
5425 		ret = amdgpu_dpm_mode1_reset(adev);
5426 	} else {
5427 		dev_info(adev->dev, "GPU psp mode1 reset\n");
5428 		ret = psp_gpu_reset(adev);
5429 	}
5430 
5431 	if (ret)
5432 		goto mode1_reset_failed;
5433 
5434 	amdgpu_device_load_pci_state(adev->pdev);
5435 	ret = amdgpu_psp_wait_for_bootloader(adev);
5436 	if (ret)
5437 		goto mode1_reset_failed;
5438 
5439 	/* wait for asic to come out of reset */
5440 	for (i = 0; i < adev->usec_timeout; i++) {
5441 		u32 memsize = adev->nbio.funcs->get_memsize(adev);
5442 
5443 		if (memsize != 0xffffffff)
5444 			break;
5445 		udelay(1);
5446 	}
5447 
5448 	if (i >= adev->usec_timeout) {
5449 		ret = -ETIMEDOUT;
5450 		goto mode1_reset_failed;
5451 	}
5452 
5453 	if (adev->bios)
5454 		amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5455 
5456 	return 0;
5457 
5458 mode1_reset_failed:
5459 	dev_err(adev->dev, "GPU mode1 reset failed\n");
5460 	return ret;
5461 }
5462 
5463 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5464 				 struct amdgpu_reset_context *reset_context)
5465 {
5466 	int i, r = 0;
5467 	struct amdgpu_job *job = NULL;
5468 	struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
5469 	bool need_full_reset =
5470 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5471 
5472 	if (reset_context->reset_req_dev == adev)
5473 		job = reset_context->job;
5474 
5475 	if (amdgpu_sriov_vf(adev))
5476 		amdgpu_virt_pre_reset(adev);
5477 
5478 	amdgpu_fence_driver_isr_toggle(adev, true);
5479 
5480 	/* block all schedulers and reset given job's ring */
5481 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5482 		struct amdgpu_ring *ring = adev->rings[i];
5483 
5484 		if (!amdgpu_ring_sched_ready(ring))
5485 			continue;
5486 
5487 		/* Clear job fence from fence drv to avoid force_completion
5488 		 * leave NULL and vm flush fence in fence drv
5489 		 */
5490 		amdgpu_fence_driver_clear_job_fences(ring);
5491 
5492 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5493 		amdgpu_fence_driver_force_completion(ring);
5494 	}
5495 
5496 	amdgpu_fence_driver_isr_toggle(adev, false);
5497 
5498 	if (job && job->vm)
5499 		drm_sched_increase_karma(&job->base);
5500 
5501 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5502 	/* If reset handler not implemented, continue; otherwise return */
5503 	if (r == -EOPNOTSUPP)
5504 		r = 0;
5505 	else
5506 		return r;
5507 
5508 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5509 	if (!amdgpu_sriov_vf(adev)) {
5510 
5511 		if (!need_full_reset)
5512 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5513 
5514 		if (!need_full_reset && amdgpu_gpu_recovery &&
5515 		    amdgpu_device_ip_check_soft_reset(adev)) {
5516 			amdgpu_device_ip_pre_soft_reset(adev);
5517 			r = amdgpu_device_ip_soft_reset(adev);
5518 			amdgpu_device_ip_post_soft_reset(adev);
5519 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5520 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5521 				need_full_reset = true;
5522 			}
5523 		}
5524 
5525 		if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
5526 			dev_info(tmp_adev->dev, "Dumping IP State\n");
5527 			/* Trigger ip dump before we reset the asic */
5528 			for (i = 0; i < tmp_adev->num_ip_blocks; i++)
5529 				if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
5530 					tmp_adev->ip_blocks[i].version->funcs
5531 						->dump_ip_state((void *)&tmp_adev->ip_blocks[i]);
5532 			dev_info(tmp_adev->dev, "Dumping IP State Completed\n");
5533 		}
5534 
5535 		if (need_full_reset)
5536 			r = amdgpu_device_ip_suspend(adev);
5537 		if (need_full_reset)
5538 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5539 		else
5540 			clear_bit(AMDGPU_NEED_FULL_RESET,
5541 				  &reset_context->flags);
5542 	}
5543 
5544 	return r;
5545 }
5546 
5547 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
5548 {
5549 	struct list_head *device_list_handle;
5550 	bool full_reset, vram_lost = false;
5551 	struct amdgpu_device *tmp_adev;
5552 	int r, init_level;
5553 
5554 	device_list_handle = reset_context->reset_device_list;
5555 
5556 	if (!device_list_handle)
5557 		return -EINVAL;
5558 
5559 	full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5560 
5561 	/**
5562 	 * If it's reset on init, it's default init level, otherwise keep level
5563 	 * as recovery level.
5564 	 */
5565 	if (reset_context->method == AMD_RESET_METHOD_ON_INIT)
5566 			init_level = AMDGPU_INIT_LEVEL_DEFAULT;
5567 	else
5568 			init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY;
5569 
5570 	r = 0;
5571 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5572 		amdgpu_set_init_level(tmp_adev, init_level);
5573 		if (full_reset) {
5574 			/* post card */
5575 			amdgpu_ras_clear_err_state(tmp_adev);
5576 			r = amdgpu_device_asic_init(tmp_adev);
5577 			if (r) {
5578 				dev_warn(tmp_adev->dev, "asic atom init failed!");
5579 			} else {
5580 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5581 
5582 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
5583 				if (r)
5584 					goto out;
5585 
5586 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5587 
5588 				if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
5589 					amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job);
5590 
5591 				if (vram_lost) {
5592 					DRM_INFO("VRAM is lost due to GPU reset!\n");
5593 					amdgpu_inc_vram_lost(tmp_adev);
5594 				}
5595 
5596 				r = amdgpu_device_fw_loading(tmp_adev);
5597 				if (r)
5598 					return r;
5599 
5600 				r = amdgpu_xcp_restore_partition_mode(
5601 					tmp_adev->xcp_mgr);
5602 				if (r)
5603 					goto out;
5604 
5605 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
5606 				if (r)
5607 					goto out;
5608 
5609 				if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5610 					amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5611 
5612 				r = amdgpu_device_ip_resume_phase3(tmp_adev);
5613 				if (r)
5614 					goto out;
5615 
5616 				if (vram_lost)
5617 					amdgpu_device_fill_reset_magic(tmp_adev);
5618 
5619 				/*
5620 				 * Add this ASIC as tracked as reset was already
5621 				 * complete successfully.
5622 				 */
5623 				amdgpu_register_gpu_instance(tmp_adev);
5624 
5625 				if (!reset_context->hive &&
5626 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5627 					amdgpu_xgmi_add_device(tmp_adev);
5628 
5629 				r = amdgpu_device_ip_late_init(tmp_adev);
5630 				if (r)
5631 					goto out;
5632 
5633 				drm_client_dev_resume(adev_to_drm(tmp_adev), false);
5634 
5635 				/*
5636 				 * The GPU enters bad state once faulty pages
5637 				 * by ECC has reached the threshold, and ras
5638 				 * recovery is scheduled next. So add one check
5639 				 * here to break recovery if it indeed exceeds
5640 				 * bad page threshold, and remind user to
5641 				 * retire this GPU or setting one bigger
5642 				 * bad_page_threshold value to fix this once
5643 				 * probing driver again.
5644 				 */
5645 				if (!amdgpu_ras_is_rma(tmp_adev)) {
5646 					/* must succeed. */
5647 					amdgpu_ras_resume(tmp_adev);
5648 				} else {
5649 					r = -EINVAL;
5650 					goto out;
5651 				}
5652 
5653 				/* Update PSP FW topology after reset */
5654 				if (reset_context->hive &&
5655 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5656 					r = amdgpu_xgmi_update_topology(
5657 						reset_context->hive, tmp_adev);
5658 			}
5659 		}
5660 
5661 out:
5662 		if (!r) {
5663 			/* IP init is complete now, set level as default */
5664 			amdgpu_set_init_level(tmp_adev,
5665 					      AMDGPU_INIT_LEVEL_DEFAULT);
5666 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5667 			r = amdgpu_ib_ring_tests(tmp_adev);
5668 			if (r) {
5669 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5670 				r = -EAGAIN;
5671 				goto end;
5672 			}
5673 		}
5674 
5675 		if (r)
5676 			tmp_adev->asic_reset_res = r;
5677 	}
5678 
5679 end:
5680 	return r;
5681 }
5682 
5683 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5684 			 struct amdgpu_reset_context *reset_context)
5685 {
5686 	struct amdgpu_device *tmp_adev = NULL;
5687 	bool need_full_reset, skip_hw_reset;
5688 	int r = 0;
5689 
5690 	/* Try reset handler method first */
5691 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5692 				    reset_list);
5693 
5694 	reset_context->reset_device_list = device_list_handle;
5695 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5696 	/* If reset handler not implemented, continue; otherwise return */
5697 	if (r == -EOPNOTSUPP)
5698 		r = 0;
5699 	else
5700 		return r;
5701 
5702 	/* Reset handler not implemented, use the default method */
5703 	need_full_reset =
5704 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5705 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5706 
5707 	/*
5708 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
5709 	 * to allow proper links negotiation in FW (within 1 sec)
5710 	 */
5711 	if (!skip_hw_reset && need_full_reset) {
5712 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5713 			/* For XGMI run all resets in parallel to speed up the process */
5714 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5715 				if (!queue_work(system_unbound_wq,
5716 						&tmp_adev->xgmi_reset_work))
5717 					r = -EALREADY;
5718 			} else
5719 				r = amdgpu_asic_reset(tmp_adev);
5720 
5721 			if (r) {
5722 				dev_err(tmp_adev->dev,
5723 					"ASIC reset failed with error, %d for drm dev, %s",
5724 					r, adev_to_drm(tmp_adev)->unique);
5725 				goto out;
5726 			}
5727 		}
5728 
5729 		/* For XGMI wait for all resets to complete before proceed */
5730 		if (!r) {
5731 			list_for_each_entry(tmp_adev, device_list_handle,
5732 					    reset_list) {
5733 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5734 					flush_work(&tmp_adev->xgmi_reset_work);
5735 					r = tmp_adev->asic_reset_res;
5736 					if (r)
5737 						break;
5738 				}
5739 			}
5740 		}
5741 	}
5742 
5743 	if (!r && amdgpu_ras_intr_triggered()) {
5744 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5745 			amdgpu_ras_reset_error_count(tmp_adev,
5746 						     AMDGPU_RAS_BLOCK__MMHUB);
5747 		}
5748 
5749 		amdgpu_ras_intr_cleared();
5750 	}
5751 
5752 	r = amdgpu_device_reinit_after_reset(reset_context);
5753 	if (r == -EAGAIN)
5754 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5755 	else
5756 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5757 
5758 out:
5759 	return r;
5760 }
5761 
5762 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5763 {
5764 
5765 	switch (amdgpu_asic_reset_method(adev)) {
5766 	case AMD_RESET_METHOD_MODE1:
5767 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5768 		break;
5769 	case AMD_RESET_METHOD_MODE2:
5770 		adev->mp1_state = PP_MP1_STATE_RESET;
5771 		break;
5772 	default:
5773 		adev->mp1_state = PP_MP1_STATE_NONE;
5774 		break;
5775 	}
5776 }
5777 
5778 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5779 {
5780 	amdgpu_vf_error_trans_all(adev);
5781 	adev->mp1_state = PP_MP1_STATE_NONE;
5782 }
5783 
5784 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5785 {
5786 	struct pci_dev *p = NULL;
5787 
5788 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5789 			adev->pdev->bus->number, 1);
5790 	if (p) {
5791 		pm_runtime_enable(&(p->dev));
5792 		pm_runtime_resume(&(p->dev));
5793 	}
5794 
5795 	pci_dev_put(p);
5796 }
5797 
5798 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5799 {
5800 	enum amd_reset_method reset_method;
5801 	struct pci_dev *p = NULL;
5802 	u64 expires;
5803 
5804 	/*
5805 	 * For now, only BACO and mode1 reset are confirmed
5806 	 * to suffer the audio issue without proper suspended.
5807 	 */
5808 	reset_method = amdgpu_asic_reset_method(adev);
5809 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5810 	     (reset_method != AMD_RESET_METHOD_MODE1))
5811 		return -EINVAL;
5812 
5813 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5814 			adev->pdev->bus->number, 1);
5815 	if (!p)
5816 		return -ENODEV;
5817 
5818 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5819 	if (!expires)
5820 		/*
5821 		 * If we cannot get the audio device autosuspend delay,
5822 		 * a fixed 4S interval will be used. Considering 3S is
5823 		 * the audio controller default autosuspend delay setting.
5824 		 * 4S used here is guaranteed to cover that.
5825 		 */
5826 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5827 
5828 	while (!pm_runtime_status_suspended(&(p->dev))) {
5829 		if (!pm_runtime_suspend(&(p->dev)))
5830 			break;
5831 
5832 		if (expires < ktime_get_mono_fast_ns()) {
5833 			dev_warn(adev->dev, "failed to suspend display audio\n");
5834 			pci_dev_put(p);
5835 			/* TODO: abort the succeeding gpu reset? */
5836 			return -ETIMEDOUT;
5837 		}
5838 	}
5839 
5840 	pm_runtime_disable(&(p->dev));
5841 
5842 	pci_dev_put(p);
5843 	return 0;
5844 }
5845 
5846 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5847 {
5848 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5849 
5850 #if defined(CONFIG_DEBUG_FS)
5851 	if (!amdgpu_sriov_vf(adev))
5852 		cancel_work(&adev->reset_work);
5853 #endif
5854 
5855 	if (adev->kfd.dev)
5856 		cancel_work(&adev->kfd.reset_work);
5857 
5858 	if (amdgpu_sriov_vf(adev))
5859 		cancel_work(&adev->virt.flr_work);
5860 
5861 	if (con && adev->ras_enabled)
5862 		cancel_work(&con->recovery_work);
5863 
5864 }
5865 
5866 static int amdgpu_device_health_check(struct list_head *device_list_handle)
5867 {
5868 	struct amdgpu_device *tmp_adev;
5869 	int ret = 0;
5870 	u32 status;
5871 
5872 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5873 		pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status);
5874 		if (PCI_POSSIBLE_ERROR(status)) {
5875 			dev_err(tmp_adev->dev, "device lost from bus!");
5876 			ret = -ENODEV;
5877 		}
5878 	}
5879 
5880 	return ret;
5881 }
5882 
5883 /**
5884  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5885  *
5886  * @adev: amdgpu_device pointer
5887  * @job: which job trigger hang
5888  * @reset_context: amdgpu reset context pointer
5889  *
5890  * Attempt to reset the GPU if it has hung (all asics).
5891  * Attempt to do soft-reset or full-reset and reinitialize Asic
5892  * Returns 0 for success or an error on failure.
5893  */
5894 
5895 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5896 			      struct amdgpu_job *job,
5897 			      struct amdgpu_reset_context *reset_context)
5898 {
5899 	struct list_head device_list, *device_list_handle =  NULL;
5900 	bool job_signaled = false;
5901 	struct amdgpu_hive_info *hive = NULL;
5902 	struct amdgpu_device *tmp_adev = NULL;
5903 	int i, r = 0;
5904 	bool need_emergency_restart = false;
5905 	bool audio_suspended = false;
5906 	int retry_limit = AMDGPU_MAX_RETRY_LIMIT;
5907 
5908 	/*
5909 	 * If it reaches here because of hang/timeout and a RAS error is
5910 	 * detected at the same time, let RAS recovery take care of it.
5911 	 */
5912 	if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) &&
5913 	    !amdgpu_sriov_vf(adev) &&
5914 	    reset_context->src != AMDGPU_RESET_SRC_RAS) {
5915 		dev_dbg(adev->dev,
5916 			"Gpu recovery from source: %d yielding to RAS error recovery handling",
5917 			reset_context->src);
5918 		return 0;
5919 	}
5920 	/*
5921 	 * Special case: RAS triggered and full reset isn't supported
5922 	 */
5923 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5924 
5925 	/*
5926 	 * Flush RAM to disk so that after reboot
5927 	 * the user can read log and see why the system rebooted.
5928 	 */
5929 	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5930 		amdgpu_ras_get_context(adev)->reboot) {
5931 		DRM_WARN("Emergency reboot.");
5932 
5933 		ksys_sync_helper();
5934 		emergency_restart();
5935 	}
5936 
5937 	dev_info(adev->dev, "GPU %s begin!\n",
5938 		need_emergency_restart ? "jobs stop":"reset");
5939 
5940 	if (!amdgpu_sriov_vf(adev))
5941 		hive = amdgpu_get_xgmi_hive(adev);
5942 	if (hive)
5943 		mutex_lock(&hive->hive_lock);
5944 
5945 	reset_context->job = job;
5946 	reset_context->hive = hive;
5947 	/*
5948 	 * Build list of devices to reset.
5949 	 * In case we are in XGMI hive mode, resort the device list
5950 	 * to put adev in the 1st position.
5951 	 */
5952 	INIT_LIST_HEAD(&device_list);
5953 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
5954 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5955 			list_add_tail(&tmp_adev->reset_list, &device_list);
5956 			if (adev->shutdown)
5957 				tmp_adev->shutdown = true;
5958 		}
5959 		if (!list_is_first(&adev->reset_list, &device_list))
5960 			list_rotate_to_front(&adev->reset_list, &device_list);
5961 		device_list_handle = &device_list;
5962 	} else {
5963 		list_add_tail(&adev->reset_list, &device_list);
5964 		device_list_handle = &device_list;
5965 	}
5966 
5967 	if (!amdgpu_sriov_vf(adev)) {
5968 		r = amdgpu_device_health_check(device_list_handle);
5969 		if (r)
5970 			goto end_reset;
5971 	}
5972 
5973 	/* We need to lock reset domain only once both for XGMI and single device */
5974 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5975 				    reset_list);
5976 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5977 
5978 	/* block all schedulers and reset given job's ring */
5979 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5980 
5981 		amdgpu_device_set_mp1_state(tmp_adev);
5982 
5983 		/*
5984 		 * Try to put the audio codec into suspend state
5985 		 * before gpu reset started.
5986 		 *
5987 		 * Due to the power domain of the graphics device
5988 		 * is shared with AZ power domain. Without this,
5989 		 * we may change the audio hardware from behind
5990 		 * the audio driver's back. That will trigger
5991 		 * some audio codec errors.
5992 		 */
5993 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5994 			audio_suspended = true;
5995 
5996 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5997 
5998 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5999 
6000 		amdgpu_amdkfd_pre_reset(tmp_adev, reset_context);
6001 
6002 		/*
6003 		 * Mark these ASICs to be reset as untracked first
6004 		 * And add them back after reset completed
6005 		 */
6006 		amdgpu_unregister_gpu_instance(tmp_adev);
6007 
6008 		drm_client_dev_suspend(adev_to_drm(tmp_adev), false);
6009 
6010 		/* disable ras on ALL IPs */
6011 		if (!need_emergency_restart &&
6012 		      amdgpu_device_ip_need_full_reset(tmp_adev))
6013 			amdgpu_ras_suspend(tmp_adev);
6014 
6015 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6016 			struct amdgpu_ring *ring = tmp_adev->rings[i];
6017 
6018 			if (!amdgpu_ring_sched_ready(ring))
6019 				continue;
6020 
6021 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
6022 
6023 			if (need_emergency_restart)
6024 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
6025 		}
6026 		atomic_inc(&tmp_adev->gpu_reset_counter);
6027 	}
6028 
6029 	if (need_emergency_restart)
6030 		goto skip_sched_resume;
6031 
6032 	/*
6033 	 * Must check guilty signal here since after this point all old
6034 	 * HW fences are force signaled.
6035 	 *
6036 	 * job->base holds a reference to parent fence
6037 	 */
6038 	if (job && dma_fence_is_signaled(&job->hw_fence)) {
6039 		job_signaled = true;
6040 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
6041 		goto skip_hw_reset;
6042 	}
6043 
6044 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
6045 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6046 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
6047 		/*TODO Should we stop ?*/
6048 		if (r) {
6049 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
6050 				  r, adev_to_drm(tmp_adev)->unique);
6051 			tmp_adev->asic_reset_res = r;
6052 		}
6053 	}
6054 
6055 	/* Actual ASIC resets if needed.*/
6056 	/* Host driver will handle XGMI hive reset for SRIOV */
6057 	if (amdgpu_sriov_vf(adev)) {
6058 		if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
6059 			dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
6060 			amdgpu_ras_set_fed(adev, true);
6061 			set_bit(AMDGPU_HOST_FLR, &reset_context->flags);
6062 		}
6063 
6064 		r = amdgpu_device_reset_sriov(adev, reset_context);
6065 		if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) {
6066 			amdgpu_virt_release_full_gpu(adev, true);
6067 			goto retry;
6068 		}
6069 		if (r)
6070 			adev->asic_reset_res = r;
6071 	} else {
6072 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
6073 		if (r && r == -EAGAIN)
6074 			goto retry;
6075 	}
6076 
6077 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6078 		/*
6079 		 * Drop any pending non scheduler resets queued before reset is done.
6080 		 * Any reset scheduled after this point would be valid. Scheduler resets
6081 		 * were already dropped during drm_sched_stop and no new ones can come
6082 		 * in before drm_sched_start.
6083 		 */
6084 		amdgpu_device_stop_pending_resets(tmp_adev);
6085 	}
6086 
6087 skip_hw_reset:
6088 
6089 	/* Post ASIC reset for all devs .*/
6090 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6091 
6092 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6093 			struct amdgpu_ring *ring = tmp_adev->rings[i];
6094 
6095 			if (!amdgpu_ring_sched_ready(ring))
6096 				continue;
6097 
6098 			drm_sched_start(&ring->sched, 0);
6099 		}
6100 
6101 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
6102 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
6103 
6104 		if (tmp_adev->asic_reset_res)
6105 			r = tmp_adev->asic_reset_res;
6106 
6107 		tmp_adev->asic_reset_res = 0;
6108 
6109 		if (r) {
6110 			/* bad news, how to tell it to userspace ?
6111 			 * for ras error, we should report GPU bad status instead of
6112 			 * reset failure
6113 			 */
6114 			if (reset_context->src != AMDGPU_RESET_SRC_RAS ||
6115 			    !amdgpu_ras_eeprom_check_err_threshold(tmp_adev))
6116 				dev_info(tmp_adev->dev, "GPU reset(%d) failed\n",
6117 					atomic_read(&tmp_adev->gpu_reset_counter));
6118 			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
6119 		} else {
6120 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
6121 			if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
6122 				DRM_WARN("smart shift update failed\n");
6123 		}
6124 	}
6125 
6126 skip_sched_resume:
6127 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
6128 		/* unlock kfd: SRIOV would do it separately */
6129 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
6130 			amdgpu_amdkfd_post_reset(tmp_adev);
6131 
6132 		/* kfd_post_reset will do nothing if kfd device is not initialized,
6133 		 * need to bring up kfd here if it's not be initialized before
6134 		 */
6135 		if (!adev->kfd.init_complete)
6136 			amdgpu_amdkfd_device_init(adev);
6137 
6138 		if (audio_suspended)
6139 			amdgpu_device_resume_display_audio(tmp_adev);
6140 
6141 		amdgpu_device_unset_mp1_state(tmp_adev);
6142 
6143 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
6144 	}
6145 
6146 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
6147 					    reset_list);
6148 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
6149 
6150 end_reset:
6151 	if (hive) {
6152 		mutex_unlock(&hive->hive_lock);
6153 		amdgpu_put_xgmi_hive(hive);
6154 	}
6155 
6156 	if (r)
6157 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
6158 
6159 	atomic_set(&adev->reset_domain->reset_res, r);
6160 	return r;
6161 }
6162 
6163 /**
6164  * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
6165  *
6166  * @adev: amdgpu_device pointer
6167  * @speed: pointer to the speed of the link
6168  * @width: pointer to the width of the link
6169  *
6170  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
6171  * first physical partner to an AMD dGPU.
6172  * This will exclude any virtual switches and links.
6173  */
6174 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
6175 					    enum pci_bus_speed *speed,
6176 					    enum pcie_link_width *width)
6177 {
6178 	struct pci_dev *parent = adev->pdev;
6179 
6180 	if (!speed || !width)
6181 		return;
6182 
6183 	*speed = PCI_SPEED_UNKNOWN;
6184 	*width = PCIE_LNK_WIDTH_UNKNOWN;
6185 
6186 	if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
6187 		while ((parent = pci_upstream_bridge(parent))) {
6188 			/* skip upstream/downstream switches internal to dGPU*/
6189 			if (parent->vendor == PCI_VENDOR_ID_ATI)
6190 				continue;
6191 			*speed = pcie_get_speed_cap(parent);
6192 			*width = pcie_get_width_cap(parent);
6193 			break;
6194 		}
6195 	} else {
6196 		/* use the current speeds rather than max if switching is not supported */
6197 		pcie_bandwidth_available(adev->pdev, NULL, speed, width);
6198 	}
6199 }
6200 
6201 /**
6202  * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU
6203  *
6204  * @adev: amdgpu_device pointer
6205  * @speed: pointer to the speed of the link
6206  * @width: pointer to the width of the link
6207  *
6208  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
6209  * AMD dGPU which may be a virtual upstream bridge.
6210  */
6211 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,
6212 					enum pci_bus_speed *speed,
6213 					enum pcie_link_width *width)
6214 {
6215 	struct pci_dev *parent = adev->pdev;
6216 
6217 	if (!speed || !width)
6218 		return;
6219 
6220 	parent = pci_upstream_bridge(parent);
6221 	if (parent && parent->vendor == PCI_VENDOR_ID_ATI) {
6222 		/* use the upstream/downstream switches internal to dGPU */
6223 		*speed = pcie_get_speed_cap(parent);
6224 		*width = pcie_get_width_cap(parent);
6225 		while ((parent = pci_upstream_bridge(parent))) {
6226 			if (parent->vendor == PCI_VENDOR_ID_ATI) {
6227 				/* use the upstream/downstream switches internal to dGPU */
6228 				*speed = pcie_get_speed_cap(parent);
6229 				*width = pcie_get_width_cap(parent);
6230 			}
6231 		}
6232 	} else {
6233 		/* use the device itself */
6234 		*speed = pcie_get_speed_cap(adev->pdev);
6235 		*width = pcie_get_width_cap(adev->pdev);
6236 	}
6237 }
6238 
6239 /**
6240  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
6241  *
6242  * @adev: amdgpu_device pointer
6243  *
6244  * Fetches and stores in the driver the PCIE capabilities (gen speed
6245  * and lanes) of the slot the device is in. Handles APUs and
6246  * virtualized environments where PCIE config space may not be available.
6247  */
6248 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
6249 {
6250 	enum pci_bus_speed speed_cap, platform_speed_cap;
6251 	enum pcie_link_width platform_link_width, link_width;
6252 
6253 	if (amdgpu_pcie_gen_cap)
6254 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
6255 
6256 	if (amdgpu_pcie_lane_cap)
6257 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
6258 
6259 	/* covers APUs as well */
6260 	if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
6261 		if (adev->pm.pcie_gen_mask == 0)
6262 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
6263 		if (adev->pm.pcie_mlw_mask == 0)
6264 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
6265 		return;
6266 	}
6267 
6268 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
6269 		return;
6270 
6271 	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
6272 					&platform_link_width);
6273 	amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width);
6274 
6275 	if (adev->pm.pcie_gen_mask == 0) {
6276 		/* asic caps */
6277 		if (speed_cap == PCI_SPEED_UNKNOWN) {
6278 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6279 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6280 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6281 		} else {
6282 			if (speed_cap == PCIE_SPEED_32_0GT)
6283 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6284 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6285 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6286 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6287 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
6288 			else if (speed_cap == PCIE_SPEED_16_0GT)
6289 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6290 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6291 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6292 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
6293 			else if (speed_cap == PCIE_SPEED_8_0GT)
6294 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6295 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6296 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
6297 			else if (speed_cap == PCIE_SPEED_5_0GT)
6298 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6299 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
6300 			else
6301 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
6302 		}
6303 		/* platform caps */
6304 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
6305 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6306 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6307 		} else {
6308 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
6309 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6310 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6311 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6312 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
6313 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
6314 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
6315 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6316 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6317 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6318 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
6319 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
6320 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6321 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6322 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
6323 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
6324 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6325 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6326 			else
6327 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
6328 
6329 		}
6330 	}
6331 	if (adev->pm.pcie_mlw_mask == 0) {
6332 		/* asic caps */
6333 		if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6334 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK;
6335 		} else {
6336 			switch (link_width) {
6337 			case PCIE_LNK_X32:
6338 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 |
6339 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6340 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6341 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6342 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6343 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6344 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6345 				break;
6346 			case PCIE_LNK_X16:
6347 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6348 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6349 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6350 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6351 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6352 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6353 				break;
6354 			case PCIE_LNK_X12:
6355 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6356 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6357 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6358 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6359 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6360 				break;
6361 			case PCIE_LNK_X8:
6362 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6363 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6364 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6365 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6366 				break;
6367 			case PCIE_LNK_X4:
6368 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6369 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6370 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6371 				break;
6372 			case PCIE_LNK_X2:
6373 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6374 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6375 				break;
6376 			case PCIE_LNK_X1:
6377 				adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1;
6378 				break;
6379 			default:
6380 				break;
6381 			}
6382 		}
6383 		/* platform caps */
6384 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6385 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
6386 		} else {
6387 			switch (platform_link_width) {
6388 			case PCIE_LNK_X32:
6389 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
6390 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6391 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6392 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6393 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6394 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6395 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6396 				break;
6397 			case PCIE_LNK_X16:
6398 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6399 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6400 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6401 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6402 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6403 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6404 				break;
6405 			case PCIE_LNK_X12:
6406 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6407 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6408 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6409 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6410 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6411 				break;
6412 			case PCIE_LNK_X8:
6413 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6414 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6415 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6416 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6417 				break;
6418 			case PCIE_LNK_X4:
6419 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6420 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6421 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6422 				break;
6423 			case PCIE_LNK_X2:
6424 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6425 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6426 				break;
6427 			case PCIE_LNK_X1:
6428 				adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
6429 				break;
6430 			default:
6431 				break;
6432 			}
6433 		}
6434 	}
6435 }
6436 
6437 /**
6438  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6439  *
6440  * @adev: amdgpu_device pointer
6441  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6442  *
6443  * Return true if @peer_adev can access (DMA) @adev through the PCIe
6444  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6445  * @peer_adev.
6446  */
6447 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6448 				      struct amdgpu_device *peer_adev)
6449 {
6450 #ifdef CONFIG_HSA_AMD_P2P
6451 	bool p2p_access =
6452 		!adev->gmc.xgmi.connected_to_cpu &&
6453 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6454 	if (!p2p_access)
6455 		dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",
6456 			pci_name(peer_adev->pdev));
6457 
6458 	bool is_large_bar = adev->gmc.visible_vram_size &&
6459 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size;
6460 	bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev);
6461 
6462 	if (!p2p_addressable) {
6463 		uint64_t address_mask = peer_adev->dev->dma_mask ?
6464 			~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6465 		resource_size_t aper_limit =
6466 			adev->gmc.aper_base + adev->gmc.aper_size - 1;
6467 
6468 		p2p_addressable = !(adev->gmc.aper_base & address_mask ||
6469 				     aper_limit & address_mask);
6470 	}
6471 	return pcie_p2p && is_large_bar && p2p_access && p2p_addressable;
6472 #else
6473 	return false;
6474 #endif
6475 }
6476 
6477 int amdgpu_device_baco_enter(struct drm_device *dev)
6478 {
6479 	struct amdgpu_device *adev = drm_to_adev(dev);
6480 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6481 
6482 	if (!amdgpu_device_supports_baco(dev))
6483 		return -ENOTSUPP;
6484 
6485 	if (ras && adev->ras_enabled &&
6486 	    adev->nbio.funcs->enable_doorbell_interrupt)
6487 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6488 
6489 	return amdgpu_dpm_baco_enter(adev);
6490 }
6491 
6492 int amdgpu_device_baco_exit(struct drm_device *dev)
6493 {
6494 	struct amdgpu_device *adev = drm_to_adev(dev);
6495 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6496 	int ret = 0;
6497 
6498 	if (!amdgpu_device_supports_baco(dev))
6499 		return -ENOTSUPP;
6500 
6501 	ret = amdgpu_dpm_baco_exit(adev);
6502 	if (ret)
6503 		return ret;
6504 
6505 	if (ras && adev->ras_enabled &&
6506 	    adev->nbio.funcs->enable_doorbell_interrupt)
6507 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6508 
6509 	if (amdgpu_passthrough(adev) && adev->nbio.funcs &&
6510 	    adev->nbio.funcs->clear_doorbell_interrupt)
6511 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
6512 
6513 	return 0;
6514 }
6515 
6516 /**
6517  * amdgpu_pci_error_detected - Called when a PCI error is detected.
6518  * @pdev: PCI device struct
6519  * @state: PCI channel state
6520  *
6521  * Description: Called when a PCI error is detected.
6522  *
6523  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6524  */
6525 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6526 {
6527 	struct drm_device *dev = pci_get_drvdata(pdev);
6528 	struct amdgpu_device *adev = drm_to_adev(dev);
6529 	int i;
6530 
6531 	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6532 
6533 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
6534 		DRM_WARN("No support for XGMI hive yet...");
6535 		return PCI_ERS_RESULT_DISCONNECT;
6536 	}
6537 
6538 	adev->pci_channel_state = state;
6539 
6540 	switch (state) {
6541 	case pci_channel_io_normal:
6542 		return PCI_ERS_RESULT_CAN_RECOVER;
6543 	/* Fatal error, prepare for slot reset */
6544 	case pci_channel_io_frozen:
6545 		/*
6546 		 * Locking adev->reset_domain->sem will prevent any external access
6547 		 * to GPU during PCI error recovery
6548 		 */
6549 		amdgpu_device_lock_reset_domain(adev->reset_domain);
6550 		amdgpu_device_set_mp1_state(adev);
6551 
6552 		/*
6553 		 * Block any work scheduling as we do for regular GPU reset
6554 		 * for the duration of the recovery
6555 		 */
6556 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6557 			struct amdgpu_ring *ring = adev->rings[i];
6558 
6559 			if (!amdgpu_ring_sched_ready(ring))
6560 				continue;
6561 
6562 			drm_sched_stop(&ring->sched, NULL);
6563 		}
6564 		atomic_inc(&adev->gpu_reset_counter);
6565 		return PCI_ERS_RESULT_NEED_RESET;
6566 	case pci_channel_io_perm_failure:
6567 		/* Permanent error, prepare for device removal */
6568 		return PCI_ERS_RESULT_DISCONNECT;
6569 	}
6570 
6571 	return PCI_ERS_RESULT_NEED_RESET;
6572 }
6573 
6574 /**
6575  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6576  * @pdev: pointer to PCI device
6577  */
6578 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6579 {
6580 
6581 	DRM_INFO("PCI error: mmio enabled callback!!\n");
6582 
6583 	/* TODO - dump whatever for debugging purposes */
6584 
6585 	/* This called only if amdgpu_pci_error_detected returns
6586 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6587 	 * works, no need to reset slot.
6588 	 */
6589 
6590 	return PCI_ERS_RESULT_RECOVERED;
6591 }
6592 
6593 /**
6594  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6595  * @pdev: PCI device struct
6596  *
6597  * Description: This routine is called by the pci error recovery
6598  * code after the PCI slot has been reset, just before we
6599  * should resume normal operations.
6600  */
6601 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6602 {
6603 	struct drm_device *dev = pci_get_drvdata(pdev);
6604 	struct amdgpu_device *adev = drm_to_adev(dev);
6605 	int r, i;
6606 	struct amdgpu_reset_context reset_context;
6607 	u32 memsize;
6608 	struct list_head device_list;
6609 
6610 	/* PCI error slot reset should be skipped During RAS recovery */
6611 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
6612 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) &&
6613 	    amdgpu_ras_in_recovery(adev))
6614 		return PCI_ERS_RESULT_RECOVERED;
6615 
6616 	DRM_INFO("PCI error: slot reset callback!!\n");
6617 
6618 	memset(&reset_context, 0, sizeof(reset_context));
6619 
6620 	INIT_LIST_HEAD(&device_list);
6621 	list_add_tail(&adev->reset_list, &device_list);
6622 
6623 	/* wait for asic to come out of reset */
6624 	msleep(500);
6625 
6626 	/* Restore PCI confspace */
6627 	amdgpu_device_load_pci_state(pdev);
6628 
6629 	/* confirm  ASIC came out of reset */
6630 	for (i = 0; i < adev->usec_timeout; i++) {
6631 		memsize = amdgpu_asic_get_config_memsize(adev);
6632 
6633 		if (memsize != 0xffffffff)
6634 			break;
6635 		udelay(1);
6636 	}
6637 	if (memsize == 0xffffffff) {
6638 		r = -ETIME;
6639 		goto out;
6640 	}
6641 
6642 	reset_context.method = AMD_RESET_METHOD_NONE;
6643 	reset_context.reset_req_dev = adev;
6644 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6645 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6646 
6647 	adev->no_hw_access = true;
6648 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6649 	adev->no_hw_access = false;
6650 	if (r)
6651 		goto out;
6652 
6653 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
6654 
6655 out:
6656 	if (!r) {
6657 		if (amdgpu_device_cache_pci_state(adev->pdev))
6658 			pci_restore_state(adev->pdev);
6659 
6660 		DRM_INFO("PCIe error recovery succeeded\n");
6661 	} else {
6662 		DRM_ERROR("PCIe error recovery failed, err:%d", r);
6663 		amdgpu_device_unset_mp1_state(adev);
6664 		amdgpu_device_unlock_reset_domain(adev->reset_domain);
6665 	}
6666 
6667 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6668 }
6669 
6670 /**
6671  * amdgpu_pci_resume() - resume normal ops after PCI reset
6672  * @pdev: pointer to PCI device
6673  *
6674  * Called when the error recovery driver tells us that its
6675  * OK to resume normal operation.
6676  */
6677 void amdgpu_pci_resume(struct pci_dev *pdev)
6678 {
6679 	struct drm_device *dev = pci_get_drvdata(pdev);
6680 	struct amdgpu_device *adev = drm_to_adev(dev);
6681 	int i;
6682 
6683 
6684 	DRM_INFO("PCI error: resume callback!!\n");
6685 
6686 	/* Only continue execution for the case of pci_channel_io_frozen */
6687 	if (adev->pci_channel_state != pci_channel_io_frozen)
6688 		return;
6689 
6690 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6691 		struct amdgpu_ring *ring = adev->rings[i];
6692 
6693 		if (!amdgpu_ring_sched_ready(ring))
6694 			continue;
6695 
6696 		drm_sched_start(&ring->sched, 0);
6697 	}
6698 
6699 	amdgpu_device_unset_mp1_state(adev);
6700 	amdgpu_device_unlock_reset_domain(adev->reset_domain);
6701 }
6702 
6703 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6704 {
6705 	struct drm_device *dev = pci_get_drvdata(pdev);
6706 	struct amdgpu_device *adev = drm_to_adev(dev);
6707 	int r;
6708 
6709 	if (amdgpu_sriov_vf(adev))
6710 		return false;
6711 
6712 	r = pci_save_state(pdev);
6713 	if (!r) {
6714 		kfree(adev->pci_state);
6715 
6716 		adev->pci_state = pci_store_saved_state(pdev);
6717 
6718 		if (!adev->pci_state) {
6719 			DRM_ERROR("Failed to store PCI saved state");
6720 			return false;
6721 		}
6722 	} else {
6723 		DRM_WARN("Failed to save PCI state, err:%d\n", r);
6724 		return false;
6725 	}
6726 
6727 	return true;
6728 }
6729 
6730 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6731 {
6732 	struct drm_device *dev = pci_get_drvdata(pdev);
6733 	struct amdgpu_device *adev = drm_to_adev(dev);
6734 	int r;
6735 
6736 	if (!adev->pci_state)
6737 		return false;
6738 
6739 	r = pci_load_saved_state(pdev, adev->pci_state);
6740 
6741 	if (!r) {
6742 		pci_restore_state(pdev);
6743 	} else {
6744 		DRM_WARN("Failed to load PCI state, err:%d\n", r);
6745 		return false;
6746 	}
6747 
6748 	return true;
6749 }
6750 
6751 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6752 		struct amdgpu_ring *ring)
6753 {
6754 #ifdef CONFIG_X86_64
6755 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6756 		return;
6757 #endif
6758 	if (adev->gmc.xgmi.connected_to_cpu)
6759 		return;
6760 
6761 	if (ring && ring->funcs->emit_hdp_flush)
6762 		amdgpu_ring_emit_hdp_flush(ring);
6763 	else
6764 		amdgpu_asic_flush_hdp(adev, ring);
6765 }
6766 
6767 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6768 		struct amdgpu_ring *ring)
6769 {
6770 #ifdef CONFIG_X86_64
6771 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6772 		return;
6773 #endif
6774 	if (adev->gmc.xgmi.connected_to_cpu)
6775 		return;
6776 
6777 	amdgpu_asic_invalidate_hdp(adev, ring);
6778 }
6779 
6780 int amdgpu_in_reset(struct amdgpu_device *adev)
6781 {
6782 	return atomic_read(&adev->reset_domain->in_gpu_reset);
6783 }
6784 
6785 /**
6786  * amdgpu_device_halt() - bring hardware to some kind of halt state
6787  *
6788  * @adev: amdgpu_device pointer
6789  *
6790  * Bring hardware to some kind of halt state so that no one can touch it
6791  * any more. It will help to maintain error context when error occurred.
6792  * Compare to a simple hang, the system will keep stable at least for SSH
6793  * access. Then it should be trivial to inspect the hardware state and
6794  * see what's going on. Implemented as following:
6795  *
6796  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6797  *    clears all CPU mappings to device, disallows remappings through page faults
6798  * 2. amdgpu_irq_disable_all() disables all interrupts
6799  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6800  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6801  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6802  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6803  *    flush any in flight DMA operations
6804  */
6805 void amdgpu_device_halt(struct amdgpu_device *adev)
6806 {
6807 	struct pci_dev *pdev = adev->pdev;
6808 	struct drm_device *ddev = adev_to_drm(adev);
6809 
6810 	amdgpu_xcp_dev_unplug(adev);
6811 	drm_dev_unplug(ddev);
6812 
6813 	amdgpu_irq_disable_all(adev);
6814 
6815 	amdgpu_fence_driver_hw_fini(adev);
6816 
6817 	adev->no_hw_access = true;
6818 
6819 	amdgpu_device_unmap_mmio(adev);
6820 
6821 	pci_disable_device(pdev);
6822 	pci_wait_for_pending_transaction(pdev);
6823 }
6824 
6825 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6826 				u32 reg)
6827 {
6828 	unsigned long flags, address, data;
6829 	u32 r;
6830 
6831 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6832 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6833 
6834 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6835 	WREG32(address, reg * 4);
6836 	(void)RREG32(address);
6837 	r = RREG32(data);
6838 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6839 	return r;
6840 }
6841 
6842 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6843 				u32 reg, u32 v)
6844 {
6845 	unsigned long flags, address, data;
6846 
6847 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6848 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6849 
6850 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6851 	WREG32(address, reg * 4);
6852 	(void)RREG32(address);
6853 	WREG32(data, v);
6854 	(void)RREG32(data);
6855 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6856 }
6857 
6858 /**
6859  * amdgpu_device_get_gang - return a reference to the current gang
6860  * @adev: amdgpu_device pointer
6861  *
6862  * Returns: A new reference to the current gang leader.
6863  */
6864 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)
6865 {
6866 	struct dma_fence *fence;
6867 
6868 	rcu_read_lock();
6869 	fence = dma_fence_get_rcu_safe(&adev->gang_submit);
6870 	rcu_read_unlock();
6871 	return fence;
6872 }
6873 
6874 /**
6875  * amdgpu_device_switch_gang - switch to a new gang
6876  * @adev: amdgpu_device pointer
6877  * @gang: the gang to switch to
6878  *
6879  * Try to switch to a new gang.
6880  * Returns: NULL if we switched to the new gang or a reference to the current
6881  * gang leader.
6882  */
6883 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6884 					    struct dma_fence *gang)
6885 {
6886 	struct dma_fence *old = NULL;
6887 
6888 	do {
6889 		dma_fence_put(old);
6890 		old = amdgpu_device_get_gang(adev);
6891 		if (old == gang)
6892 			break;
6893 
6894 		if (!dma_fence_is_signaled(old))
6895 			return old;
6896 
6897 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6898 			 old, gang) != old);
6899 
6900 	dma_fence_put(old);
6901 	return NULL;
6902 }
6903 
6904 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6905 {
6906 	switch (adev->asic_type) {
6907 #ifdef CONFIG_DRM_AMDGPU_SI
6908 	case CHIP_HAINAN:
6909 #endif
6910 	case CHIP_TOPAZ:
6911 		/* chips with no display hardware */
6912 		return false;
6913 #ifdef CONFIG_DRM_AMDGPU_SI
6914 	case CHIP_TAHITI:
6915 	case CHIP_PITCAIRN:
6916 	case CHIP_VERDE:
6917 	case CHIP_OLAND:
6918 #endif
6919 #ifdef CONFIG_DRM_AMDGPU_CIK
6920 	case CHIP_BONAIRE:
6921 	case CHIP_HAWAII:
6922 	case CHIP_KAVERI:
6923 	case CHIP_KABINI:
6924 	case CHIP_MULLINS:
6925 #endif
6926 	case CHIP_TONGA:
6927 	case CHIP_FIJI:
6928 	case CHIP_POLARIS10:
6929 	case CHIP_POLARIS11:
6930 	case CHIP_POLARIS12:
6931 	case CHIP_VEGAM:
6932 	case CHIP_CARRIZO:
6933 	case CHIP_STONEY:
6934 		/* chips with display hardware */
6935 		return true;
6936 	default:
6937 		/* IP discovery */
6938 		if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6939 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6940 			return false;
6941 		return true;
6942 	}
6943 }
6944 
6945 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6946 		uint32_t inst, uint32_t reg_addr, char reg_name[],
6947 		uint32_t expected_value, uint32_t mask)
6948 {
6949 	uint32_t ret = 0;
6950 	uint32_t old_ = 0;
6951 	uint32_t tmp_ = RREG32(reg_addr);
6952 	uint32_t loop = adev->usec_timeout;
6953 
6954 	while ((tmp_ & (mask)) != (expected_value)) {
6955 		if (old_ != tmp_) {
6956 			loop = adev->usec_timeout;
6957 			old_ = tmp_;
6958 		} else
6959 			udelay(1);
6960 		tmp_ = RREG32(reg_addr);
6961 		loop--;
6962 		if (!loop) {
6963 			DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6964 				  inst, reg_name, (uint32_t)expected_value,
6965 				  (uint32_t)(tmp_ & (mask)));
6966 			ret = -ETIMEDOUT;
6967 			break;
6968 		}
6969 	}
6970 	return ret;
6971 }
6972 
6973 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
6974 {
6975 	ssize_t size = 0;
6976 
6977 	if (!ring || !ring->adev)
6978 		return size;
6979 
6980 	if (amdgpu_device_should_recover_gpu(ring->adev))
6981 		size |= AMDGPU_RESET_TYPE_FULL;
6982 
6983 	if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
6984 	    !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
6985 		size |= AMDGPU_RESET_TYPE_SOFT_RESET;
6986 
6987 	return size;
6988 }
6989 
6990 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
6991 {
6992 	ssize_t size = 0;
6993 
6994 	if (supported_reset == 0) {
6995 		size += sysfs_emit_at(buf, size, "unsupported");
6996 		size += sysfs_emit_at(buf, size, "\n");
6997 		return size;
6998 
6999 	}
7000 
7001 	if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET)
7002 		size += sysfs_emit_at(buf, size, "soft ");
7003 
7004 	if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
7005 		size += sysfs_emit_at(buf, size, "queue ");
7006 
7007 	if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE)
7008 		size += sysfs_emit_at(buf, size, "pipe ");
7009 
7010 	if (supported_reset & AMDGPU_RESET_TYPE_FULL)
7011 		size += sysfs_emit_at(buf, size, "full ");
7012 
7013 	size += sysfs_emit_at(buf, size, "\n");
7014 	return size;
7015 }
7016