xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c (revision 4b660dbd9ee2059850fd30e0df420ca7a38a1856)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/power_supply.h>
29 #include <linux/kthread.h>
30 #include <linux/module.h>
31 #include <linux/console.h>
32 #include <linux/slab.h>
33 #include <linux/iommu.h>
34 #include <linux/pci.h>
35 #include <linux/pci-p2pdma.h>
36 #include <linux/apple-gmux.h>
37 
38 #include <drm/drm_aperture.h>
39 #include <drm/drm_atomic_helper.h>
40 #include <drm/drm_crtc_helper.h>
41 #include <drm/drm_fb_helper.h>
42 #include <drm/drm_probe_helper.h>
43 #include <drm/amdgpu_drm.h>
44 #include <linux/device.h>
45 #include <linux/vgaarb.h>
46 #include <linux/vga_switcheroo.h>
47 #include <linux/efi.h>
48 #include "amdgpu.h"
49 #include "amdgpu_trace.h"
50 #include "amdgpu_i2c.h"
51 #include "atom.h"
52 #include "amdgpu_atombios.h"
53 #include "amdgpu_atomfirmware.h"
54 #include "amd_pcie.h"
55 #ifdef CONFIG_DRM_AMDGPU_SI
56 #include "si.h"
57 #endif
58 #ifdef CONFIG_DRM_AMDGPU_CIK
59 #include "cik.h"
60 #endif
61 #include "vi.h"
62 #include "soc15.h"
63 #include "nv.h"
64 #include "bif/bif_4_1_d.h"
65 #include <linux/firmware.h>
66 #include "amdgpu_vf_error.h"
67 
68 #include "amdgpu_amdkfd.h"
69 #include "amdgpu_pm.h"
70 
71 #include "amdgpu_xgmi.h"
72 #include "amdgpu_ras.h"
73 #include "amdgpu_pmu.h"
74 #include "amdgpu_fru_eeprom.h"
75 #include "amdgpu_reset.h"
76 #include "amdgpu_virt.h"
77 
78 #include <linux/suspend.h>
79 #include <drm/task_barrier.h>
80 #include <linux/pm_runtime.h>
81 
82 #include <drm/drm_drv.h>
83 
84 #if IS_ENABLED(CONFIG_X86)
85 #include <asm/intel-family.h>
86 #endif
87 
88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
95 
96 #define AMDGPU_RESUME_MS		2000
97 #define AMDGPU_MAX_RETRY_LIMIT		2
98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
99 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
100 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
101 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
102 
103 static const struct drm_driver amdgpu_kms_driver;
104 
105 const char *amdgpu_asic_name[] = {
106 	"TAHITI",
107 	"PITCAIRN",
108 	"VERDE",
109 	"OLAND",
110 	"HAINAN",
111 	"BONAIRE",
112 	"KAVERI",
113 	"KABINI",
114 	"HAWAII",
115 	"MULLINS",
116 	"TOPAZ",
117 	"TONGA",
118 	"FIJI",
119 	"CARRIZO",
120 	"STONEY",
121 	"POLARIS10",
122 	"POLARIS11",
123 	"POLARIS12",
124 	"VEGAM",
125 	"VEGA10",
126 	"VEGA12",
127 	"VEGA20",
128 	"RAVEN",
129 	"ARCTURUS",
130 	"RENOIR",
131 	"ALDEBARAN",
132 	"NAVI10",
133 	"CYAN_SKILLFISH",
134 	"NAVI14",
135 	"NAVI12",
136 	"SIENNA_CICHLID",
137 	"NAVY_FLOUNDER",
138 	"VANGOGH",
139 	"DIMGREY_CAVEFISH",
140 	"BEIGE_GOBY",
141 	"YELLOW_CARP",
142 	"IP DISCOVERY",
143 	"LAST",
144 };
145 
146 /**
147  * DOC: pcie_replay_count
148  *
149  * The amdgpu driver provides a sysfs API for reporting the total number
150  * of PCIe replays (NAKs)
151  * The file pcie_replay_count is used for this and returns the total
152  * number of replays as a sum of the NAKs generated and NAKs received
153  */
154 
155 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
156 		struct device_attribute *attr, char *buf)
157 {
158 	struct drm_device *ddev = dev_get_drvdata(dev);
159 	struct amdgpu_device *adev = drm_to_adev(ddev);
160 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
161 
162 	return sysfs_emit(buf, "%llu\n", cnt);
163 }
164 
165 static DEVICE_ATTR(pcie_replay_count, 0444,
166 		amdgpu_device_get_pcie_replay_count, NULL);
167 
168 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
169 					  struct bin_attribute *attr, char *buf,
170 					  loff_t ppos, size_t count)
171 {
172 	struct device *dev = kobj_to_dev(kobj);
173 	struct drm_device *ddev = dev_get_drvdata(dev);
174 	struct amdgpu_device *adev = drm_to_adev(ddev);
175 	ssize_t bytes_read;
176 
177 	switch (ppos) {
178 	case AMDGPU_SYS_REG_STATE_XGMI:
179 		bytes_read = amdgpu_asic_get_reg_state(
180 			adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
181 		break;
182 	case AMDGPU_SYS_REG_STATE_WAFL:
183 		bytes_read = amdgpu_asic_get_reg_state(
184 			adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
185 		break;
186 	case AMDGPU_SYS_REG_STATE_PCIE:
187 		bytes_read = amdgpu_asic_get_reg_state(
188 			adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
189 		break;
190 	case AMDGPU_SYS_REG_STATE_USR:
191 		bytes_read = amdgpu_asic_get_reg_state(
192 			adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
193 		break;
194 	case AMDGPU_SYS_REG_STATE_USR_1:
195 		bytes_read = amdgpu_asic_get_reg_state(
196 			adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
197 		break;
198 	default:
199 		return -EINVAL;
200 	}
201 
202 	return bytes_read;
203 }
204 
205 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
206 	 AMDGPU_SYS_REG_STATE_END);
207 
208 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
209 {
210 	int ret;
211 
212 	if (!amdgpu_asic_get_reg_state_supported(adev))
213 		return 0;
214 
215 	ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
216 
217 	return ret;
218 }
219 
220 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
221 {
222 	if (!amdgpu_asic_get_reg_state_supported(adev))
223 		return;
224 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
225 }
226 
227 /**
228  * DOC: board_info
229  *
230  * The amdgpu driver provides a sysfs API for giving board related information.
231  * It provides the form factor information in the format
232  *
233  *   type : form factor
234  *
235  * Possible form factor values
236  *
237  * - "cem"		- PCIE CEM card
238  * - "oam"		- Open Compute Accelerator Module
239  * - "unknown"	- Not known
240  *
241  */
242 
243 static ssize_t amdgpu_device_get_board_info(struct device *dev,
244 					    struct device_attribute *attr,
245 					    char *buf)
246 {
247 	struct drm_device *ddev = dev_get_drvdata(dev);
248 	struct amdgpu_device *adev = drm_to_adev(ddev);
249 	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
250 	const char *pkg;
251 
252 	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
253 		pkg_type = adev->smuio.funcs->get_pkg_type(adev);
254 
255 	switch (pkg_type) {
256 	case AMDGPU_PKG_TYPE_CEM:
257 		pkg = "cem";
258 		break;
259 	case AMDGPU_PKG_TYPE_OAM:
260 		pkg = "oam";
261 		break;
262 	default:
263 		pkg = "unknown";
264 		break;
265 	}
266 
267 	return sysfs_emit(buf, "%s : %s\n", "type", pkg);
268 }
269 
270 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
271 
272 static struct attribute *amdgpu_board_attrs[] = {
273 	&dev_attr_board_info.attr,
274 	NULL,
275 };
276 
277 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
278 					     struct attribute *attr, int n)
279 {
280 	struct device *dev = kobj_to_dev(kobj);
281 	struct drm_device *ddev = dev_get_drvdata(dev);
282 	struct amdgpu_device *adev = drm_to_adev(ddev);
283 
284 	if (adev->flags & AMD_IS_APU)
285 		return 0;
286 
287 	return attr->mode;
288 }
289 
290 static const struct attribute_group amdgpu_board_attrs_group = {
291 	.attrs = amdgpu_board_attrs,
292 	.is_visible = amdgpu_board_attrs_is_visible
293 };
294 
295 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
296 
297 
298 /**
299  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
300  *
301  * @dev: drm_device pointer
302  *
303  * Returns true if the device is a dGPU with ATPX power control,
304  * otherwise return false.
305  */
306 bool amdgpu_device_supports_px(struct drm_device *dev)
307 {
308 	struct amdgpu_device *adev = drm_to_adev(dev);
309 
310 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
311 		return true;
312 	return false;
313 }
314 
315 /**
316  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
317  *
318  * @dev: drm_device pointer
319  *
320  * Returns true if the device is a dGPU with ACPI power control,
321  * otherwise return false.
322  */
323 bool amdgpu_device_supports_boco(struct drm_device *dev)
324 {
325 	struct amdgpu_device *adev = drm_to_adev(dev);
326 
327 	if (adev->has_pr3 ||
328 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
329 		return true;
330 	return false;
331 }
332 
333 /**
334  * amdgpu_device_supports_baco - Does the device support BACO
335  *
336  * @dev: drm_device pointer
337  *
338  * Returns true if the device supporte BACO,
339  * otherwise return false.
340  */
341 bool amdgpu_device_supports_baco(struct drm_device *dev)
342 {
343 	struct amdgpu_device *adev = drm_to_adev(dev);
344 
345 	return amdgpu_asic_supports_baco(adev);
346 }
347 
348 /**
349  * amdgpu_device_supports_smart_shift - Is the device dGPU with
350  * smart shift support
351  *
352  * @dev: drm_device pointer
353  *
354  * Returns true if the device is a dGPU with Smart Shift support,
355  * otherwise returns false.
356  */
357 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
358 {
359 	return (amdgpu_device_supports_boco(dev) &&
360 		amdgpu_acpi_is_power_shift_control_supported());
361 }
362 
363 /*
364  * VRAM access helper functions
365  */
366 
367 /**
368  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
369  *
370  * @adev: amdgpu_device pointer
371  * @pos: offset of the buffer in vram
372  * @buf: virtual address of the buffer in system memory
373  * @size: read/write size, sizeof(@buf) must > @size
374  * @write: true - write to vram, otherwise - read from vram
375  */
376 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
377 			     void *buf, size_t size, bool write)
378 {
379 	unsigned long flags;
380 	uint32_t hi = ~0, tmp = 0;
381 	uint32_t *data = buf;
382 	uint64_t last;
383 	int idx;
384 
385 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
386 		return;
387 
388 	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
389 
390 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
391 	for (last = pos + size; pos < last; pos += 4) {
392 		tmp = pos >> 31;
393 
394 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
395 		if (tmp != hi) {
396 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
397 			hi = tmp;
398 		}
399 		if (write)
400 			WREG32_NO_KIQ(mmMM_DATA, *data++);
401 		else
402 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
403 	}
404 
405 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
406 	drm_dev_exit(idx);
407 }
408 
409 /**
410  * amdgpu_device_aper_access - access vram by vram aperature
411  *
412  * @adev: amdgpu_device pointer
413  * @pos: offset of the buffer in vram
414  * @buf: virtual address of the buffer in system memory
415  * @size: read/write size, sizeof(@buf) must > @size
416  * @write: true - write to vram, otherwise - read from vram
417  *
418  * The return value means how many bytes have been transferred.
419  */
420 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
421 				 void *buf, size_t size, bool write)
422 {
423 #ifdef CONFIG_64BIT
424 	void __iomem *addr;
425 	size_t count = 0;
426 	uint64_t last;
427 
428 	if (!adev->mman.aper_base_kaddr)
429 		return 0;
430 
431 	last = min(pos + size, adev->gmc.visible_vram_size);
432 	if (last > pos) {
433 		addr = adev->mman.aper_base_kaddr + pos;
434 		count = last - pos;
435 
436 		if (write) {
437 			memcpy_toio(addr, buf, count);
438 			/* Make sure HDP write cache flush happens without any reordering
439 			 * after the system memory contents are sent over PCIe device
440 			 */
441 			mb();
442 			amdgpu_device_flush_hdp(adev, NULL);
443 		} else {
444 			amdgpu_device_invalidate_hdp(adev, NULL);
445 			/* Make sure HDP read cache is invalidated before issuing a read
446 			 * to the PCIe device
447 			 */
448 			mb();
449 			memcpy_fromio(buf, addr, count);
450 		}
451 
452 	}
453 
454 	return count;
455 #else
456 	return 0;
457 #endif
458 }
459 
460 /**
461  * amdgpu_device_vram_access - read/write a buffer in vram
462  *
463  * @adev: amdgpu_device pointer
464  * @pos: offset of the buffer in vram
465  * @buf: virtual address of the buffer in system memory
466  * @size: read/write size, sizeof(@buf) must > @size
467  * @write: true - write to vram, otherwise - read from vram
468  */
469 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
470 			       void *buf, size_t size, bool write)
471 {
472 	size_t count;
473 
474 	/* try to using vram apreature to access vram first */
475 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
476 	size -= count;
477 	if (size) {
478 		/* using MM to access rest vram */
479 		pos += count;
480 		buf += count;
481 		amdgpu_device_mm_access(adev, pos, buf, size, write);
482 	}
483 }
484 
485 /*
486  * register access helper functions.
487  */
488 
489 /* Check if hw access should be skipped because of hotplug or device error */
490 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
491 {
492 	if (adev->no_hw_access)
493 		return true;
494 
495 #ifdef CONFIG_LOCKDEP
496 	/*
497 	 * This is a bit complicated to understand, so worth a comment. What we assert
498 	 * here is that the GPU reset is not running on another thread in parallel.
499 	 *
500 	 * For this we trylock the read side of the reset semaphore, if that succeeds
501 	 * we know that the reset is not running in paralell.
502 	 *
503 	 * If the trylock fails we assert that we are either already holding the read
504 	 * side of the lock or are the reset thread itself and hold the write side of
505 	 * the lock.
506 	 */
507 	if (in_task()) {
508 		if (down_read_trylock(&adev->reset_domain->sem))
509 			up_read(&adev->reset_domain->sem);
510 		else
511 			lockdep_assert_held(&adev->reset_domain->sem);
512 	}
513 #endif
514 	return false;
515 }
516 
517 /**
518  * amdgpu_device_rreg - read a memory mapped IO or indirect register
519  *
520  * @adev: amdgpu_device pointer
521  * @reg: dword aligned register offset
522  * @acc_flags: access flags which require special behavior
523  *
524  * Returns the 32 bit value from the offset specified.
525  */
526 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
527 			    uint32_t reg, uint32_t acc_flags)
528 {
529 	uint32_t ret;
530 
531 	if (amdgpu_device_skip_hw_access(adev))
532 		return 0;
533 
534 	if ((reg * 4) < adev->rmmio_size) {
535 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
536 		    amdgpu_sriov_runtime(adev) &&
537 		    down_read_trylock(&adev->reset_domain->sem)) {
538 			ret = amdgpu_kiq_rreg(adev, reg, 0);
539 			up_read(&adev->reset_domain->sem);
540 		} else {
541 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
542 		}
543 	} else {
544 		ret = adev->pcie_rreg(adev, reg * 4);
545 	}
546 
547 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
548 
549 	return ret;
550 }
551 
552 /*
553  * MMIO register read with bytes helper functions
554  * @offset:bytes offset from MMIO start
555  */
556 
557 /**
558  * amdgpu_mm_rreg8 - read a memory mapped IO register
559  *
560  * @adev: amdgpu_device pointer
561  * @offset: byte aligned register offset
562  *
563  * Returns the 8 bit value from the offset specified.
564  */
565 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
566 {
567 	if (amdgpu_device_skip_hw_access(adev))
568 		return 0;
569 
570 	if (offset < adev->rmmio_size)
571 		return (readb(adev->rmmio + offset));
572 	BUG();
573 }
574 
575 
576 /**
577  * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
578  *
579  * @adev: amdgpu_device pointer
580  * @reg: dword aligned register offset
581  * @acc_flags: access flags which require special behavior
582  * @xcc_id: xcc accelerated compute core id
583  *
584  * Returns the 32 bit value from the offset specified.
585  */
586 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
587 				uint32_t reg, uint32_t acc_flags,
588 				uint32_t xcc_id)
589 {
590 	uint32_t ret, rlcg_flag;
591 
592 	if (amdgpu_device_skip_hw_access(adev))
593 		return 0;
594 
595 	if ((reg * 4) < adev->rmmio_size) {
596 		if (amdgpu_sriov_vf(adev) &&
597 		    !amdgpu_sriov_runtime(adev) &&
598 		    adev->gfx.rlc.rlcg_reg_access_supported &&
599 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
600 							 GC_HWIP, false,
601 							 &rlcg_flag)) {
602 			ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, xcc_id);
603 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
604 		    amdgpu_sriov_runtime(adev) &&
605 		    down_read_trylock(&adev->reset_domain->sem)) {
606 			ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
607 			up_read(&adev->reset_domain->sem);
608 		} else {
609 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
610 		}
611 	} else {
612 		ret = adev->pcie_rreg(adev, reg * 4);
613 	}
614 
615 	return ret;
616 }
617 
618 /*
619  * MMIO register write with bytes helper functions
620  * @offset:bytes offset from MMIO start
621  * @value: the value want to be written to the register
622  */
623 
624 /**
625  * amdgpu_mm_wreg8 - read a memory mapped IO register
626  *
627  * @adev: amdgpu_device pointer
628  * @offset: byte aligned register offset
629  * @value: 8 bit value to write
630  *
631  * Writes the value specified to the offset specified.
632  */
633 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
634 {
635 	if (amdgpu_device_skip_hw_access(adev))
636 		return;
637 
638 	if (offset < adev->rmmio_size)
639 		writeb(value, adev->rmmio + offset);
640 	else
641 		BUG();
642 }
643 
644 /**
645  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
646  *
647  * @adev: amdgpu_device pointer
648  * @reg: dword aligned register offset
649  * @v: 32 bit value to write to the register
650  * @acc_flags: access flags which require special behavior
651  *
652  * Writes the value specified to the offset specified.
653  */
654 void amdgpu_device_wreg(struct amdgpu_device *adev,
655 			uint32_t reg, uint32_t v,
656 			uint32_t acc_flags)
657 {
658 	if (amdgpu_device_skip_hw_access(adev))
659 		return;
660 
661 	if ((reg * 4) < adev->rmmio_size) {
662 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
663 		    amdgpu_sriov_runtime(adev) &&
664 		    down_read_trylock(&adev->reset_domain->sem)) {
665 			amdgpu_kiq_wreg(adev, reg, v, 0);
666 			up_read(&adev->reset_domain->sem);
667 		} else {
668 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
669 		}
670 	} else {
671 		adev->pcie_wreg(adev, reg * 4, v);
672 	}
673 
674 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
675 }
676 
677 /**
678  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
679  *
680  * @adev: amdgpu_device pointer
681  * @reg: mmio/rlc register
682  * @v: value to write
683  * @xcc_id: xcc accelerated compute core id
684  *
685  * this function is invoked only for the debugfs register access
686  */
687 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
688 			     uint32_t reg, uint32_t v,
689 			     uint32_t xcc_id)
690 {
691 	if (amdgpu_device_skip_hw_access(adev))
692 		return;
693 
694 	if (amdgpu_sriov_fullaccess(adev) &&
695 	    adev->gfx.rlc.funcs &&
696 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
697 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
698 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
699 	} else if ((reg * 4) >= adev->rmmio_size) {
700 		adev->pcie_wreg(adev, reg * 4, v);
701 	} else {
702 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
703 	}
704 }
705 
706 /**
707  * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
708  *
709  * @adev: amdgpu_device pointer
710  * @reg: dword aligned register offset
711  * @v: 32 bit value to write to the register
712  * @acc_flags: access flags which require special behavior
713  * @xcc_id: xcc accelerated compute core id
714  *
715  * Writes the value specified to the offset specified.
716  */
717 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
718 			uint32_t reg, uint32_t v,
719 			uint32_t acc_flags, uint32_t xcc_id)
720 {
721 	uint32_t rlcg_flag;
722 
723 	if (amdgpu_device_skip_hw_access(adev))
724 		return;
725 
726 	if ((reg * 4) < adev->rmmio_size) {
727 		if (amdgpu_sriov_vf(adev) &&
728 		    !amdgpu_sriov_runtime(adev) &&
729 		    adev->gfx.rlc.rlcg_reg_access_supported &&
730 		    amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
731 							 GC_HWIP, true,
732 							 &rlcg_flag)) {
733 			amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, xcc_id);
734 		} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
735 		    amdgpu_sriov_runtime(adev) &&
736 		    down_read_trylock(&adev->reset_domain->sem)) {
737 			amdgpu_kiq_wreg(adev, reg, v, xcc_id);
738 			up_read(&adev->reset_domain->sem);
739 		} else {
740 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
741 		}
742 	} else {
743 		adev->pcie_wreg(adev, reg * 4, v);
744 	}
745 }
746 
747 /**
748  * amdgpu_device_indirect_rreg - read an indirect register
749  *
750  * @adev: amdgpu_device pointer
751  * @reg_addr: indirect register address to read from
752  *
753  * Returns the value of indirect register @reg_addr
754  */
755 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
756 				u32 reg_addr)
757 {
758 	unsigned long flags, pcie_index, pcie_data;
759 	void __iomem *pcie_index_offset;
760 	void __iomem *pcie_data_offset;
761 	u32 r;
762 
763 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
764 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
765 
766 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
767 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
768 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
769 
770 	writel(reg_addr, pcie_index_offset);
771 	readl(pcie_index_offset);
772 	r = readl(pcie_data_offset);
773 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
774 
775 	return r;
776 }
777 
778 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
779 				    u64 reg_addr)
780 {
781 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
782 	u32 r;
783 	void __iomem *pcie_index_offset;
784 	void __iomem *pcie_index_hi_offset;
785 	void __iomem *pcie_data_offset;
786 
787 	if (unlikely(!adev->nbio.funcs)) {
788 		pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
789 		pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
790 	} else {
791 		pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
792 		pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
793 	}
794 
795 	if (reg_addr >> 32) {
796 		if (unlikely(!adev->nbio.funcs))
797 			pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
798 		else
799 			pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
800 	} else {
801 		pcie_index_hi = 0;
802 	}
803 
804 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
805 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
806 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
807 	if (pcie_index_hi != 0)
808 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
809 				pcie_index_hi * 4;
810 
811 	writel(reg_addr, pcie_index_offset);
812 	readl(pcie_index_offset);
813 	if (pcie_index_hi != 0) {
814 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
815 		readl(pcie_index_hi_offset);
816 	}
817 	r = readl(pcie_data_offset);
818 
819 	/* clear the high bits */
820 	if (pcie_index_hi != 0) {
821 		writel(0, pcie_index_hi_offset);
822 		readl(pcie_index_hi_offset);
823 	}
824 
825 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
826 
827 	return r;
828 }
829 
830 /**
831  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
832  *
833  * @adev: amdgpu_device pointer
834  * @reg_addr: indirect register address to read from
835  *
836  * Returns the value of indirect register @reg_addr
837  */
838 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
839 				  u32 reg_addr)
840 {
841 	unsigned long flags, pcie_index, pcie_data;
842 	void __iomem *pcie_index_offset;
843 	void __iomem *pcie_data_offset;
844 	u64 r;
845 
846 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
847 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
848 
849 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
850 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
851 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
852 
853 	/* read low 32 bits */
854 	writel(reg_addr, pcie_index_offset);
855 	readl(pcie_index_offset);
856 	r = readl(pcie_data_offset);
857 	/* read high 32 bits */
858 	writel(reg_addr + 4, pcie_index_offset);
859 	readl(pcie_index_offset);
860 	r |= ((u64)readl(pcie_data_offset) << 32);
861 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
862 
863 	return r;
864 }
865 
866 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
867 				  u64 reg_addr)
868 {
869 	unsigned long flags, pcie_index, pcie_data;
870 	unsigned long pcie_index_hi = 0;
871 	void __iomem *pcie_index_offset;
872 	void __iomem *pcie_index_hi_offset;
873 	void __iomem *pcie_data_offset;
874 	u64 r;
875 
876 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
877 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
878 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
879 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
880 
881 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
882 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
883 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
884 	if (pcie_index_hi != 0)
885 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
886 			pcie_index_hi * 4;
887 
888 	/* read low 32 bits */
889 	writel(reg_addr, pcie_index_offset);
890 	readl(pcie_index_offset);
891 	if (pcie_index_hi != 0) {
892 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
893 		readl(pcie_index_hi_offset);
894 	}
895 	r = readl(pcie_data_offset);
896 	/* read high 32 bits */
897 	writel(reg_addr + 4, pcie_index_offset);
898 	readl(pcie_index_offset);
899 	if (pcie_index_hi != 0) {
900 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
901 		readl(pcie_index_hi_offset);
902 	}
903 	r |= ((u64)readl(pcie_data_offset) << 32);
904 
905 	/* clear the high bits */
906 	if (pcie_index_hi != 0) {
907 		writel(0, pcie_index_hi_offset);
908 		readl(pcie_index_hi_offset);
909 	}
910 
911 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
912 
913 	return r;
914 }
915 
916 /**
917  * amdgpu_device_indirect_wreg - write an indirect register address
918  *
919  * @adev: amdgpu_device pointer
920  * @reg_addr: indirect register offset
921  * @reg_data: indirect register data
922  *
923  */
924 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
925 				 u32 reg_addr, u32 reg_data)
926 {
927 	unsigned long flags, pcie_index, pcie_data;
928 	void __iomem *pcie_index_offset;
929 	void __iomem *pcie_data_offset;
930 
931 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
932 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
933 
934 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
935 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
936 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
937 
938 	writel(reg_addr, pcie_index_offset);
939 	readl(pcie_index_offset);
940 	writel(reg_data, pcie_data_offset);
941 	readl(pcie_data_offset);
942 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
943 }
944 
945 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
946 				     u64 reg_addr, u32 reg_data)
947 {
948 	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
949 	void __iomem *pcie_index_offset;
950 	void __iomem *pcie_index_hi_offset;
951 	void __iomem *pcie_data_offset;
952 
953 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
954 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
955 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
956 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
957 	else
958 		pcie_index_hi = 0;
959 
960 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
961 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
962 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
963 	if (pcie_index_hi != 0)
964 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
965 				pcie_index_hi * 4;
966 
967 	writel(reg_addr, pcie_index_offset);
968 	readl(pcie_index_offset);
969 	if (pcie_index_hi != 0) {
970 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
971 		readl(pcie_index_hi_offset);
972 	}
973 	writel(reg_data, pcie_data_offset);
974 	readl(pcie_data_offset);
975 
976 	/* clear the high bits */
977 	if (pcie_index_hi != 0) {
978 		writel(0, pcie_index_hi_offset);
979 		readl(pcie_index_hi_offset);
980 	}
981 
982 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
983 }
984 
985 /**
986  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
987  *
988  * @adev: amdgpu_device pointer
989  * @reg_addr: indirect register offset
990  * @reg_data: indirect register data
991  *
992  */
993 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
994 				   u32 reg_addr, u64 reg_data)
995 {
996 	unsigned long flags, pcie_index, pcie_data;
997 	void __iomem *pcie_index_offset;
998 	void __iomem *pcie_data_offset;
999 
1000 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1001 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1002 
1003 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1004 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1005 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1006 
1007 	/* write low 32 bits */
1008 	writel(reg_addr, pcie_index_offset);
1009 	readl(pcie_index_offset);
1010 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1011 	readl(pcie_data_offset);
1012 	/* write high 32 bits */
1013 	writel(reg_addr + 4, pcie_index_offset);
1014 	readl(pcie_index_offset);
1015 	writel((u32)(reg_data >> 32), pcie_data_offset);
1016 	readl(pcie_data_offset);
1017 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1018 }
1019 
1020 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1021 				   u64 reg_addr, u64 reg_data)
1022 {
1023 	unsigned long flags, pcie_index, pcie_data;
1024 	unsigned long pcie_index_hi = 0;
1025 	void __iomem *pcie_index_offset;
1026 	void __iomem *pcie_index_hi_offset;
1027 	void __iomem *pcie_data_offset;
1028 
1029 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1030 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1031 	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1032 		pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1033 
1034 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1035 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1036 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1037 	if (pcie_index_hi != 0)
1038 		pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1039 				pcie_index_hi * 4;
1040 
1041 	/* write low 32 bits */
1042 	writel(reg_addr, pcie_index_offset);
1043 	readl(pcie_index_offset);
1044 	if (pcie_index_hi != 0) {
1045 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1046 		readl(pcie_index_hi_offset);
1047 	}
1048 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1049 	readl(pcie_data_offset);
1050 	/* write high 32 bits */
1051 	writel(reg_addr + 4, pcie_index_offset);
1052 	readl(pcie_index_offset);
1053 	if (pcie_index_hi != 0) {
1054 		writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1055 		readl(pcie_index_hi_offset);
1056 	}
1057 	writel((u32)(reg_data >> 32), pcie_data_offset);
1058 	readl(pcie_data_offset);
1059 
1060 	/* clear the high bits */
1061 	if (pcie_index_hi != 0) {
1062 		writel(0, pcie_index_hi_offset);
1063 		readl(pcie_index_hi_offset);
1064 	}
1065 
1066 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1067 }
1068 
1069 /**
1070  * amdgpu_device_get_rev_id - query device rev_id
1071  *
1072  * @adev: amdgpu_device pointer
1073  *
1074  * Return device rev_id
1075  */
1076 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1077 {
1078 	return adev->nbio.funcs->get_rev_id(adev);
1079 }
1080 
1081 /**
1082  * amdgpu_invalid_rreg - dummy reg read function
1083  *
1084  * @adev: amdgpu_device pointer
1085  * @reg: offset of register
1086  *
1087  * Dummy register read function.  Used for register blocks
1088  * that certain asics don't have (all asics).
1089  * Returns the value in the register.
1090  */
1091 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1092 {
1093 	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1094 	BUG();
1095 	return 0;
1096 }
1097 
1098 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1099 {
1100 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1101 	BUG();
1102 	return 0;
1103 }
1104 
1105 /**
1106  * amdgpu_invalid_wreg - dummy reg write function
1107  *
1108  * @adev: amdgpu_device pointer
1109  * @reg: offset of register
1110  * @v: value to write to the register
1111  *
1112  * Dummy register read function.  Used for register blocks
1113  * that certain asics don't have (all asics).
1114  */
1115 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1116 {
1117 	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1118 		  reg, v);
1119 	BUG();
1120 }
1121 
1122 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1123 {
1124 	DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1125 		  reg, v);
1126 	BUG();
1127 }
1128 
1129 /**
1130  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1131  *
1132  * @adev: amdgpu_device pointer
1133  * @reg: offset of register
1134  *
1135  * Dummy register read function.  Used for register blocks
1136  * that certain asics don't have (all asics).
1137  * Returns the value in the register.
1138  */
1139 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1140 {
1141 	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1142 	BUG();
1143 	return 0;
1144 }
1145 
1146 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1147 {
1148 	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1149 	BUG();
1150 	return 0;
1151 }
1152 
1153 /**
1154  * amdgpu_invalid_wreg64 - dummy reg write function
1155  *
1156  * @adev: amdgpu_device pointer
1157  * @reg: offset of register
1158  * @v: value to write to the register
1159  *
1160  * Dummy register read function.  Used for register blocks
1161  * that certain asics don't have (all asics).
1162  */
1163 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1164 {
1165 	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1166 		  reg, v);
1167 	BUG();
1168 }
1169 
1170 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1171 {
1172 	DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1173 		  reg, v);
1174 	BUG();
1175 }
1176 
1177 /**
1178  * amdgpu_block_invalid_rreg - dummy reg read function
1179  *
1180  * @adev: amdgpu_device pointer
1181  * @block: offset of instance
1182  * @reg: offset of register
1183  *
1184  * Dummy register read function.  Used for register blocks
1185  * that certain asics don't have (all asics).
1186  * Returns the value in the register.
1187  */
1188 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1189 					  uint32_t block, uint32_t reg)
1190 {
1191 	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1192 		  reg, block);
1193 	BUG();
1194 	return 0;
1195 }
1196 
1197 /**
1198  * amdgpu_block_invalid_wreg - dummy reg write function
1199  *
1200  * @adev: amdgpu_device pointer
1201  * @block: offset of instance
1202  * @reg: offset of register
1203  * @v: value to write to the register
1204  *
1205  * Dummy register read function.  Used for register blocks
1206  * that certain asics don't have (all asics).
1207  */
1208 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1209 				      uint32_t block,
1210 				      uint32_t reg, uint32_t v)
1211 {
1212 	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1213 		  reg, block, v);
1214 	BUG();
1215 }
1216 
1217 /**
1218  * amdgpu_device_asic_init - Wrapper for atom asic_init
1219  *
1220  * @adev: amdgpu_device pointer
1221  *
1222  * Does any asic specific work and then calls atom asic init.
1223  */
1224 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1225 {
1226 	int ret;
1227 
1228 	amdgpu_asic_pre_asic_init(adev);
1229 
1230 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1231 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1232 		amdgpu_psp_wait_for_bootloader(adev);
1233 		ret = amdgpu_atomfirmware_asic_init(adev, true);
1234 		return ret;
1235 	} else {
1236 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1237 	}
1238 
1239 	return 0;
1240 }
1241 
1242 /**
1243  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1244  *
1245  * @adev: amdgpu_device pointer
1246  *
1247  * Allocates a scratch page of VRAM for use by various things in the
1248  * driver.
1249  */
1250 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1251 {
1252 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1253 				       AMDGPU_GEM_DOMAIN_VRAM |
1254 				       AMDGPU_GEM_DOMAIN_GTT,
1255 				       &adev->mem_scratch.robj,
1256 				       &adev->mem_scratch.gpu_addr,
1257 				       (void **)&adev->mem_scratch.ptr);
1258 }
1259 
1260 /**
1261  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1262  *
1263  * @adev: amdgpu_device pointer
1264  *
1265  * Frees the VRAM scratch page.
1266  */
1267 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1268 {
1269 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1270 }
1271 
1272 /**
1273  * amdgpu_device_program_register_sequence - program an array of registers.
1274  *
1275  * @adev: amdgpu_device pointer
1276  * @registers: pointer to the register array
1277  * @array_size: size of the register array
1278  *
1279  * Programs an array or registers with and or masks.
1280  * This is a helper for setting golden registers.
1281  */
1282 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1283 					     const u32 *registers,
1284 					     const u32 array_size)
1285 {
1286 	u32 tmp, reg, and_mask, or_mask;
1287 	int i;
1288 
1289 	if (array_size % 3)
1290 		return;
1291 
1292 	for (i = 0; i < array_size; i += 3) {
1293 		reg = registers[i + 0];
1294 		and_mask = registers[i + 1];
1295 		or_mask = registers[i + 2];
1296 
1297 		if (and_mask == 0xffffffff) {
1298 			tmp = or_mask;
1299 		} else {
1300 			tmp = RREG32(reg);
1301 			tmp &= ~and_mask;
1302 			if (adev->family >= AMDGPU_FAMILY_AI)
1303 				tmp |= (or_mask & and_mask);
1304 			else
1305 				tmp |= or_mask;
1306 		}
1307 		WREG32(reg, tmp);
1308 	}
1309 }
1310 
1311 /**
1312  * amdgpu_device_pci_config_reset - reset the GPU
1313  *
1314  * @adev: amdgpu_device pointer
1315  *
1316  * Resets the GPU using the pci config reset sequence.
1317  * Only applicable to asics prior to vega10.
1318  */
1319 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1320 {
1321 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1322 }
1323 
1324 /**
1325  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1326  *
1327  * @adev: amdgpu_device pointer
1328  *
1329  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1330  */
1331 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1332 {
1333 	return pci_reset_function(adev->pdev);
1334 }
1335 
1336 /*
1337  * amdgpu_device_wb_*()
1338  * Writeback is the method by which the GPU updates special pages in memory
1339  * with the status of certain GPU events (fences, ring pointers,etc.).
1340  */
1341 
1342 /**
1343  * amdgpu_device_wb_fini - Disable Writeback and free memory
1344  *
1345  * @adev: amdgpu_device pointer
1346  *
1347  * Disables Writeback and frees the Writeback memory (all asics).
1348  * Used at driver shutdown.
1349  */
1350 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1351 {
1352 	if (adev->wb.wb_obj) {
1353 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1354 				      &adev->wb.gpu_addr,
1355 				      (void **)&adev->wb.wb);
1356 		adev->wb.wb_obj = NULL;
1357 	}
1358 }
1359 
1360 /**
1361  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1362  *
1363  * @adev: amdgpu_device pointer
1364  *
1365  * Initializes writeback and allocates writeback memory (all asics).
1366  * Used at driver startup.
1367  * Returns 0 on success or an -error on failure.
1368  */
1369 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1370 {
1371 	int r;
1372 
1373 	if (adev->wb.wb_obj == NULL) {
1374 		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1375 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1376 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1377 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
1378 					    (void **)&adev->wb.wb);
1379 		if (r) {
1380 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1381 			return r;
1382 		}
1383 
1384 		adev->wb.num_wb = AMDGPU_MAX_WB;
1385 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1386 
1387 		/* clear wb memory */
1388 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1389 	}
1390 
1391 	return 0;
1392 }
1393 
1394 /**
1395  * amdgpu_device_wb_get - Allocate a wb entry
1396  *
1397  * @adev: amdgpu_device pointer
1398  * @wb: wb index
1399  *
1400  * Allocate a wb slot for use by the driver (all asics).
1401  * Returns 0 on success or -EINVAL on failure.
1402  */
1403 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1404 {
1405 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1406 
1407 	if (offset < adev->wb.num_wb) {
1408 		__set_bit(offset, adev->wb.used);
1409 		*wb = offset << 3; /* convert to dw offset */
1410 		return 0;
1411 	} else {
1412 		return -EINVAL;
1413 	}
1414 }
1415 
1416 /**
1417  * amdgpu_device_wb_free - Free a wb entry
1418  *
1419  * @adev: amdgpu_device pointer
1420  * @wb: wb index
1421  *
1422  * Free a wb slot allocated for use by the driver (all asics)
1423  */
1424 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1425 {
1426 	wb >>= 3;
1427 	if (wb < adev->wb.num_wb)
1428 		__clear_bit(wb, adev->wb.used);
1429 }
1430 
1431 /**
1432  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1433  *
1434  * @adev: amdgpu_device pointer
1435  *
1436  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1437  * to fail, but if any of the BARs is not accessible after the size we abort
1438  * driver loading by returning -ENODEV.
1439  */
1440 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1441 {
1442 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1443 	struct pci_bus *root;
1444 	struct resource *res;
1445 	unsigned int i;
1446 	u16 cmd;
1447 	int r;
1448 
1449 	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1450 		return 0;
1451 
1452 	/* Bypass for VF */
1453 	if (amdgpu_sriov_vf(adev))
1454 		return 0;
1455 
1456 	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1457 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1458 		DRM_WARN("System can't access extended configuration space,please check!!\n");
1459 
1460 	/* skip if the bios has already enabled large BAR */
1461 	if (adev->gmc.real_vram_size &&
1462 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1463 		return 0;
1464 
1465 	/* Check if the root BUS has 64bit memory resources */
1466 	root = adev->pdev->bus;
1467 	while (root->parent)
1468 		root = root->parent;
1469 
1470 	pci_bus_for_each_resource(root, res, i) {
1471 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1472 		    res->start > 0x100000000ull)
1473 			break;
1474 	}
1475 
1476 	/* Trying to resize is pointless without a root hub window above 4GB */
1477 	if (!res)
1478 		return 0;
1479 
1480 	/* Limit the BAR size to what is available */
1481 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1482 			rbar_size);
1483 
1484 	/* Disable memory decoding while we change the BAR addresses and size */
1485 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1486 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1487 			      cmd & ~PCI_COMMAND_MEMORY);
1488 
1489 	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1490 	amdgpu_doorbell_fini(adev);
1491 	if (adev->asic_type >= CHIP_BONAIRE)
1492 		pci_release_resource(adev->pdev, 2);
1493 
1494 	pci_release_resource(adev->pdev, 0);
1495 
1496 	r = pci_resize_resource(adev->pdev, 0, rbar_size);
1497 	if (r == -ENOSPC)
1498 		DRM_INFO("Not enough PCI address space for a large BAR.");
1499 	else if (r && r != -ENOTSUPP)
1500 		DRM_ERROR("Problem resizing BAR0 (%d).", r);
1501 
1502 	pci_assign_unassigned_bus_resources(adev->pdev->bus);
1503 
1504 	/* When the doorbell or fb BAR isn't available we have no chance of
1505 	 * using the device.
1506 	 */
1507 	r = amdgpu_doorbell_init(adev);
1508 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1509 		return -ENODEV;
1510 
1511 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1512 
1513 	return 0;
1514 }
1515 
1516 static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1517 {
1518 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1519 		return false;
1520 
1521 	return true;
1522 }
1523 
1524 /*
1525  * GPU helpers function.
1526  */
1527 /**
1528  * amdgpu_device_need_post - check if the hw need post or not
1529  *
1530  * @adev: amdgpu_device pointer
1531  *
1532  * Check if the asic has been initialized (all asics) at driver startup
1533  * or post is needed if  hw reset is performed.
1534  * Returns true if need or false if not.
1535  */
1536 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1537 {
1538 	uint32_t reg;
1539 
1540 	if (amdgpu_sriov_vf(adev))
1541 		return false;
1542 
1543 	if (!amdgpu_device_read_bios(adev))
1544 		return false;
1545 
1546 	if (amdgpu_passthrough(adev)) {
1547 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1548 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1549 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1550 		 * vpost executed for smc version below 22.15
1551 		 */
1552 		if (adev->asic_type == CHIP_FIJI) {
1553 			int err;
1554 			uint32_t fw_ver;
1555 
1556 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1557 			/* force vPost if error occured */
1558 			if (err)
1559 				return true;
1560 
1561 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1562 			release_firmware(adev->pm.fw);
1563 			if (fw_ver < 0x00160e00)
1564 				return true;
1565 		}
1566 	}
1567 
1568 	/* Don't post if we need to reset whole hive on init */
1569 	if (adev->gmc.xgmi.pending_reset)
1570 		return false;
1571 
1572 	if (adev->has_hw_reset) {
1573 		adev->has_hw_reset = false;
1574 		return true;
1575 	}
1576 
1577 	/* bios scratch used on CIK+ */
1578 	if (adev->asic_type >= CHIP_BONAIRE)
1579 		return amdgpu_atombios_scratch_need_asic_init(adev);
1580 
1581 	/* check MEM_SIZE for older asics */
1582 	reg = amdgpu_asic_get_config_memsize(adev);
1583 
1584 	if ((reg != 0) && (reg != 0xffffffff))
1585 		return false;
1586 
1587 	return true;
1588 }
1589 
1590 /*
1591  * Check whether seamless boot is supported.
1592  *
1593  * So far we only support seamless boot on DCE 3.0 or later.
1594  * If users report that it works on older ASICS as well, we may
1595  * loosen this.
1596  */
1597 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1598 {
1599 	switch (amdgpu_seamless) {
1600 	case -1:
1601 		break;
1602 	case 1:
1603 		return true;
1604 	case 0:
1605 		return false;
1606 	default:
1607 		DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1608 			  amdgpu_seamless);
1609 		return false;
1610 	}
1611 
1612 	if (!(adev->flags & AMD_IS_APU))
1613 		return false;
1614 
1615 	if (adev->mman.keep_stolen_vga_memory)
1616 		return false;
1617 
1618 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1619 }
1620 
1621 /*
1622  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1623  * don't support dynamic speed switching. Until we have confirmation from Intel
1624  * that a specific host supports it, it's safer that we keep it disabled for all.
1625  *
1626  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1627  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1628  */
1629 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1630 {
1631 #if IS_ENABLED(CONFIG_X86)
1632 	struct cpuinfo_x86 *c = &cpu_data(0);
1633 
1634 	/* eGPU change speeds based on USB4 fabric conditions */
1635 	if (dev_is_removable(adev->dev))
1636 		return true;
1637 
1638 	if (c->x86_vendor == X86_VENDOR_INTEL)
1639 		return false;
1640 #endif
1641 	return true;
1642 }
1643 
1644 /**
1645  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1646  *
1647  * @adev: amdgpu_device pointer
1648  *
1649  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1650  * be set for this device.
1651  *
1652  * Returns true if it should be used or false if not.
1653  */
1654 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1655 {
1656 	switch (amdgpu_aspm) {
1657 	case -1:
1658 		break;
1659 	case 0:
1660 		return false;
1661 	case 1:
1662 		return true;
1663 	default:
1664 		return false;
1665 	}
1666 	if (adev->flags & AMD_IS_APU)
1667 		return false;
1668 	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1669 		return false;
1670 	return pcie_aspm_enabled(adev->pdev);
1671 }
1672 
1673 /* if we get transitioned to only one device, take VGA back */
1674 /**
1675  * amdgpu_device_vga_set_decode - enable/disable vga decode
1676  *
1677  * @pdev: PCI device pointer
1678  * @state: enable/disable vga decode
1679  *
1680  * Enable/disable vga decode (all asics).
1681  * Returns VGA resource flags.
1682  */
1683 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1684 		bool state)
1685 {
1686 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1687 
1688 	amdgpu_asic_set_vga_state(adev, state);
1689 	if (state)
1690 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1691 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1692 	else
1693 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1694 }
1695 
1696 /**
1697  * amdgpu_device_check_block_size - validate the vm block size
1698  *
1699  * @adev: amdgpu_device pointer
1700  *
1701  * Validates the vm block size specified via module parameter.
1702  * The vm block size defines number of bits in page table versus page directory,
1703  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1704  * page table and the remaining bits are in the page directory.
1705  */
1706 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1707 {
1708 	/* defines number of bits in page table versus page directory,
1709 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1710 	 * page table and the remaining bits are in the page directory
1711 	 */
1712 	if (amdgpu_vm_block_size == -1)
1713 		return;
1714 
1715 	if (amdgpu_vm_block_size < 9) {
1716 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1717 			 amdgpu_vm_block_size);
1718 		amdgpu_vm_block_size = -1;
1719 	}
1720 }
1721 
1722 /**
1723  * amdgpu_device_check_vm_size - validate the vm size
1724  *
1725  * @adev: amdgpu_device pointer
1726  *
1727  * Validates the vm size in GB specified via module parameter.
1728  * The VM size is the size of the GPU virtual memory space in GB.
1729  */
1730 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1731 {
1732 	/* no need to check the default value */
1733 	if (amdgpu_vm_size == -1)
1734 		return;
1735 
1736 	if (amdgpu_vm_size < 1) {
1737 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1738 			 amdgpu_vm_size);
1739 		amdgpu_vm_size = -1;
1740 	}
1741 }
1742 
1743 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1744 {
1745 	struct sysinfo si;
1746 	bool is_os_64 = (sizeof(void *) == 8);
1747 	uint64_t total_memory;
1748 	uint64_t dram_size_seven_GB = 0x1B8000000;
1749 	uint64_t dram_size_three_GB = 0xB8000000;
1750 
1751 	if (amdgpu_smu_memory_pool_size == 0)
1752 		return;
1753 
1754 	if (!is_os_64) {
1755 		DRM_WARN("Not 64-bit OS, feature not supported\n");
1756 		goto def_value;
1757 	}
1758 	si_meminfo(&si);
1759 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1760 
1761 	if ((amdgpu_smu_memory_pool_size == 1) ||
1762 		(amdgpu_smu_memory_pool_size == 2)) {
1763 		if (total_memory < dram_size_three_GB)
1764 			goto def_value1;
1765 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1766 		(amdgpu_smu_memory_pool_size == 8)) {
1767 		if (total_memory < dram_size_seven_GB)
1768 			goto def_value1;
1769 	} else {
1770 		DRM_WARN("Smu memory pool size not supported\n");
1771 		goto def_value;
1772 	}
1773 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1774 
1775 	return;
1776 
1777 def_value1:
1778 	DRM_WARN("No enough system memory\n");
1779 def_value:
1780 	adev->pm.smu_prv_buffer_size = 0;
1781 }
1782 
1783 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1784 {
1785 	if (!(adev->flags & AMD_IS_APU) ||
1786 	    adev->asic_type < CHIP_RAVEN)
1787 		return 0;
1788 
1789 	switch (adev->asic_type) {
1790 	case CHIP_RAVEN:
1791 		if (adev->pdev->device == 0x15dd)
1792 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1793 		if (adev->pdev->device == 0x15d8)
1794 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1795 		break;
1796 	case CHIP_RENOIR:
1797 		if ((adev->pdev->device == 0x1636) ||
1798 		    (adev->pdev->device == 0x164c))
1799 			adev->apu_flags |= AMD_APU_IS_RENOIR;
1800 		else
1801 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1802 		break;
1803 	case CHIP_VANGOGH:
1804 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
1805 		break;
1806 	case CHIP_YELLOW_CARP:
1807 		break;
1808 	case CHIP_CYAN_SKILLFISH:
1809 		if ((adev->pdev->device == 0x13FE) ||
1810 		    (adev->pdev->device == 0x143F))
1811 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1812 		break;
1813 	default:
1814 		break;
1815 	}
1816 
1817 	return 0;
1818 }
1819 
1820 /**
1821  * amdgpu_device_check_arguments - validate module params
1822  *
1823  * @adev: amdgpu_device pointer
1824  *
1825  * Validates certain module parameters and updates
1826  * the associated values used by the driver (all asics).
1827  */
1828 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1829 {
1830 	if (amdgpu_sched_jobs < 4) {
1831 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1832 			 amdgpu_sched_jobs);
1833 		amdgpu_sched_jobs = 4;
1834 	} else if (!is_power_of_2(amdgpu_sched_jobs)) {
1835 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1836 			 amdgpu_sched_jobs);
1837 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1838 	}
1839 
1840 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1841 		/* gart size must be greater or equal to 32M */
1842 		dev_warn(adev->dev, "gart size (%d) too small\n",
1843 			 amdgpu_gart_size);
1844 		amdgpu_gart_size = -1;
1845 	}
1846 
1847 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1848 		/* gtt size must be greater or equal to 32M */
1849 		dev_warn(adev->dev, "gtt size (%d) too small\n",
1850 				 amdgpu_gtt_size);
1851 		amdgpu_gtt_size = -1;
1852 	}
1853 
1854 	/* valid range is between 4 and 9 inclusive */
1855 	if (amdgpu_vm_fragment_size != -1 &&
1856 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1857 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
1858 		amdgpu_vm_fragment_size = -1;
1859 	}
1860 
1861 	if (amdgpu_sched_hw_submission < 2) {
1862 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1863 			 amdgpu_sched_hw_submission);
1864 		amdgpu_sched_hw_submission = 2;
1865 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1866 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1867 			 amdgpu_sched_hw_submission);
1868 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1869 	}
1870 
1871 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1872 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1873 		amdgpu_reset_method = -1;
1874 	}
1875 
1876 	amdgpu_device_check_smu_prv_buffer_size(adev);
1877 
1878 	amdgpu_device_check_vm_size(adev);
1879 
1880 	amdgpu_device_check_block_size(adev);
1881 
1882 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1883 
1884 	return 0;
1885 }
1886 
1887 /**
1888  * amdgpu_switcheroo_set_state - set switcheroo state
1889  *
1890  * @pdev: pci dev pointer
1891  * @state: vga_switcheroo state
1892  *
1893  * Callback for the switcheroo driver.  Suspends or resumes
1894  * the asics before or after it is powered up using ACPI methods.
1895  */
1896 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1897 					enum vga_switcheroo_state state)
1898 {
1899 	struct drm_device *dev = pci_get_drvdata(pdev);
1900 	int r;
1901 
1902 	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1903 		return;
1904 
1905 	if (state == VGA_SWITCHEROO_ON) {
1906 		pr_info("switched on\n");
1907 		/* don't suspend or resume card normally */
1908 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1909 
1910 		pci_set_power_state(pdev, PCI_D0);
1911 		amdgpu_device_load_pci_state(pdev);
1912 		r = pci_enable_device(pdev);
1913 		if (r)
1914 			DRM_WARN("pci_enable_device failed (%d)\n", r);
1915 		amdgpu_device_resume(dev, true);
1916 
1917 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
1918 	} else {
1919 		pr_info("switched off\n");
1920 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1921 		amdgpu_device_prepare(dev);
1922 		amdgpu_device_suspend(dev, true);
1923 		amdgpu_device_cache_pci_state(pdev);
1924 		/* Shut down the device */
1925 		pci_disable_device(pdev);
1926 		pci_set_power_state(pdev, PCI_D3cold);
1927 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1928 	}
1929 }
1930 
1931 /**
1932  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1933  *
1934  * @pdev: pci dev pointer
1935  *
1936  * Callback for the switcheroo driver.  Check of the switcheroo
1937  * state can be changed.
1938  * Returns true if the state can be changed, false if not.
1939  */
1940 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1941 {
1942 	struct drm_device *dev = pci_get_drvdata(pdev);
1943 
1944        /*
1945 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1946 	* locking inversion with the driver load path. And the access here is
1947 	* completely racy anyway. So don't bother with locking for now.
1948 	*/
1949 	return atomic_read(&dev->open_count) == 0;
1950 }
1951 
1952 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1953 	.set_gpu_state = amdgpu_switcheroo_set_state,
1954 	.reprobe = NULL,
1955 	.can_switch = amdgpu_switcheroo_can_switch,
1956 };
1957 
1958 /**
1959  * amdgpu_device_ip_set_clockgating_state - set the CG state
1960  *
1961  * @dev: amdgpu_device pointer
1962  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1963  * @state: clockgating state (gate or ungate)
1964  *
1965  * Sets the requested clockgating state for all instances of
1966  * the hardware IP specified.
1967  * Returns the error code from the last instance.
1968  */
1969 int amdgpu_device_ip_set_clockgating_state(void *dev,
1970 					   enum amd_ip_block_type block_type,
1971 					   enum amd_clockgating_state state)
1972 {
1973 	struct amdgpu_device *adev = dev;
1974 	int i, r = 0;
1975 
1976 	for (i = 0; i < adev->num_ip_blocks; i++) {
1977 		if (!adev->ip_blocks[i].status.valid)
1978 			continue;
1979 		if (adev->ip_blocks[i].version->type != block_type)
1980 			continue;
1981 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1982 			continue;
1983 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1984 			(void *)adev, state);
1985 		if (r)
1986 			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1987 				  adev->ip_blocks[i].version->funcs->name, r);
1988 	}
1989 	return r;
1990 }
1991 
1992 /**
1993  * amdgpu_device_ip_set_powergating_state - set the PG state
1994  *
1995  * @dev: amdgpu_device pointer
1996  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1997  * @state: powergating state (gate or ungate)
1998  *
1999  * Sets the requested powergating state for all instances of
2000  * the hardware IP specified.
2001  * Returns the error code from the last instance.
2002  */
2003 int amdgpu_device_ip_set_powergating_state(void *dev,
2004 					   enum amd_ip_block_type block_type,
2005 					   enum amd_powergating_state state)
2006 {
2007 	struct amdgpu_device *adev = dev;
2008 	int i, r = 0;
2009 
2010 	for (i = 0; i < adev->num_ip_blocks; i++) {
2011 		if (!adev->ip_blocks[i].status.valid)
2012 			continue;
2013 		if (adev->ip_blocks[i].version->type != block_type)
2014 			continue;
2015 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2016 			continue;
2017 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2018 			(void *)adev, state);
2019 		if (r)
2020 			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2021 				  adev->ip_blocks[i].version->funcs->name, r);
2022 	}
2023 	return r;
2024 }
2025 
2026 /**
2027  * amdgpu_device_ip_get_clockgating_state - get the CG state
2028  *
2029  * @adev: amdgpu_device pointer
2030  * @flags: clockgating feature flags
2031  *
2032  * Walks the list of IPs on the device and updates the clockgating
2033  * flags for each IP.
2034  * Updates @flags with the feature flags for each hardware IP where
2035  * clockgating is enabled.
2036  */
2037 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2038 					    u64 *flags)
2039 {
2040 	int i;
2041 
2042 	for (i = 0; i < adev->num_ip_blocks; i++) {
2043 		if (!adev->ip_blocks[i].status.valid)
2044 			continue;
2045 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2046 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2047 	}
2048 }
2049 
2050 /**
2051  * amdgpu_device_ip_wait_for_idle - wait for idle
2052  *
2053  * @adev: amdgpu_device pointer
2054  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2055  *
2056  * Waits for the request hardware IP to be idle.
2057  * Returns 0 for success or a negative error code on failure.
2058  */
2059 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2060 				   enum amd_ip_block_type block_type)
2061 {
2062 	int i, r;
2063 
2064 	for (i = 0; i < adev->num_ip_blocks; i++) {
2065 		if (!adev->ip_blocks[i].status.valid)
2066 			continue;
2067 		if (adev->ip_blocks[i].version->type == block_type) {
2068 			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
2069 			if (r)
2070 				return r;
2071 			break;
2072 		}
2073 	}
2074 	return 0;
2075 
2076 }
2077 
2078 /**
2079  * amdgpu_device_ip_is_idle - is the hardware IP idle
2080  *
2081  * @adev: amdgpu_device pointer
2082  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2083  *
2084  * Check if the hardware IP is idle or not.
2085  * Returns true if it the IP is idle, false if not.
2086  */
2087 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
2088 			      enum amd_ip_block_type block_type)
2089 {
2090 	int i;
2091 
2092 	for (i = 0; i < adev->num_ip_blocks; i++) {
2093 		if (!adev->ip_blocks[i].status.valid)
2094 			continue;
2095 		if (adev->ip_blocks[i].version->type == block_type)
2096 			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
2097 	}
2098 	return true;
2099 
2100 }
2101 
2102 /**
2103  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2104  *
2105  * @adev: amdgpu_device pointer
2106  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2107  *
2108  * Returns a pointer to the hardware IP block structure
2109  * if it exists for the asic, otherwise NULL.
2110  */
2111 struct amdgpu_ip_block *
2112 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2113 			      enum amd_ip_block_type type)
2114 {
2115 	int i;
2116 
2117 	for (i = 0; i < adev->num_ip_blocks; i++)
2118 		if (adev->ip_blocks[i].version->type == type)
2119 			return &adev->ip_blocks[i];
2120 
2121 	return NULL;
2122 }
2123 
2124 /**
2125  * amdgpu_device_ip_block_version_cmp
2126  *
2127  * @adev: amdgpu_device pointer
2128  * @type: enum amd_ip_block_type
2129  * @major: major version
2130  * @minor: minor version
2131  *
2132  * return 0 if equal or greater
2133  * return 1 if smaller or the ip_block doesn't exist
2134  */
2135 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2136 				       enum amd_ip_block_type type,
2137 				       u32 major, u32 minor)
2138 {
2139 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2140 
2141 	if (ip_block && ((ip_block->version->major > major) ||
2142 			((ip_block->version->major == major) &&
2143 			(ip_block->version->minor >= minor))))
2144 		return 0;
2145 
2146 	return 1;
2147 }
2148 
2149 /**
2150  * amdgpu_device_ip_block_add
2151  *
2152  * @adev: amdgpu_device pointer
2153  * @ip_block_version: pointer to the IP to add
2154  *
2155  * Adds the IP block driver information to the collection of IPs
2156  * on the asic.
2157  */
2158 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2159 			       const struct amdgpu_ip_block_version *ip_block_version)
2160 {
2161 	if (!ip_block_version)
2162 		return -EINVAL;
2163 
2164 	switch (ip_block_version->type) {
2165 	case AMD_IP_BLOCK_TYPE_VCN:
2166 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2167 			return 0;
2168 		break;
2169 	case AMD_IP_BLOCK_TYPE_JPEG:
2170 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2171 			return 0;
2172 		break;
2173 	default:
2174 		break;
2175 	}
2176 
2177 	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2178 		  ip_block_version->funcs->name);
2179 
2180 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2181 
2182 	return 0;
2183 }
2184 
2185 /**
2186  * amdgpu_device_enable_virtual_display - enable virtual display feature
2187  *
2188  * @adev: amdgpu_device pointer
2189  *
2190  * Enabled the virtual display feature if the user has enabled it via
2191  * the module parameter virtual_display.  This feature provides a virtual
2192  * display hardware on headless boards or in virtualized environments.
2193  * This function parses and validates the configuration string specified by
2194  * the user and configues the virtual display configuration (number of
2195  * virtual connectors, crtcs, etc.) specified.
2196  */
2197 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2198 {
2199 	adev->enable_virtual_display = false;
2200 
2201 	if (amdgpu_virtual_display) {
2202 		const char *pci_address_name = pci_name(adev->pdev);
2203 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2204 
2205 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2206 		pciaddstr_tmp = pciaddstr;
2207 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2208 			pciaddname = strsep(&pciaddname_tmp, ",");
2209 			if (!strcmp("all", pciaddname)
2210 			    || !strcmp(pci_address_name, pciaddname)) {
2211 				long num_crtc;
2212 				int res = -1;
2213 
2214 				adev->enable_virtual_display = true;
2215 
2216 				if (pciaddname_tmp)
2217 					res = kstrtol(pciaddname_tmp, 10,
2218 						      &num_crtc);
2219 
2220 				if (!res) {
2221 					if (num_crtc < 1)
2222 						num_crtc = 1;
2223 					if (num_crtc > 6)
2224 						num_crtc = 6;
2225 					adev->mode_info.num_crtc = num_crtc;
2226 				} else {
2227 					adev->mode_info.num_crtc = 1;
2228 				}
2229 				break;
2230 			}
2231 		}
2232 
2233 		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2234 			 amdgpu_virtual_display, pci_address_name,
2235 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2236 
2237 		kfree(pciaddstr);
2238 	}
2239 }
2240 
2241 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2242 {
2243 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2244 		adev->mode_info.num_crtc = 1;
2245 		adev->enable_virtual_display = true;
2246 		DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2247 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
2248 	}
2249 }
2250 
2251 /**
2252  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2253  *
2254  * @adev: amdgpu_device pointer
2255  *
2256  * Parses the asic configuration parameters specified in the gpu info
2257  * firmware and makes them availale to the driver for use in configuring
2258  * the asic.
2259  * Returns 0 on success, -EINVAL on failure.
2260  */
2261 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2262 {
2263 	const char *chip_name;
2264 	char fw_name[40];
2265 	int err;
2266 	const struct gpu_info_firmware_header_v1_0 *hdr;
2267 
2268 	adev->firmware.gpu_info_fw = NULL;
2269 
2270 	if (adev->mman.discovery_bin)
2271 		return 0;
2272 
2273 	switch (adev->asic_type) {
2274 	default:
2275 		return 0;
2276 	case CHIP_VEGA10:
2277 		chip_name = "vega10";
2278 		break;
2279 	case CHIP_VEGA12:
2280 		chip_name = "vega12";
2281 		break;
2282 	case CHIP_RAVEN:
2283 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2284 			chip_name = "raven2";
2285 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2286 			chip_name = "picasso";
2287 		else
2288 			chip_name = "raven";
2289 		break;
2290 	case CHIP_ARCTURUS:
2291 		chip_name = "arcturus";
2292 		break;
2293 	case CHIP_NAVI12:
2294 		chip_name = "navi12";
2295 		break;
2296 	}
2297 
2298 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2299 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2300 	if (err) {
2301 		dev_err(adev->dev,
2302 			"Failed to get gpu_info firmware \"%s\"\n",
2303 			fw_name);
2304 		goto out;
2305 	}
2306 
2307 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2308 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2309 
2310 	switch (hdr->version_major) {
2311 	case 1:
2312 	{
2313 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2314 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2315 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2316 
2317 		/*
2318 		 * Should be droped when DAL no longer needs it.
2319 		 */
2320 		if (adev->asic_type == CHIP_NAVI12)
2321 			goto parse_soc_bounding_box;
2322 
2323 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2324 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2325 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2326 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2327 		adev->gfx.config.max_texture_channel_caches =
2328 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
2329 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2330 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2331 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2332 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2333 		adev->gfx.config.double_offchip_lds_buf =
2334 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2335 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2336 		adev->gfx.cu_info.max_waves_per_simd =
2337 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2338 		adev->gfx.cu_info.max_scratch_slots_per_cu =
2339 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2340 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2341 		if (hdr->version_minor >= 1) {
2342 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2343 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2344 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2345 			adev->gfx.config.num_sc_per_sh =
2346 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2347 			adev->gfx.config.num_packer_per_sc =
2348 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2349 		}
2350 
2351 parse_soc_bounding_box:
2352 		/*
2353 		 * soc bounding box info is not integrated in disocovery table,
2354 		 * we always need to parse it from gpu info firmware if needed.
2355 		 */
2356 		if (hdr->version_minor == 2) {
2357 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2358 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2359 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2360 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2361 		}
2362 		break;
2363 	}
2364 	default:
2365 		dev_err(adev->dev,
2366 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2367 		err = -EINVAL;
2368 		goto out;
2369 	}
2370 out:
2371 	return err;
2372 }
2373 
2374 /**
2375  * amdgpu_device_ip_early_init - run early init for hardware IPs
2376  *
2377  * @adev: amdgpu_device pointer
2378  *
2379  * Early initialization pass for hardware IPs.  The hardware IPs that make
2380  * up each asic are discovered each IP's early_init callback is run.  This
2381  * is the first stage in initializing the asic.
2382  * Returns 0 on success, negative error code on failure.
2383  */
2384 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2385 {
2386 	struct pci_dev *parent;
2387 	int i, r;
2388 	bool total;
2389 
2390 	amdgpu_device_enable_virtual_display(adev);
2391 
2392 	if (amdgpu_sriov_vf(adev)) {
2393 		r = amdgpu_virt_request_full_gpu(adev, true);
2394 		if (r)
2395 			return r;
2396 	}
2397 
2398 	switch (adev->asic_type) {
2399 #ifdef CONFIG_DRM_AMDGPU_SI
2400 	case CHIP_VERDE:
2401 	case CHIP_TAHITI:
2402 	case CHIP_PITCAIRN:
2403 	case CHIP_OLAND:
2404 	case CHIP_HAINAN:
2405 		adev->family = AMDGPU_FAMILY_SI;
2406 		r = si_set_ip_blocks(adev);
2407 		if (r)
2408 			return r;
2409 		break;
2410 #endif
2411 #ifdef CONFIG_DRM_AMDGPU_CIK
2412 	case CHIP_BONAIRE:
2413 	case CHIP_HAWAII:
2414 	case CHIP_KAVERI:
2415 	case CHIP_KABINI:
2416 	case CHIP_MULLINS:
2417 		if (adev->flags & AMD_IS_APU)
2418 			adev->family = AMDGPU_FAMILY_KV;
2419 		else
2420 			adev->family = AMDGPU_FAMILY_CI;
2421 
2422 		r = cik_set_ip_blocks(adev);
2423 		if (r)
2424 			return r;
2425 		break;
2426 #endif
2427 	case CHIP_TOPAZ:
2428 	case CHIP_TONGA:
2429 	case CHIP_FIJI:
2430 	case CHIP_POLARIS10:
2431 	case CHIP_POLARIS11:
2432 	case CHIP_POLARIS12:
2433 	case CHIP_VEGAM:
2434 	case CHIP_CARRIZO:
2435 	case CHIP_STONEY:
2436 		if (adev->flags & AMD_IS_APU)
2437 			adev->family = AMDGPU_FAMILY_CZ;
2438 		else
2439 			adev->family = AMDGPU_FAMILY_VI;
2440 
2441 		r = vi_set_ip_blocks(adev);
2442 		if (r)
2443 			return r;
2444 		break;
2445 	default:
2446 		r = amdgpu_discovery_set_ip_blocks(adev);
2447 		if (r)
2448 			return r;
2449 		break;
2450 	}
2451 
2452 	if (amdgpu_has_atpx() &&
2453 	    (amdgpu_is_atpx_hybrid() ||
2454 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2455 	    ((adev->flags & AMD_IS_APU) == 0) &&
2456 	    !dev_is_removable(&adev->pdev->dev))
2457 		adev->flags |= AMD_IS_PX;
2458 
2459 	if (!(adev->flags & AMD_IS_APU)) {
2460 		parent = pcie_find_root_port(adev->pdev);
2461 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2462 	}
2463 
2464 
2465 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2466 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2467 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2468 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2469 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2470 	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2471 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2472 
2473 	total = true;
2474 	for (i = 0; i < adev->num_ip_blocks; i++) {
2475 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2476 			DRM_WARN("disabled ip block: %d <%s>\n",
2477 				  i, adev->ip_blocks[i].version->funcs->name);
2478 			adev->ip_blocks[i].status.valid = false;
2479 		} else {
2480 			if (adev->ip_blocks[i].version->funcs->early_init) {
2481 				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2482 				if (r == -ENOENT) {
2483 					adev->ip_blocks[i].status.valid = false;
2484 				} else if (r) {
2485 					DRM_ERROR("early_init of IP block <%s> failed %d\n",
2486 						  adev->ip_blocks[i].version->funcs->name, r);
2487 					total = false;
2488 				} else {
2489 					adev->ip_blocks[i].status.valid = true;
2490 				}
2491 			} else {
2492 				adev->ip_blocks[i].status.valid = true;
2493 			}
2494 		}
2495 		/* get the vbios after the asic_funcs are set up */
2496 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2497 			r = amdgpu_device_parse_gpu_info_fw(adev);
2498 			if (r)
2499 				return r;
2500 
2501 			/* Read BIOS */
2502 			if (amdgpu_device_read_bios(adev)) {
2503 				if (!amdgpu_get_bios(adev))
2504 					return -EINVAL;
2505 
2506 				r = amdgpu_atombios_init(adev);
2507 				if (r) {
2508 					dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2509 					amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2510 					return r;
2511 				}
2512 			}
2513 
2514 			/*get pf2vf msg info at it's earliest time*/
2515 			if (amdgpu_sriov_vf(adev))
2516 				amdgpu_virt_init_data_exchange(adev);
2517 
2518 		}
2519 	}
2520 	if (!total)
2521 		return -ENODEV;
2522 
2523 	amdgpu_amdkfd_device_probe(adev);
2524 	adev->cg_flags &= amdgpu_cg_mask;
2525 	adev->pg_flags &= amdgpu_pg_mask;
2526 
2527 	return 0;
2528 }
2529 
2530 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2531 {
2532 	int i, r;
2533 
2534 	for (i = 0; i < adev->num_ip_blocks; i++) {
2535 		if (!adev->ip_blocks[i].status.sw)
2536 			continue;
2537 		if (adev->ip_blocks[i].status.hw)
2538 			continue;
2539 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2540 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2541 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2542 			r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2543 			if (r) {
2544 				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2545 					  adev->ip_blocks[i].version->funcs->name, r);
2546 				return r;
2547 			}
2548 			adev->ip_blocks[i].status.hw = true;
2549 		}
2550 	}
2551 
2552 	return 0;
2553 }
2554 
2555 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2556 {
2557 	int i, r;
2558 
2559 	for (i = 0; i < adev->num_ip_blocks; i++) {
2560 		if (!adev->ip_blocks[i].status.sw)
2561 			continue;
2562 		if (adev->ip_blocks[i].status.hw)
2563 			continue;
2564 		r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2565 		if (r) {
2566 			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2567 				  adev->ip_blocks[i].version->funcs->name, r);
2568 			return r;
2569 		}
2570 		adev->ip_blocks[i].status.hw = true;
2571 	}
2572 
2573 	return 0;
2574 }
2575 
2576 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2577 {
2578 	int r = 0;
2579 	int i;
2580 	uint32_t smu_version;
2581 
2582 	if (adev->asic_type >= CHIP_VEGA10) {
2583 		for (i = 0; i < adev->num_ip_blocks; i++) {
2584 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2585 				continue;
2586 
2587 			if (!adev->ip_blocks[i].status.sw)
2588 				continue;
2589 
2590 			/* no need to do the fw loading again if already done*/
2591 			if (adev->ip_blocks[i].status.hw == true)
2592 				break;
2593 
2594 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2595 				r = adev->ip_blocks[i].version->funcs->resume(adev);
2596 				if (r) {
2597 					DRM_ERROR("resume of IP block <%s> failed %d\n",
2598 							  adev->ip_blocks[i].version->funcs->name, r);
2599 					return r;
2600 				}
2601 			} else {
2602 				r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2603 				if (r) {
2604 					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2605 							  adev->ip_blocks[i].version->funcs->name, r);
2606 					return r;
2607 				}
2608 			}
2609 
2610 			adev->ip_blocks[i].status.hw = true;
2611 			break;
2612 		}
2613 	}
2614 
2615 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2616 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2617 
2618 	return r;
2619 }
2620 
2621 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2622 {
2623 	long timeout;
2624 	int r, i;
2625 
2626 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2627 		struct amdgpu_ring *ring = adev->rings[i];
2628 
2629 		/* No need to setup the GPU scheduler for rings that don't need it */
2630 		if (!ring || ring->no_scheduler)
2631 			continue;
2632 
2633 		switch (ring->funcs->type) {
2634 		case AMDGPU_RING_TYPE_GFX:
2635 			timeout = adev->gfx_timeout;
2636 			break;
2637 		case AMDGPU_RING_TYPE_COMPUTE:
2638 			timeout = adev->compute_timeout;
2639 			break;
2640 		case AMDGPU_RING_TYPE_SDMA:
2641 			timeout = adev->sdma_timeout;
2642 			break;
2643 		default:
2644 			timeout = adev->video_timeout;
2645 			break;
2646 		}
2647 
2648 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2649 				   DRM_SCHED_PRIORITY_COUNT,
2650 				   ring->num_hw_submission, 0,
2651 				   timeout, adev->reset_domain->wq,
2652 				   ring->sched_score, ring->name,
2653 				   adev->dev);
2654 		if (r) {
2655 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
2656 				  ring->name);
2657 			return r;
2658 		}
2659 		r = amdgpu_uvd_entity_init(adev, ring);
2660 		if (r) {
2661 			DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2662 				  ring->name);
2663 			return r;
2664 		}
2665 		r = amdgpu_vce_entity_init(adev, ring);
2666 		if (r) {
2667 			DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2668 				  ring->name);
2669 			return r;
2670 		}
2671 	}
2672 
2673 	amdgpu_xcp_update_partition_sched_list(adev);
2674 
2675 	return 0;
2676 }
2677 
2678 
2679 /**
2680  * amdgpu_device_ip_init - run init for hardware IPs
2681  *
2682  * @adev: amdgpu_device pointer
2683  *
2684  * Main initialization pass for hardware IPs.  The list of all the hardware
2685  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2686  * are run.  sw_init initializes the software state associated with each IP
2687  * and hw_init initializes the hardware associated with each IP.
2688  * Returns 0 on success, negative error code on failure.
2689  */
2690 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2691 {
2692 	int i, r;
2693 
2694 	r = amdgpu_ras_init(adev);
2695 	if (r)
2696 		return r;
2697 
2698 	for (i = 0; i < adev->num_ip_blocks; i++) {
2699 		if (!adev->ip_blocks[i].status.valid)
2700 			continue;
2701 		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2702 		if (r) {
2703 			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2704 				  adev->ip_blocks[i].version->funcs->name, r);
2705 			goto init_failed;
2706 		}
2707 		adev->ip_blocks[i].status.sw = true;
2708 
2709 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2710 			/* need to do common hw init early so everything is set up for gmc */
2711 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2712 			if (r) {
2713 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2714 				goto init_failed;
2715 			}
2716 			adev->ip_blocks[i].status.hw = true;
2717 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2718 			/* need to do gmc hw init early so we can allocate gpu mem */
2719 			/* Try to reserve bad pages early */
2720 			if (amdgpu_sriov_vf(adev))
2721 				amdgpu_virt_exchange_data(adev);
2722 
2723 			r = amdgpu_device_mem_scratch_init(adev);
2724 			if (r) {
2725 				DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2726 				goto init_failed;
2727 			}
2728 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2729 			if (r) {
2730 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2731 				goto init_failed;
2732 			}
2733 			r = amdgpu_device_wb_init(adev);
2734 			if (r) {
2735 				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2736 				goto init_failed;
2737 			}
2738 			adev->ip_blocks[i].status.hw = true;
2739 
2740 			/* right after GMC hw init, we create CSA */
2741 			if (adev->gfx.mcbp) {
2742 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2743 							       AMDGPU_GEM_DOMAIN_VRAM |
2744 							       AMDGPU_GEM_DOMAIN_GTT,
2745 							       AMDGPU_CSA_SIZE);
2746 				if (r) {
2747 					DRM_ERROR("allocate CSA failed %d\n", r);
2748 					goto init_failed;
2749 				}
2750 			}
2751 
2752 			r = amdgpu_seq64_init(adev);
2753 			if (r) {
2754 				DRM_ERROR("allocate seq64 failed %d\n", r);
2755 				goto init_failed;
2756 			}
2757 		}
2758 	}
2759 
2760 	if (amdgpu_sriov_vf(adev))
2761 		amdgpu_virt_init_data_exchange(adev);
2762 
2763 	r = amdgpu_ib_pool_init(adev);
2764 	if (r) {
2765 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2766 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2767 		goto init_failed;
2768 	}
2769 
2770 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2771 	if (r)
2772 		goto init_failed;
2773 
2774 	r = amdgpu_device_ip_hw_init_phase1(adev);
2775 	if (r)
2776 		goto init_failed;
2777 
2778 	r = amdgpu_device_fw_loading(adev);
2779 	if (r)
2780 		goto init_failed;
2781 
2782 	r = amdgpu_device_ip_hw_init_phase2(adev);
2783 	if (r)
2784 		goto init_failed;
2785 
2786 	/*
2787 	 * retired pages will be loaded from eeprom and reserved here,
2788 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
2789 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2790 	 * for I2C communication which only true at this point.
2791 	 *
2792 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2793 	 * failure from bad gpu situation and stop amdgpu init process
2794 	 * accordingly. For other failed cases, it will still release all
2795 	 * the resource and print error message, rather than returning one
2796 	 * negative value to upper level.
2797 	 *
2798 	 * Note: theoretically, this should be called before all vram allocations
2799 	 * to protect retired page from abusing
2800 	 */
2801 	r = amdgpu_ras_recovery_init(adev);
2802 	if (r)
2803 		goto init_failed;
2804 
2805 	/**
2806 	 * In case of XGMI grab extra reference for reset domain for this device
2807 	 */
2808 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2809 		if (amdgpu_xgmi_add_device(adev) == 0) {
2810 			if (!amdgpu_sriov_vf(adev)) {
2811 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2812 
2813 				if (WARN_ON(!hive)) {
2814 					r = -ENOENT;
2815 					goto init_failed;
2816 				}
2817 
2818 				if (!hive->reset_domain ||
2819 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2820 					r = -ENOENT;
2821 					amdgpu_put_xgmi_hive(hive);
2822 					goto init_failed;
2823 				}
2824 
2825 				/* Drop the early temporary reset domain we created for device */
2826 				amdgpu_reset_put_reset_domain(adev->reset_domain);
2827 				adev->reset_domain = hive->reset_domain;
2828 				amdgpu_put_xgmi_hive(hive);
2829 			}
2830 		}
2831 	}
2832 
2833 	r = amdgpu_device_init_schedulers(adev);
2834 	if (r)
2835 		goto init_failed;
2836 
2837 	if (adev->mman.buffer_funcs_ring->sched.ready)
2838 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
2839 
2840 	/* Don't init kfd if whole hive need to be reset during init */
2841 	if (!adev->gmc.xgmi.pending_reset) {
2842 		kgd2kfd_init_zone_device(adev);
2843 		amdgpu_amdkfd_device_init(adev);
2844 	}
2845 
2846 	amdgpu_fru_get_product_info(adev);
2847 
2848 init_failed:
2849 
2850 	return r;
2851 }
2852 
2853 /**
2854  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2855  *
2856  * @adev: amdgpu_device pointer
2857  *
2858  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2859  * this function before a GPU reset.  If the value is retained after a
2860  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2861  */
2862 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2863 {
2864 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2865 }
2866 
2867 /**
2868  * amdgpu_device_check_vram_lost - check if vram is valid
2869  *
2870  * @adev: amdgpu_device pointer
2871  *
2872  * Checks the reset magic value written to the gart pointer in VRAM.
2873  * The driver calls this after a GPU reset to see if the contents of
2874  * VRAM is lost or now.
2875  * returns true if vram is lost, false if not.
2876  */
2877 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2878 {
2879 	if (memcmp(adev->gart.ptr, adev->reset_magic,
2880 			AMDGPU_RESET_MAGIC_NUM))
2881 		return true;
2882 
2883 	if (!amdgpu_in_reset(adev))
2884 		return false;
2885 
2886 	/*
2887 	 * For all ASICs with baco/mode1 reset, the VRAM is
2888 	 * always assumed to be lost.
2889 	 */
2890 	switch (amdgpu_asic_reset_method(adev)) {
2891 	case AMD_RESET_METHOD_BACO:
2892 	case AMD_RESET_METHOD_MODE1:
2893 		return true;
2894 	default:
2895 		return false;
2896 	}
2897 }
2898 
2899 /**
2900  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2901  *
2902  * @adev: amdgpu_device pointer
2903  * @state: clockgating state (gate or ungate)
2904  *
2905  * The list of all the hardware IPs that make up the asic is walked and the
2906  * set_clockgating_state callbacks are run.
2907  * Late initialization pass enabling clockgating for hardware IPs.
2908  * Fini or suspend, pass disabling clockgating for hardware IPs.
2909  * Returns 0 on success, negative error code on failure.
2910  */
2911 
2912 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2913 			       enum amd_clockgating_state state)
2914 {
2915 	int i, j, r;
2916 
2917 	if (amdgpu_emu_mode == 1)
2918 		return 0;
2919 
2920 	for (j = 0; j < adev->num_ip_blocks; j++) {
2921 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2922 		if (!adev->ip_blocks[i].status.late_initialized)
2923 			continue;
2924 		/* skip CG for GFX, SDMA on S0ix */
2925 		if (adev->in_s0ix &&
2926 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2927 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2928 			continue;
2929 		/* skip CG for VCE/UVD, it's handled specially */
2930 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2931 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2932 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2933 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2934 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2935 			/* enable clockgating to save power */
2936 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2937 										     state);
2938 			if (r) {
2939 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2940 					  adev->ip_blocks[i].version->funcs->name, r);
2941 				return r;
2942 			}
2943 		}
2944 	}
2945 
2946 	return 0;
2947 }
2948 
2949 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2950 			       enum amd_powergating_state state)
2951 {
2952 	int i, j, r;
2953 
2954 	if (amdgpu_emu_mode == 1)
2955 		return 0;
2956 
2957 	for (j = 0; j < adev->num_ip_blocks; j++) {
2958 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2959 		if (!adev->ip_blocks[i].status.late_initialized)
2960 			continue;
2961 		/* skip PG for GFX, SDMA on S0ix */
2962 		if (adev->in_s0ix &&
2963 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2964 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2965 			continue;
2966 		/* skip CG for VCE/UVD, it's handled specially */
2967 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2968 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2969 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2970 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2971 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
2972 			/* enable powergating to save power */
2973 			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2974 											state);
2975 			if (r) {
2976 				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2977 					  adev->ip_blocks[i].version->funcs->name, r);
2978 				return r;
2979 			}
2980 		}
2981 	}
2982 	return 0;
2983 }
2984 
2985 static int amdgpu_device_enable_mgpu_fan_boost(void)
2986 {
2987 	struct amdgpu_gpu_instance *gpu_ins;
2988 	struct amdgpu_device *adev;
2989 	int i, ret = 0;
2990 
2991 	mutex_lock(&mgpu_info.mutex);
2992 
2993 	/*
2994 	 * MGPU fan boost feature should be enabled
2995 	 * only when there are two or more dGPUs in
2996 	 * the system
2997 	 */
2998 	if (mgpu_info.num_dgpu < 2)
2999 		goto out;
3000 
3001 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
3002 		gpu_ins = &(mgpu_info.gpu_ins[i]);
3003 		adev = gpu_ins->adev;
3004 		if (!(adev->flags & AMD_IS_APU) &&
3005 		    !gpu_ins->mgpu_fan_enabled) {
3006 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3007 			if (ret)
3008 				break;
3009 
3010 			gpu_ins->mgpu_fan_enabled = 1;
3011 		}
3012 	}
3013 
3014 out:
3015 	mutex_unlock(&mgpu_info.mutex);
3016 
3017 	return ret;
3018 }
3019 
3020 /**
3021  * amdgpu_device_ip_late_init - run late init for hardware IPs
3022  *
3023  * @adev: amdgpu_device pointer
3024  *
3025  * Late initialization pass for hardware IPs.  The list of all the hardware
3026  * IPs that make up the asic is walked and the late_init callbacks are run.
3027  * late_init covers any special initialization that an IP requires
3028  * after all of the have been initialized or something that needs to happen
3029  * late in the init process.
3030  * Returns 0 on success, negative error code on failure.
3031  */
3032 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3033 {
3034 	struct amdgpu_gpu_instance *gpu_instance;
3035 	int i = 0, r;
3036 
3037 	for (i = 0; i < adev->num_ip_blocks; i++) {
3038 		if (!adev->ip_blocks[i].status.hw)
3039 			continue;
3040 		if (adev->ip_blocks[i].version->funcs->late_init) {
3041 			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
3042 			if (r) {
3043 				DRM_ERROR("late_init of IP block <%s> failed %d\n",
3044 					  adev->ip_blocks[i].version->funcs->name, r);
3045 				return r;
3046 			}
3047 		}
3048 		adev->ip_blocks[i].status.late_initialized = true;
3049 	}
3050 
3051 	r = amdgpu_ras_late_init(adev);
3052 	if (r) {
3053 		DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3054 		return r;
3055 	}
3056 
3057 	amdgpu_ras_set_error_query_ready(adev, true);
3058 
3059 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3060 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3061 
3062 	amdgpu_device_fill_reset_magic(adev);
3063 
3064 	r = amdgpu_device_enable_mgpu_fan_boost();
3065 	if (r)
3066 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3067 
3068 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3069 	if (amdgpu_passthrough(adev) &&
3070 	    ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3071 	     adev->asic_type == CHIP_ALDEBARAN))
3072 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
3073 
3074 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3075 		mutex_lock(&mgpu_info.mutex);
3076 
3077 		/*
3078 		 * Reset device p-state to low as this was booted with high.
3079 		 *
3080 		 * This should be performed only after all devices from the same
3081 		 * hive get initialized.
3082 		 *
3083 		 * However, it's unknown how many device in the hive in advance.
3084 		 * As this is counted one by one during devices initializations.
3085 		 *
3086 		 * So, we wait for all XGMI interlinked devices initialized.
3087 		 * This may bring some delays as those devices may come from
3088 		 * different hives. But that should be OK.
3089 		 */
3090 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3091 			for (i = 0; i < mgpu_info.num_gpu; i++) {
3092 				gpu_instance = &(mgpu_info.gpu_ins[i]);
3093 				if (gpu_instance->adev->flags & AMD_IS_APU)
3094 					continue;
3095 
3096 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3097 						AMDGPU_XGMI_PSTATE_MIN);
3098 				if (r) {
3099 					DRM_ERROR("pstate setting failed (%d).\n", r);
3100 					break;
3101 				}
3102 			}
3103 		}
3104 
3105 		mutex_unlock(&mgpu_info.mutex);
3106 	}
3107 
3108 	return 0;
3109 }
3110 
3111 /**
3112  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3113  *
3114  * @adev: amdgpu_device pointer
3115  *
3116  * For ASICs need to disable SMC first
3117  */
3118 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3119 {
3120 	int i, r;
3121 
3122 	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3123 		return;
3124 
3125 	for (i = 0; i < adev->num_ip_blocks; i++) {
3126 		if (!adev->ip_blocks[i].status.hw)
3127 			continue;
3128 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3129 			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3130 			/* XXX handle errors */
3131 			if (r) {
3132 				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3133 					  adev->ip_blocks[i].version->funcs->name, r);
3134 			}
3135 			adev->ip_blocks[i].status.hw = false;
3136 			break;
3137 		}
3138 	}
3139 }
3140 
3141 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3142 {
3143 	int i, r;
3144 
3145 	for (i = 0; i < adev->num_ip_blocks; i++) {
3146 		if (!adev->ip_blocks[i].version->funcs->early_fini)
3147 			continue;
3148 
3149 		r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
3150 		if (r) {
3151 			DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3152 				  adev->ip_blocks[i].version->funcs->name, r);
3153 		}
3154 	}
3155 
3156 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3157 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3158 
3159 	amdgpu_amdkfd_suspend(adev, false);
3160 
3161 	/* Workaroud for ASICs need to disable SMC first */
3162 	amdgpu_device_smu_fini_early(adev);
3163 
3164 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3165 		if (!adev->ip_blocks[i].status.hw)
3166 			continue;
3167 
3168 		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3169 		/* XXX handle errors */
3170 		if (r) {
3171 			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3172 				  adev->ip_blocks[i].version->funcs->name, r);
3173 		}
3174 
3175 		adev->ip_blocks[i].status.hw = false;
3176 	}
3177 
3178 	if (amdgpu_sriov_vf(adev)) {
3179 		if (amdgpu_virt_release_full_gpu(adev, false))
3180 			DRM_ERROR("failed to release exclusive mode on fini\n");
3181 	}
3182 
3183 	return 0;
3184 }
3185 
3186 /**
3187  * amdgpu_device_ip_fini - run fini for hardware IPs
3188  *
3189  * @adev: amdgpu_device pointer
3190  *
3191  * Main teardown pass for hardware IPs.  The list of all the hardware
3192  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3193  * are run.  hw_fini tears down the hardware associated with each IP
3194  * and sw_fini tears down any software state associated with each IP.
3195  * Returns 0 on success, negative error code on failure.
3196  */
3197 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3198 {
3199 	int i, r;
3200 
3201 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3202 		amdgpu_virt_release_ras_err_handler_data(adev);
3203 
3204 	if (adev->gmc.xgmi.num_physical_nodes > 1)
3205 		amdgpu_xgmi_remove_device(adev);
3206 
3207 	amdgpu_amdkfd_device_fini_sw(adev);
3208 
3209 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3210 		if (!adev->ip_blocks[i].status.sw)
3211 			continue;
3212 
3213 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3214 			amdgpu_ucode_free_bo(adev);
3215 			amdgpu_free_static_csa(&adev->virt.csa_obj);
3216 			amdgpu_device_wb_fini(adev);
3217 			amdgpu_device_mem_scratch_fini(adev);
3218 			amdgpu_ib_pool_fini(adev);
3219 			amdgpu_seq64_fini(adev);
3220 		}
3221 
3222 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3223 		/* XXX handle errors */
3224 		if (r) {
3225 			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3226 				  adev->ip_blocks[i].version->funcs->name, r);
3227 		}
3228 		adev->ip_blocks[i].status.sw = false;
3229 		adev->ip_blocks[i].status.valid = false;
3230 	}
3231 
3232 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3233 		if (!adev->ip_blocks[i].status.late_initialized)
3234 			continue;
3235 		if (adev->ip_blocks[i].version->funcs->late_fini)
3236 			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3237 		adev->ip_blocks[i].status.late_initialized = false;
3238 	}
3239 
3240 	amdgpu_ras_fini(adev);
3241 
3242 	return 0;
3243 }
3244 
3245 /**
3246  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3247  *
3248  * @work: work_struct.
3249  */
3250 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3251 {
3252 	struct amdgpu_device *adev =
3253 		container_of(work, struct amdgpu_device, delayed_init_work.work);
3254 	int r;
3255 
3256 	r = amdgpu_ib_ring_tests(adev);
3257 	if (r)
3258 		DRM_ERROR("ib ring test failed (%d).\n", r);
3259 }
3260 
3261 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3262 {
3263 	struct amdgpu_device *adev =
3264 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3265 
3266 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3267 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3268 
3269 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3270 		adev->gfx.gfx_off_state = true;
3271 }
3272 
3273 /**
3274  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3275  *
3276  * @adev: amdgpu_device pointer
3277  *
3278  * Main suspend function for hardware IPs.  The list of all the hardware
3279  * IPs that make up the asic is walked, clockgating is disabled and the
3280  * suspend callbacks are run.  suspend puts the hardware and software state
3281  * in each IP into a state suitable for suspend.
3282  * Returns 0 on success, negative error code on failure.
3283  */
3284 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3285 {
3286 	int i, r;
3287 
3288 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3289 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3290 
3291 	/*
3292 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
3293 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3294 	 * scenario. Add the missing df cstate disablement here.
3295 	 */
3296 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3297 		dev_warn(adev->dev, "Failed to disallow df cstate");
3298 
3299 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3300 		if (!adev->ip_blocks[i].status.valid)
3301 			continue;
3302 
3303 		/* displays are handled separately */
3304 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3305 			continue;
3306 
3307 		/* XXX handle errors */
3308 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
3309 		/* XXX handle errors */
3310 		if (r) {
3311 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
3312 				  adev->ip_blocks[i].version->funcs->name, r);
3313 			return r;
3314 		}
3315 
3316 		adev->ip_blocks[i].status.hw = false;
3317 	}
3318 
3319 	return 0;
3320 }
3321 
3322 /**
3323  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3324  *
3325  * @adev: amdgpu_device pointer
3326  *
3327  * Main suspend function for hardware IPs.  The list of all the hardware
3328  * IPs that make up the asic is walked, clockgating is disabled and the
3329  * suspend callbacks are run.  suspend puts the hardware and software state
3330  * in each IP into a state suitable for suspend.
3331  * Returns 0 on success, negative error code on failure.
3332  */
3333 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3334 {
3335 	int i, r;
3336 
3337 	if (adev->in_s0ix)
3338 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3339 
3340 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3341 		if (!adev->ip_blocks[i].status.valid)
3342 			continue;
3343 		/* displays are handled in phase1 */
3344 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3345 			continue;
3346 		/* PSP lost connection when err_event_athub occurs */
3347 		if (amdgpu_ras_intr_triggered() &&
3348 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3349 			adev->ip_blocks[i].status.hw = false;
3350 			continue;
3351 		}
3352 
3353 		/* skip unnecessary suspend if we do not initialize them yet */
3354 		if (adev->gmc.xgmi.pending_reset &&
3355 		    !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3356 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3357 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3358 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3359 			adev->ip_blocks[i].status.hw = false;
3360 			continue;
3361 		}
3362 
3363 		/* skip suspend of gfx/mes and psp for S0ix
3364 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3365 		 * like at runtime. PSP is also part of the always on hardware
3366 		 * so no need to suspend it.
3367 		 */
3368 		if (adev->in_s0ix &&
3369 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3370 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3371 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3372 			continue;
3373 
3374 		/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3375 		if (adev->in_s0ix &&
3376 		    (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3377 		     IP_VERSION(5, 0, 0)) &&
3378 		    (adev->ip_blocks[i].version->type ==
3379 		     AMD_IP_BLOCK_TYPE_SDMA))
3380 			continue;
3381 
3382 		/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3383 		 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3384 		 * from this location and RLC Autoload automatically also gets loaded
3385 		 * from here based on PMFW -> PSP message during re-init sequence.
3386 		 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3387 		 * the TMR and reload FWs again for IMU enabled APU ASICs.
3388 		 */
3389 		if (amdgpu_in_reset(adev) &&
3390 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3391 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3392 			continue;
3393 
3394 		/* XXX handle errors */
3395 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
3396 		/* XXX handle errors */
3397 		if (r) {
3398 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
3399 				  adev->ip_blocks[i].version->funcs->name, r);
3400 		}
3401 		adev->ip_blocks[i].status.hw = false;
3402 		/* handle putting the SMC in the appropriate state */
3403 		if (!amdgpu_sriov_vf(adev)) {
3404 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3405 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3406 				if (r) {
3407 					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3408 							adev->mp1_state, r);
3409 					return r;
3410 				}
3411 			}
3412 		}
3413 	}
3414 
3415 	return 0;
3416 }
3417 
3418 /**
3419  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3420  *
3421  * @adev: amdgpu_device pointer
3422  *
3423  * Main suspend function for hardware IPs.  The list of all the hardware
3424  * IPs that make up the asic is walked, clockgating is disabled and the
3425  * suspend callbacks are run.  suspend puts the hardware and software state
3426  * in each IP into a state suitable for suspend.
3427  * Returns 0 on success, negative error code on failure.
3428  */
3429 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3430 {
3431 	int r;
3432 
3433 	if (amdgpu_sriov_vf(adev)) {
3434 		amdgpu_virt_fini_data_exchange(adev);
3435 		amdgpu_virt_request_full_gpu(adev, false);
3436 	}
3437 
3438 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
3439 
3440 	r = amdgpu_device_ip_suspend_phase1(adev);
3441 	if (r)
3442 		return r;
3443 	r = amdgpu_device_ip_suspend_phase2(adev);
3444 
3445 	if (amdgpu_sriov_vf(adev))
3446 		amdgpu_virt_release_full_gpu(adev, false);
3447 
3448 	return r;
3449 }
3450 
3451 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3452 {
3453 	int i, r;
3454 
3455 	static enum amd_ip_block_type ip_order[] = {
3456 		AMD_IP_BLOCK_TYPE_COMMON,
3457 		AMD_IP_BLOCK_TYPE_GMC,
3458 		AMD_IP_BLOCK_TYPE_PSP,
3459 		AMD_IP_BLOCK_TYPE_IH,
3460 	};
3461 
3462 	for (i = 0; i < adev->num_ip_blocks; i++) {
3463 		int j;
3464 		struct amdgpu_ip_block *block;
3465 
3466 		block = &adev->ip_blocks[i];
3467 		block->status.hw = false;
3468 
3469 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3470 
3471 			if (block->version->type != ip_order[j] ||
3472 				!block->status.valid)
3473 				continue;
3474 
3475 			r = block->version->funcs->hw_init(adev);
3476 			DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3477 			if (r)
3478 				return r;
3479 			block->status.hw = true;
3480 		}
3481 	}
3482 
3483 	return 0;
3484 }
3485 
3486 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3487 {
3488 	int i, r;
3489 
3490 	static enum amd_ip_block_type ip_order[] = {
3491 		AMD_IP_BLOCK_TYPE_SMC,
3492 		AMD_IP_BLOCK_TYPE_DCE,
3493 		AMD_IP_BLOCK_TYPE_GFX,
3494 		AMD_IP_BLOCK_TYPE_SDMA,
3495 		AMD_IP_BLOCK_TYPE_MES,
3496 		AMD_IP_BLOCK_TYPE_UVD,
3497 		AMD_IP_BLOCK_TYPE_VCE,
3498 		AMD_IP_BLOCK_TYPE_VCN,
3499 		AMD_IP_BLOCK_TYPE_JPEG
3500 	};
3501 
3502 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3503 		int j;
3504 		struct amdgpu_ip_block *block;
3505 
3506 		for (j = 0; j < adev->num_ip_blocks; j++) {
3507 			block = &adev->ip_blocks[j];
3508 
3509 			if (block->version->type != ip_order[i] ||
3510 				!block->status.valid ||
3511 				block->status.hw)
3512 				continue;
3513 
3514 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3515 				r = block->version->funcs->resume(adev);
3516 			else
3517 				r = block->version->funcs->hw_init(adev);
3518 
3519 			DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3520 			if (r)
3521 				return r;
3522 			block->status.hw = true;
3523 		}
3524 	}
3525 
3526 	return 0;
3527 }
3528 
3529 /**
3530  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3531  *
3532  * @adev: amdgpu_device pointer
3533  *
3534  * First resume function for hardware IPs.  The list of all the hardware
3535  * IPs that make up the asic is walked and the resume callbacks are run for
3536  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3537  * after a suspend and updates the software state as necessary.  This
3538  * function is also used for restoring the GPU after a GPU reset.
3539  * Returns 0 on success, negative error code on failure.
3540  */
3541 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3542 {
3543 	int i, r;
3544 
3545 	for (i = 0; i < adev->num_ip_blocks; i++) {
3546 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3547 			continue;
3548 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3549 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3550 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3551 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3552 
3553 			r = adev->ip_blocks[i].version->funcs->resume(adev);
3554 			if (r) {
3555 				DRM_ERROR("resume of IP block <%s> failed %d\n",
3556 					  adev->ip_blocks[i].version->funcs->name, r);
3557 				return r;
3558 			}
3559 			adev->ip_blocks[i].status.hw = true;
3560 		}
3561 	}
3562 
3563 	return 0;
3564 }
3565 
3566 /**
3567  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3568  *
3569  * @adev: amdgpu_device pointer
3570  *
3571  * First resume function for hardware IPs.  The list of all the hardware
3572  * IPs that make up the asic is walked and the resume callbacks are run for
3573  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3574  * functional state after a suspend and updates the software state as
3575  * necessary.  This function is also used for restoring the GPU after a GPU
3576  * reset.
3577  * Returns 0 on success, negative error code on failure.
3578  */
3579 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3580 {
3581 	int i, r;
3582 
3583 	for (i = 0; i < adev->num_ip_blocks; i++) {
3584 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3585 			continue;
3586 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3587 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3588 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3589 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3590 			continue;
3591 		r = adev->ip_blocks[i].version->funcs->resume(adev);
3592 		if (r) {
3593 			DRM_ERROR("resume of IP block <%s> failed %d\n",
3594 				  adev->ip_blocks[i].version->funcs->name, r);
3595 			return r;
3596 		}
3597 		adev->ip_blocks[i].status.hw = true;
3598 	}
3599 
3600 	return 0;
3601 }
3602 
3603 /**
3604  * amdgpu_device_ip_resume - run resume for hardware IPs
3605  *
3606  * @adev: amdgpu_device pointer
3607  *
3608  * Main resume function for hardware IPs.  The hardware IPs
3609  * are split into two resume functions because they are
3610  * also used in recovering from a GPU reset and some additional
3611  * steps need to be take between them.  In this case (S3/S4) they are
3612  * run sequentially.
3613  * Returns 0 on success, negative error code on failure.
3614  */
3615 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3616 {
3617 	int r;
3618 
3619 	r = amdgpu_device_ip_resume_phase1(adev);
3620 	if (r)
3621 		return r;
3622 
3623 	r = amdgpu_device_fw_loading(adev);
3624 	if (r)
3625 		return r;
3626 
3627 	r = amdgpu_device_ip_resume_phase2(adev);
3628 
3629 	if (adev->mman.buffer_funcs_ring->sched.ready)
3630 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
3631 
3632 	return r;
3633 }
3634 
3635 /**
3636  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3637  *
3638  * @adev: amdgpu_device pointer
3639  *
3640  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3641  */
3642 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3643 {
3644 	if (amdgpu_sriov_vf(adev)) {
3645 		if (adev->is_atom_fw) {
3646 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3647 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3648 		} else {
3649 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3650 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3651 		}
3652 
3653 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3654 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3655 	}
3656 }
3657 
3658 /**
3659  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3660  *
3661  * @asic_type: AMD asic type
3662  *
3663  * Check if there is DC (new modesetting infrastructre) support for an asic.
3664  * returns true if DC has support, false if not.
3665  */
3666 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3667 {
3668 	switch (asic_type) {
3669 #ifdef CONFIG_DRM_AMDGPU_SI
3670 	case CHIP_HAINAN:
3671 #endif
3672 	case CHIP_TOPAZ:
3673 		/* chips with no display hardware */
3674 		return false;
3675 #if defined(CONFIG_DRM_AMD_DC)
3676 	case CHIP_TAHITI:
3677 	case CHIP_PITCAIRN:
3678 	case CHIP_VERDE:
3679 	case CHIP_OLAND:
3680 		/*
3681 		 * We have systems in the wild with these ASICs that require
3682 		 * LVDS and VGA support which is not supported with DC.
3683 		 *
3684 		 * Fallback to the non-DC driver here by default so as not to
3685 		 * cause regressions.
3686 		 */
3687 #if defined(CONFIG_DRM_AMD_DC_SI)
3688 		return amdgpu_dc > 0;
3689 #else
3690 		return false;
3691 #endif
3692 	case CHIP_BONAIRE:
3693 	case CHIP_KAVERI:
3694 	case CHIP_KABINI:
3695 	case CHIP_MULLINS:
3696 		/*
3697 		 * We have systems in the wild with these ASICs that require
3698 		 * VGA support which is not supported with DC.
3699 		 *
3700 		 * Fallback to the non-DC driver here by default so as not to
3701 		 * cause regressions.
3702 		 */
3703 		return amdgpu_dc > 0;
3704 	default:
3705 		return amdgpu_dc != 0;
3706 #else
3707 	default:
3708 		if (amdgpu_dc > 0)
3709 			DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3710 		return false;
3711 #endif
3712 	}
3713 }
3714 
3715 /**
3716  * amdgpu_device_has_dc_support - check if dc is supported
3717  *
3718  * @adev: amdgpu_device pointer
3719  *
3720  * Returns true for supported, false for not supported
3721  */
3722 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3723 {
3724 	if (adev->enable_virtual_display ||
3725 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3726 		return false;
3727 
3728 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
3729 }
3730 
3731 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3732 {
3733 	struct amdgpu_device *adev =
3734 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
3735 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3736 
3737 	/* It's a bug to not have a hive within this function */
3738 	if (WARN_ON(!hive))
3739 		return;
3740 
3741 	/*
3742 	 * Use task barrier to synchronize all xgmi reset works across the
3743 	 * hive. task_barrier_enter and task_barrier_exit will block
3744 	 * until all the threads running the xgmi reset works reach
3745 	 * those points. task_barrier_full will do both blocks.
3746 	 */
3747 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3748 
3749 		task_barrier_enter(&hive->tb);
3750 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3751 
3752 		if (adev->asic_reset_res)
3753 			goto fail;
3754 
3755 		task_barrier_exit(&hive->tb);
3756 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3757 
3758 		if (adev->asic_reset_res)
3759 			goto fail;
3760 
3761 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3762 	} else {
3763 
3764 		task_barrier_full(&hive->tb);
3765 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
3766 	}
3767 
3768 fail:
3769 	if (adev->asic_reset_res)
3770 		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3771 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
3772 	amdgpu_put_xgmi_hive(hive);
3773 }
3774 
3775 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3776 {
3777 	char *input = amdgpu_lockup_timeout;
3778 	char *timeout_setting = NULL;
3779 	int index = 0;
3780 	long timeout;
3781 	int ret = 0;
3782 
3783 	/*
3784 	 * By default timeout for non compute jobs is 10000
3785 	 * and 60000 for compute jobs.
3786 	 * In SR-IOV or passthrough mode, timeout for compute
3787 	 * jobs are 60000 by default.
3788 	 */
3789 	adev->gfx_timeout = msecs_to_jiffies(10000);
3790 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3791 	if (amdgpu_sriov_vf(adev))
3792 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3793 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3794 	else
3795 		adev->compute_timeout =  msecs_to_jiffies(60000);
3796 
3797 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3798 		while ((timeout_setting = strsep(&input, ",")) &&
3799 				strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3800 			ret = kstrtol(timeout_setting, 0, &timeout);
3801 			if (ret)
3802 				return ret;
3803 
3804 			if (timeout == 0) {
3805 				index++;
3806 				continue;
3807 			} else if (timeout < 0) {
3808 				timeout = MAX_SCHEDULE_TIMEOUT;
3809 				dev_warn(adev->dev, "lockup timeout disabled");
3810 				add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3811 			} else {
3812 				timeout = msecs_to_jiffies(timeout);
3813 			}
3814 
3815 			switch (index++) {
3816 			case 0:
3817 				adev->gfx_timeout = timeout;
3818 				break;
3819 			case 1:
3820 				adev->compute_timeout = timeout;
3821 				break;
3822 			case 2:
3823 				adev->sdma_timeout = timeout;
3824 				break;
3825 			case 3:
3826 				adev->video_timeout = timeout;
3827 				break;
3828 			default:
3829 				break;
3830 			}
3831 		}
3832 		/*
3833 		 * There is only one value specified and
3834 		 * it should apply to all non-compute jobs.
3835 		 */
3836 		if (index == 1) {
3837 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3838 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3839 				adev->compute_timeout = adev->gfx_timeout;
3840 		}
3841 	}
3842 
3843 	return ret;
3844 }
3845 
3846 /**
3847  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3848  *
3849  * @adev: amdgpu_device pointer
3850  *
3851  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3852  */
3853 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3854 {
3855 	struct iommu_domain *domain;
3856 
3857 	domain = iommu_get_domain_for_dev(adev->dev);
3858 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3859 		adev->ram_is_direct_mapped = true;
3860 }
3861 
3862 static const struct attribute *amdgpu_dev_attributes[] = {
3863 	&dev_attr_pcie_replay_count.attr,
3864 	NULL
3865 };
3866 
3867 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3868 {
3869 	if (amdgpu_mcbp == 1)
3870 		adev->gfx.mcbp = true;
3871 	else if (amdgpu_mcbp == 0)
3872 		adev->gfx.mcbp = false;
3873 
3874 	if (amdgpu_sriov_vf(adev))
3875 		adev->gfx.mcbp = true;
3876 
3877 	if (adev->gfx.mcbp)
3878 		DRM_INFO("MCBP is enabled\n");
3879 }
3880 
3881 /**
3882  * amdgpu_device_init - initialize the driver
3883  *
3884  * @adev: amdgpu_device pointer
3885  * @flags: driver flags
3886  *
3887  * Initializes the driver info and hw (all asics).
3888  * Returns 0 for success or an error on failure.
3889  * Called at driver startup.
3890  */
3891 int amdgpu_device_init(struct amdgpu_device *adev,
3892 		       uint32_t flags)
3893 {
3894 	struct drm_device *ddev = adev_to_drm(adev);
3895 	struct pci_dev *pdev = adev->pdev;
3896 	int r, i;
3897 	bool px = false;
3898 	u32 max_MBps;
3899 	int tmp;
3900 
3901 	adev->shutdown = false;
3902 	adev->flags = flags;
3903 
3904 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3905 		adev->asic_type = amdgpu_force_asic_type;
3906 	else
3907 		adev->asic_type = flags & AMD_ASIC_MASK;
3908 
3909 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3910 	if (amdgpu_emu_mode == 1)
3911 		adev->usec_timeout *= 10;
3912 	adev->gmc.gart_size = 512 * 1024 * 1024;
3913 	adev->accel_working = false;
3914 	adev->num_rings = 0;
3915 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3916 	adev->mman.buffer_funcs = NULL;
3917 	adev->mman.buffer_funcs_ring = NULL;
3918 	adev->vm_manager.vm_pte_funcs = NULL;
3919 	adev->vm_manager.vm_pte_num_scheds = 0;
3920 	adev->gmc.gmc_funcs = NULL;
3921 	adev->harvest_ip_mask = 0x0;
3922 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3923 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3924 
3925 	adev->smc_rreg = &amdgpu_invalid_rreg;
3926 	adev->smc_wreg = &amdgpu_invalid_wreg;
3927 	adev->pcie_rreg = &amdgpu_invalid_rreg;
3928 	adev->pcie_wreg = &amdgpu_invalid_wreg;
3929 	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3930 	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3931 	adev->pciep_rreg = &amdgpu_invalid_rreg;
3932 	adev->pciep_wreg = &amdgpu_invalid_wreg;
3933 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3934 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3935 	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3936 	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3937 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3938 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3939 	adev->didt_rreg = &amdgpu_invalid_rreg;
3940 	adev->didt_wreg = &amdgpu_invalid_wreg;
3941 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3942 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3943 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3944 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3945 
3946 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3947 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3948 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3949 
3950 	/* mutex initialization are all done here so we
3951 	 * can recall function without having locking issues
3952 	 */
3953 	mutex_init(&adev->firmware.mutex);
3954 	mutex_init(&adev->pm.mutex);
3955 	mutex_init(&adev->gfx.gpu_clock_mutex);
3956 	mutex_init(&adev->srbm_mutex);
3957 	mutex_init(&adev->gfx.pipe_reserve_mutex);
3958 	mutex_init(&adev->gfx.gfx_off_mutex);
3959 	mutex_init(&adev->gfx.partition_mutex);
3960 	mutex_init(&adev->grbm_idx_mutex);
3961 	mutex_init(&adev->mn_lock);
3962 	mutex_init(&adev->virt.vf_errors.lock);
3963 	hash_init(adev->mn_hash);
3964 	mutex_init(&adev->psp.mutex);
3965 	mutex_init(&adev->notifier_lock);
3966 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3967 	mutex_init(&adev->benchmark_mutex);
3968 
3969 	amdgpu_device_init_apu_flags(adev);
3970 
3971 	r = amdgpu_device_check_arguments(adev);
3972 	if (r)
3973 		return r;
3974 
3975 	spin_lock_init(&adev->mmio_idx_lock);
3976 	spin_lock_init(&adev->smc_idx_lock);
3977 	spin_lock_init(&adev->pcie_idx_lock);
3978 	spin_lock_init(&adev->uvd_ctx_idx_lock);
3979 	spin_lock_init(&adev->didt_idx_lock);
3980 	spin_lock_init(&adev->gc_cac_idx_lock);
3981 	spin_lock_init(&adev->se_cac_idx_lock);
3982 	spin_lock_init(&adev->audio_endpt_idx_lock);
3983 	spin_lock_init(&adev->mm_stats.lock);
3984 
3985 	INIT_LIST_HEAD(&adev->shadow_list);
3986 	mutex_init(&adev->shadow_list_lock);
3987 
3988 	INIT_LIST_HEAD(&adev->reset_list);
3989 
3990 	INIT_LIST_HEAD(&adev->ras_list);
3991 
3992 	INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3993 
3994 	INIT_DELAYED_WORK(&adev->delayed_init_work,
3995 			  amdgpu_device_delayed_init_work_handler);
3996 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3997 			  amdgpu_device_delay_enable_gfx_off);
3998 
3999 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4000 
4001 	adev->gfx.gfx_off_req_count = 1;
4002 	adev->gfx.gfx_off_residency = 0;
4003 	adev->gfx.gfx_off_entrycount = 0;
4004 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4005 
4006 	atomic_set(&adev->throttling_logging_enabled, 1);
4007 	/*
4008 	 * If throttling continues, logging will be performed every minute
4009 	 * to avoid log flooding. "-1" is subtracted since the thermal
4010 	 * throttling interrupt comes every second. Thus, the total logging
4011 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
4012 	 * for throttling interrupt) = 60 seconds.
4013 	 */
4014 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4015 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4016 
4017 	/* Registers mapping */
4018 	/* TODO: block userspace mapping of io register */
4019 	if (adev->asic_type >= CHIP_BONAIRE) {
4020 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4021 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4022 	} else {
4023 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4024 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4025 	}
4026 
4027 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4028 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4029 
4030 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4031 	if (!adev->rmmio)
4032 		return -ENOMEM;
4033 
4034 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4035 	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4036 
4037 	/*
4038 	 * Reset domain needs to be present early, before XGMI hive discovered
4039 	 * (if any) and intitialized to use reset sem and in_gpu reset flag
4040 	 * early on during init and before calling to RREG32.
4041 	 */
4042 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4043 	if (!adev->reset_domain)
4044 		return -ENOMEM;
4045 
4046 	/* detect hw virtualization here */
4047 	amdgpu_detect_virtualization(adev);
4048 
4049 	amdgpu_device_get_pcie_info(adev);
4050 
4051 	r = amdgpu_device_get_job_timeout_settings(adev);
4052 	if (r) {
4053 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4054 		return r;
4055 	}
4056 
4057 	amdgpu_device_set_mcbp(adev);
4058 
4059 	/* early init functions */
4060 	r = amdgpu_device_ip_early_init(adev);
4061 	if (r)
4062 		return r;
4063 
4064 	/* Get rid of things like offb */
4065 	r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
4066 	if (r)
4067 		return r;
4068 
4069 	/* Enable TMZ based on IP_VERSION */
4070 	amdgpu_gmc_tmz_set(adev);
4071 
4072 	amdgpu_gmc_noretry_set(adev);
4073 	/* Need to get xgmi info early to decide the reset behavior*/
4074 	if (adev->gmc.xgmi.supported) {
4075 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
4076 		if (r)
4077 			return r;
4078 	}
4079 
4080 	/* enable PCIE atomic ops */
4081 	if (amdgpu_sriov_vf(adev)) {
4082 		if (adev->virt.fw_reserve.p_pf2vf)
4083 			adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4084 						      adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4085 				(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4086 	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4087 	 * internal path natively support atomics, set have_atomics_support to true.
4088 	 */
4089 	} else if ((adev->flags & AMD_IS_APU) &&
4090 		   (amdgpu_ip_version(adev, GC_HWIP, 0) >
4091 		    IP_VERSION(9, 0, 0))) {
4092 		adev->have_atomics_support = true;
4093 	} else {
4094 		adev->have_atomics_support =
4095 			!pci_enable_atomic_ops_to_root(adev->pdev,
4096 					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4097 					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4098 	}
4099 
4100 	if (!adev->have_atomics_support)
4101 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4102 
4103 	/* doorbell bar mapping and doorbell index init*/
4104 	amdgpu_doorbell_init(adev);
4105 
4106 	if (amdgpu_emu_mode == 1) {
4107 		/* post the asic on emulation mode */
4108 		emu_soc_asic_init(adev);
4109 		goto fence_driver_init;
4110 	}
4111 
4112 	amdgpu_reset_init(adev);
4113 
4114 	/* detect if we are with an SRIOV vbios */
4115 	if (adev->bios)
4116 		amdgpu_device_detect_sriov_bios(adev);
4117 
4118 	/* check if we need to reset the asic
4119 	 *  E.g., driver was not cleanly unloaded previously, etc.
4120 	 */
4121 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4122 		if (adev->gmc.xgmi.num_physical_nodes) {
4123 			dev_info(adev->dev, "Pending hive reset.\n");
4124 			adev->gmc.xgmi.pending_reset = true;
4125 			/* Only need to init necessary block for SMU to handle the reset */
4126 			for (i = 0; i < adev->num_ip_blocks; i++) {
4127 				if (!adev->ip_blocks[i].status.valid)
4128 					continue;
4129 				if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
4130 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
4131 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
4132 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
4133 					DRM_DEBUG("IP %s disabled for hw_init.\n",
4134 						adev->ip_blocks[i].version->funcs->name);
4135 					adev->ip_blocks[i].status.hw = true;
4136 				}
4137 			}
4138 		} else {
4139 			tmp = amdgpu_reset_method;
4140 			/* It should do a default reset when loading or reloading the driver,
4141 			 * regardless of the module parameter reset_method.
4142 			 */
4143 			amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4144 			r = amdgpu_asic_reset(adev);
4145 			amdgpu_reset_method = tmp;
4146 			if (r) {
4147 				dev_err(adev->dev, "asic reset on init failed\n");
4148 				goto failed;
4149 			}
4150 		}
4151 	}
4152 
4153 	/* Post card if necessary */
4154 	if (amdgpu_device_need_post(adev)) {
4155 		if (!adev->bios) {
4156 			dev_err(adev->dev, "no vBIOS found\n");
4157 			r = -EINVAL;
4158 			goto failed;
4159 		}
4160 		DRM_INFO("GPU posting now...\n");
4161 		r = amdgpu_device_asic_init(adev);
4162 		if (r) {
4163 			dev_err(adev->dev, "gpu post error!\n");
4164 			goto failed;
4165 		}
4166 	}
4167 
4168 	if (adev->bios) {
4169 		if (adev->is_atom_fw) {
4170 			/* Initialize clocks */
4171 			r = amdgpu_atomfirmware_get_clock_info(adev);
4172 			if (r) {
4173 				dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4174 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4175 				goto failed;
4176 			}
4177 		} else {
4178 			/* Initialize clocks */
4179 			r = amdgpu_atombios_get_clock_info(adev);
4180 			if (r) {
4181 				dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4182 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4183 				goto failed;
4184 			}
4185 			/* init i2c buses */
4186 			if (!amdgpu_device_has_dc_support(adev))
4187 				amdgpu_atombios_i2c_init(adev);
4188 		}
4189 	}
4190 
4191 fence_driver_init:
4192 	/* Fence driver */
4193 	r = amdgpu_fence_driver_sw_init(adev);
4194 	if (r) {
4195 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4196 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4197 		goto failed;
4198 	}
4199 
4200 	/* init the mode config */
4201 	drm_mode_config_init(adev_to_drm(adev));
4202 
4203 	r = amdgpu_device_ip_init(adev);
4204 	if (r) {
4205 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4206 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4207 		goto release_ras_con;
4208 	}
4209 
4210 	amdgpu_fence_driver_hw_init(adev);
4211 
4212 	dev_info(adev->dev,
4213 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4214 			adev->gfx.config.max_shader_engines,
4215 			adev->gfx.config.max_sh_per_se,
4216 			adev->gfx.config.max_cu_per_sh,
4217 			adev->gfx.cu_info.number);
4218 
4219 	adev->accel_working = true;
4220 
4221 	amdgpu_vm_check_compute_bug(adev);
4222 
4223 	/* Initialize the buffer migration limit. */
4224 	if (amdgpu_moverate >= 0)
4225 		max_MBps = amdgpu_moverate;
4226 	else
4227 		max_MBps = 8; /* Allow 8 MB/s. */
4228 	/* Get a log2 for easy divisions. */
4229 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4230 
4231 	/*
4232 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4233 	 * Otherwise the mgpu fan boost feature will be skipped due to the
4234 	 * gpu instance is counted less.
4235 	 */
4236 	amdgpu_register_gpu_instance(adev);
4237 
4238 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4239 	 * explicit gating rather than handling it automatically.
4240 	 */
4241 	if (!adev->gmc.xgmi.pending_reset) {
4242 		r = amdgpu_device_ip_late_init(adev);
4243 		if (r) {
4244 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4245 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4246 			goto release_ras_con;
4247 		}
4248 		/* must succeed. */
4249 		amdgpu_ras_resume(adev);
4250 		queue_delayed_work(system_wq, &adev->delayed_init_work,
4251 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
4252 	}
4253 
4254 	if (amdgpu_sriov_vf(adev)) {
4255 		amdgpu_virt_release_full_gpu(adev, true);
4256 		flush_delayed_work(&adev->delayed_init_work);
4257 	}
4258 
4259 	/*
4260 	 * Place those sysfs registering after `late_init`. As some of those
4261 	 * operations performed in `late_init` might affect the sysfs
4262 	 * interfaces creating.
4263 	 */
4264 	r = amdgpu_atombios_sysfs_init(adev);
4265 	if (r)
4266 		drm_err(&adev->ddev,
4267 			"registering atombios sysfs failed (%d).\n", r);
4268 
4269 	r = amdgpu_pm_sysfs_init(adev);
4270 	if (r)
4271 		DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4272 
4273 	r = amdgpu_ucode_sysfs_init(adev);
4274 	if (r) {
4275 		adev->ucode_sysfs_en = false;
4276 		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4277 	} else
4278 		adev->ucode_sysfs_en = true;
4279 
4280 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4281 	if (r)
4282 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
4283 
4284 	r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4285 	if (r)
4286 		dev_err(adev->dev,
4287 			"Could not create amdgpu board attributes\n");
4288 
4289 	amdgpu_fru_sysfs_init(adev);
4290 	amdgpu_reg_state_sysfs_init(adev);
4291 
4292 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4293 		r = amdgpu_pmu_init(adev);
4294 	if (r)
4295 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4296 
4297 	/* Have stored pci confspace at hand for restore in sudden PCI error */
4298 	if (amdgpu_device_cache_pci_state(adev->pdev))
4299 		pci_restore_state(pdev);
4300 
4301 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4302 	/* this will fail for cards that aren't VGA class devices, just
4303 	 * ignore it
4304 	 */
4305 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4306 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4307 
4308 	px = amdgpu_device_supports_px(ddev);
4309 
4310 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4311 				apple_gmux_detect(NULL, NULL)))
4312 		vga_switcheroo_register_client(adev->pdev,
4313 					       &amdgpu_switcheroo_ops, px);
4314 
4315 	if (px)
4316 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4317 
4318 	if (adev->gmc.xgmi.pending_reset)
4319 		queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4320 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
4321 
4322 	amdgpu_device_check_iommu_direct_map(adev);
4323 
4324 	return 0;
4325 
4326 release_ras_con:
4327 	if (amdgpu_sriov_vf(adev))
4328 		amdgpu_virt_release_full_gpu(adev, true);
4329 
4330 	/* failed in exclusive mode due to timeout */
4331 	if (amdgpu_sriov_vf(adev) &&
4332 		!amdgpu_sriov_runtime(adev) &&
4333 		amdgpu_virt_mmio_blocked(adev) &&
4334 		!amdgpu_virt_wait_reset(adev)) {
4335 		dev_err(adev->dev, "VF exclusive mode timeout\n");
4336 		/* Don't send request since VF is inactive. */
4337 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4338 		adev->virt.ops = NULL;
4339 		r = -EAGAIN;
4340 	}
4341 	amdgpu_release_ras_context(adev);
4342 
4343 failed:
4344 	amdgpu_vf_error_trans_all(adev);
4345 
4346 	return r;
4347 }
4348 
4349 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4350 {
4351 
4352 	/* Clear all CPU mappings pointing to this device */
4353 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4354 
4355 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4356 	amdgpu_doorbell_fini(adev);
4357 
4358 	iounmap(adev->rmmio);
4359 	adev->rmmio = NULL;
4360 	if (adev->mman.aper_base_kaddr)
4361 		iounmap(adev->mman.aper_base_kaddr);
4362 	adev->mman.aper_base_kaddr = NULL;
4363 
4364 	/* Memory manager related */
4365 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4366 		arch_phys_wc_del(adev->gmc.vram_mtrr);
4367 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4368 	}
4369 }
4370 
4371 /**
4372  * amdgpu_device_fini_hw - tear down the driver
4373  *
4374  * @adev: amdgpu_device pointer
4375  *
4376  * Tear down the driver info (all asics).
4377  * Called at driver shutdown.
4378  */
4379 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4380 {
4381 	dev_info(adev->dev, "amdgpu: finishing device.\n");
4382 	flush_delayed_work(&adev->delayed_init_work);
4383 	adev->shutdown = true;
4384 
4385 	/* make sure IB test finished before entering exclusive mode
4386 	 * to avoid preemption on IB test
4387 	 */
4388 	if (amdgpu_sriov_vf(adev)) {
4389 		amdgpu_virt_request_full_gpu(adev, false);
4390 		amdgpu_virt_fini_data_exchange(adev);
4391 	}
4392 
4393 	/* disable all interrupts */
4394 	amdgpu_irq_disable_all(adev);
4395 	if (adev->mode_info.mode_config_initialized) {
4396 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4397 			drm_helper_force_disable_all(adev_to_drm(adev));
4398 		else
4399 			drm_atomic_helper_shutdown(adev_to_drm(adev));
4400 	}
4401 	amdgpu_fence_driver_hw_fini(adev);
4402 
4403 	if (adev->mman.initialized)
4404 		drain_workqueue(adev->mman.bdev.wq);
4405 
4406 	if (adev->pm.sysfs_initialized)
4407 		amdgpu_pm_sysfs_fini(adev);
4408 	if (adev->ucode_sysfs_en)
4409 		amdgpu_ucode_sysfs_fini(adev);
4410 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4411 	amdgpu_fru_sysfs_fini(adev);
4412 
4413 	amdgpu_reg_state_sysfs_fini(adev);
4414 
4415 	/* disable ras feature must before hw fini */
4416 	amdgpu_ras_pre_fini(adev);
4417 
4418 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4419 
4420 	amdgpu_device_ip_fini_early(adev);
4421 
4422 	amdgpu_irq_fini_hw(adev);
4423 
4424 	if (adev->mman.initialized)
4425 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4426 
4427 	amdgpu_gart_dummy_page_fini(adev);
4428 
4429 	if (drm_dev_is_unplugged(adev_to_drm(adev)))
4430 		amdgpu_device_unmap_mmio(adev);
4431 
4432 }
4433 
4434 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4435 {
4436 	int idx;
4437 	bool px;
4438 
4439 	amdgpu_fence_driver_sw_fini(adev);
4440 	amdgpu_device_ip_fini(adev);
4441 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4442 	adev->accel_working = false;
4443 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4444 
4445 	amdgpu_reset_fini(adev);
4446 
4447 	/* free i2c buses */
4448 	if (!amdgpu_device_has_dc_support(adev))
4449 		amdgpu_i2c_fini(adev);
4450 
4451 	if (amdgpu_emu_mode != 1)
4452 		amdgpu_atombios_fini(adev);
4453 
4454 	kfree(adev->bios);
4455 	adev->bios = NULL;
4456 
4457 	kfree(adev->fru_info);
4458 	adev->fru_info = NULL;
4459 
4460 	px = amdgpu_device_supports_px(adev_to_drm(adev));
4461 
4462 	if (px || (!dev_is_removable(&adev->pdev->dev) &&
4463 				apple_gmux_detect(NULL, NULL)))
4464 		vga_switcheroo_unregister_client(adev->pdev);
4465 
4466 	if (px)
4467 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4468 
4469 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4470 		vga_client_unregister(adev->pdev);
4471 
4472 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4473 
4474 		iounmap(adev->rmmio);
4475 		adev->rmmio = NULL;
4476 		amdgpu_doorbell_fini(adev);
4477 		drm_dev_exit(idx);
4478 	}
4479 
4480 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4481 		amdgpu_pmu_fini(adev);
4482 	if (adev->mman.discovery_bin)
4483 		amdgpu_discovery_fini(adev);
4484 
4485 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4486 	adev->reset_domain = NULL;
4487 
4488 	kfree(adev->pci_state);
4489 
4490 }
4491 
4492 /**
4493  * amdgpu_device_evict_resources - evict device resources
4494  * @adev: amdgpu device object
4495  *
4496  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4497  * of the vram memory type. Mainly used for evicting device resources
4498  * at suspend time.
4499  *
4500  */
4501 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4502 {
4503 	int ret;
4504 
4505 	/* No need to evict vram on APUs for suspend to ram or s2idle */
4506 	if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4507 		return 0;
4508 
4509 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4510 	if (ret)
4511 		DRM_WARN("evicting device resources failed\n");
4512 	return ret;
4513 }
4514 
4515 /*
4516  * Suspend & resume.
4517  */
4518 /**
4519  * amdgpu_device_prepare - prepare for device suspend
4520  *
4521  * @dev: drm dev pointer
4522  *
4523  * Prepare to put the hw in the suspend state (all asics).
4524  * Returns 0 for success or an error on failure.
4525  * Called at driver suspend.
4526  */
4527 int amdgpu_device_prepare(struct drm_device *dev)
4528 {
4529 	struct amdgpu_device *adev = drm_to_adev(dev);
4530 	int i, r;
4531 
4532 	amdgpu_choose_low_power_state(adev);
4533 
4534 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4535 		return 0;
4536 
4537 	/* Evict the majority of BOs before starting suspend sequence */
4538 	r = amdgpu_device_evict_resources(adev);
4539 	if (r)
4540 		goto unprepare;
4541 
4542 	for (i = 0; i < adev->num_ip_blocks; i++) {
4543 		if (!adev->ip_blocks[i].status.valid)
4544 			continue;
4545 		if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4546 			continue;
4547 		r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4548 		if (r)
4549 			goto unprepare;
4550 	}
4551 
4552 	return 0;
4553 
4554 unprepare:
4555 	adev->in_s0ix = adev->in_s3 = false;
4556 
4557 	return r;
4558 }
4559 
4560 /**
4561  * amdgpu_device_suspend - initiate device suspend
4562  *
4563  * @dev: drm dev pointer
4564  * @fbcon : notify the fbdev of suspend
4565  *
4566  * Puts the hw in the suspend state (all asics).
4567  * Returns 0 for success or an error on failure.
4568  * Called at driver suspend.
4569  */
4570 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4571 {
4572 	struct amdgpu_device *adev = drm_to_adev(dev);
4573 	int r = 0;
4574 
4575 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4576 		return 0;
4577 
4578 	adev->in_suspend = true;
4579 
4580 	if (amdgpu_sriov_vf(adev)) {
4581 		amdgpu_virt_fini_data_exchange(adev);
4582 		r = amdgpu_virt_request_full_gpu(adev, false);
4583 		if (r)
4584 			return r;
4585 	}
4586 
4587 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4588 		DRM_WARN("smart shift update failed\n");
4589 
4590 	if (fbcon)
4591 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4592 
4593 	cancel_delayed_work_sync(&adev->delayed_init_work);
4594 
4595 	amdgpu_ras_suspend(adev);
4596 
4597 	amdgpu_device_ip_suspend_phase1(adev);
4598 
4599 	if (!adev->in_s0ix)
4600 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4601 
4602 	r = amdgpu_device_evict_resources(adev);
4603 	if (r)
4604 		return r;
4605 
4606 	amdgpu_ttm_set_buffer_funcs_status(adev, false);
4607 
4608 	amdgpu_fence_driver_hw_fini(adev);
4609 
4610 	amdgpu_device_ip_suspend_phase2(adev);
4611 
4612 	if (amdgpu_sriov_vf(adev))
4613 		amdgpu_virt_release_full_gpu(adev, false);
4614 
4615 	r = amdgpu_dpm_notify_rlc_state(adev, false);
4616 	if (r)
4617 		return r;
4618 
4619 	return 0;
4620 }
4621 
4622 /**
4623  * amdgpu_device_resume - initiate device resume
4624  *
4625  * @dev: drm dev pointer
4626  * @fbcon : notify the fbdev of resume
4627  *
4628  * Bring the hw back to operating state (all asics).
4629  * Returns 0 for success or an error on failure.
4630  * Called at driver resume.
4631  */
4632 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4633 {
4634 	struct amdgpu_device *adev = drm_to_adev(dev);
4635 	int r = 0;
4636 
4637 	if (amdgpu_sriov_vf(adev)) {
4638 		r = amdgpu_virt_request_full_gpu(adev, true);
4639 		if (r)
4640 			return r;
4641 	}
4642 
4643 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4644 		return 0;
4645 
4646 	if (adev->in_s0ix)
4647 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4648 
4649 	/* post card */
4650 	if (amdgpu_device_need_post(adev)) {
4651 		r = amdgpu_device_asic_init(adev);
4652 		if (r)
4653 			dev_err(adev->dev, "amdgpu asic init failed\n");
4654 	}
4655 
4656 	r = amdgpu_device_ip_resume(adev);
4657 
4658 	if (r) {
4659 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4660 		goto exit;
4661 	}
4662 	amdgpu_fence_driver_hw_init(adev);
4663 
4664 	if (!adev->in_s0ix) {
4665 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4666 		if (r)
4667 			goto exit;
4668 	}
4669 
4670 	r = amdgpu_device_ip_late_init(adev);
4671 	if (r)
4672 		goto exit;
4673 
4674 	queue_delayed_work(system_wq, &adev->delayed_init_work,
4675 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
4676 exit:
4677 	if (amdgpu_sriov_vf(adev)) {
4678 		amdgpu_virt_init_data_exchange(adev);
4679 		amdgpu_virt_release_full_gpu(adev, true);
4680 	}
4681 
4682 	if (r)
4683 		return r;
4684 
4685 	/* Make sure IB tests flushed */
4686 	flush_delayed_work(&adev->delayed_init_work);
4687 
4688 	if (fbcon)
4689 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4690 
4691 	amdgpu_ras_resume(adev);
4692 
4693 	if (adev->mode_info.num_crtc) {
4694 		/*
4695 		 * Most of the connector probing functions try to acquire runtime pm
4696 		 * refs to ensure that the GPU is powered on when connector polling is
4697 		 * performed. Since we're calling this from a runtime PM callback,
4698 		 * trying to acquire rpm refs will cause us to deadlock.
4699 		 *
4700 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
4701 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
4702 		 */
4703 #ifdef CONFIG_PM
4704 		dev->dev->power.disable_depth++;
4705 #endif
4706 		if (!adev->dc_enabled)
4707 			drm_helper_hpd_irq_event(dev);
4708 		else
4709 			drm_kms_helper_hotplug_event(dev);
4710 #ifdef CONFIG_PM
4711 		dev->dev->power.disable_depth--;
4712 #endif
4713 	}
4714 	adev->in_suspend = false;
4715 
4716 	if (adev->enable_mes)
4717 		amdgpu_mes_self_test(adev);
4718 
4719 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4720 		DRM_WARN("smart shift update failed\n");
4721 
4722 	return 0;
4723 }
4724 
4725 /**
4726  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4727  *
4728  * @adev: amdgpu_device pointer
4729  *
4730  * The list of all the hardware IPs that make up the asic is walked and
4731  * the check_soft_reset callbacks are run.  check_soft_reset determines
4732  * if the asic is still hung or not.
4733  * Returns true if any of the IPs are still in a hung state, false if not.
4734  */
4735 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4736 {
4737 	int i;
4738 	bool asic_hang = false;
4739 
4740 	if (amdgpu_sriov_vf(adev))
4741 		return true;
4742 
4743 	if (amdgpu_asic_need_full_reset(adev))
4744 		return true;
4745 
4746 	for (i = 0; i < adev->num_ip_blocks; i++) {
4747 		if (!adev->ip_blocks[i].status.valid)
4748 			continue;
4749 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4750 			adev->ip_blocks[i].status.hang =
4751 				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4752 		if (adev->ip_blocks[i].status.hang) {
4753 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4754 			asic_hang = true;
4755 		}
4756 	}
4757 	return asic_hang;
4758 }
4759 
4760 /**
4761  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4762  *
4763  * @adev: amdgpu_device pointer
4764  *
4765  * The list of all the hardware IPs that make up the asic is walked and the
4766  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4767  * handles any IP specific hardware or software state changes that are
4768  * necessary for a soft reset to succeed.
4769  * Returns 0 on success, negative error code on failure.
4770  */
4771 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4772 {
4773 	int i, r = 0;
4774 
4775 	for (i = 0; i < adev->num_ip_blocks; i++) {
4776 		if (!adev->ip_blocks[i].status.valid)
4777 			continue;
4778 		if (adev->ip_blocks[i].status.hang &&
4779 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4780 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4781 			if (r)
4782 				return r;
4783 		}
4784 	}
4785 
4786 	return 0;
4787 }
4788 
4789 /**
4790  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4791  *
4792  * @adev: amdgpu_device pointer
4793  *
4794  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4795  * reset is necessary to recover.
4796  * Returns true if a full asic reset is required, false if not.
4797  */
4798 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4799 {
4800 	int i;
4801 
4802 	if (amdgpu_asic_need_full_reset(adev))
4803 		return true;
4804 
4805 	for (i = 0; i < adev->num_ip_blocks; i++) {
4806 		if (!adev->ip_blocks[i].status.valid)
4807 			continue;
4808 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4809 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4810 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4811 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4812 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4813 			if (adev->ip_blocks[i].status.hang) {
4814 				dev_info(adev->dev, "Some block need full reset!\n");
4815 				return true;
4816 			}
4817 		}
4818 	}
4819 	return false;
4820 }
4821 
4822 /**
4823  * amdgpu_device_ip_soft_reset - do a soft reset
4824  *
4825  * @adev: amdgpu_device pointer
4826  *
4827  * The list of all the hardware IPs that make up the asic is walked and the
4828  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4829  * IP specific hardware or software state changes that are necessary to soft
4830  * reset the IP.
4831  * Returns 0 on success, negative error code on failure.
4832  */
4833 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4834 {
4835 	int i, r = 0;
4836 
4837 	for (i = 0; i < adev->num_ip_blocks; i++) {
4838 		if (!adev->ip_blocks[i].status.valid)
4839 			continue;
4840 		if (adev->ip_blocks[i].status.hang &&
4841 		    adev->ip_blocks[i].version->funcs->soft_reset) {
4842 			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4843 			if (r)
4844 				return r;
4845 		}
4846 	}
4847 
4848 	return 0;
4849 }
4850 
4851 /**
4852  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4853  *
4854  * @adev: amdgpu_device pointer
4855  *
4856  * The list of all the hardware IPs that make up the asic is walked and the
4857  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4858  * handles any IP specific hardware or software state changes that are
4859  * necessary after the IP has been soft reset.
4860  * Returns 0 on success, negative error code on failure.
4861  */
4862 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4863 {
4864 	int i, r = 0;
4865 
4866 	for (i = 0; i < adev->num_ip_blocks; i++) {
4867 		if (!adev->ip_blocks[i].status.valid)
4868 			continue;
4869 		if (adev->ip_blocks[i].status.hang &&
4870 		    adev->ip_blocks[i].version->funcs->post_soft_reset)
4871 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4872 		if (r)
4873 			return r;
4874 	}
4875 
4876 	return 0;
4877 }
4878 
4879 /**
4880  * amdgpu_device_recover_vram - Recover some VRAM contents
4881  *
4882  * @adev: amdgpu_device pointer
4883  *
4884  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4885  * restore things like GPUVM page tables after a GPU reset where
4886  * the contents of VRAM might be lost.
4887  *
4888  * Returns:
4889  * 0 on success, negative error code on failure.
4890  */
4891 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4892 {
4893 	struct dma_fence *fence = NULL, *next = NULL;
4894 	struct amdgpu_bo *shadow;
4895 	struct amdgpu_bo_vm *vmbo;
4896 	long r = 1, tmo;
4897 
4898 	if (amdgpu_sriov_runtime(adev))
4899 		tmo = msecs_to_jiffies(8000);
4900 	else
4901 		tmo = msecs_to_jiffies(100);
4902 
4903 	dev_info(adev->dev, "recover vram bo from shadow start\n");
4904 	mutex_lock(&adev->shadow_list_lock);
4905 	list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4906 		/* If vm is compute context or adev is APU, shadow will be NULL */
4907 		if (!vmbo->shadow)
4908 			continue;
4909 		shadow = vmbo->shadow;
4910 
4911 		/* No need to recover an evicted BO */
4912 		if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4913 		    shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4914 		    shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4915 			continue;
4916 
4917 		r = amdgpu_bo_restore_shadow(shadow, &next);
4918 		if (r)
4919 			break;
4920 
4921 		if (fence) {
4922 			tmo = dma_fence_wait_timeout(fence, false, tmo);
4923 			dma_fence_put(fence);
4924 			fence = next;
4925 			if (tmo == 0) {
4926 				r = -ETIMEDOUT;
4927 				break;
4928 			} else if (tmo < 0) {
4929 				r = tmo;
4930 				break;
4931 			}
4932 		} else {
4933 			fence = next;
4934 		}
4935 	}
4936 	mutex_unlock(&adev->shadow_list_lock);
4937 
4938 	if (fence)
4939 		tmo = dma_fence_wait_timeout(fence, false, tmo);
4940 	dma_fence_put(fence);
4941 
4942 	if (r < 0 || tmo <= 0) {
4943 		dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4944 		return -EIO;
4945 	}
4946 
4947 	dev_info(adev->dev, "recover vram bo from shadow done\n");
4948 	return 0;
4949 }
4950 
4951 
4952 /**
4953  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4954  *
4955  * @adev: amdgpu_device pointer
4956  * @from_hypervisor: request from hypervisor
4957  *
4958  * do VF FLR and reinitialize Asic
4959  * return 0 means succeeded otherwise failed
4960  */
4961 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4962 				     bool from_hypervisor)
4963 {
4964 	int r;
4965 	struct amdgpu_hive_info *hive = NULL;
4966 	int retry_limit = 0;
4967 
4968 retry:
4969 	amdgpu_amdkfd_pre_reset(adev);
4970 
4971 	if (from_hypervisor)
4972 		r = amdgpu_virt_request_full_gpu(adev, true);
4973 	else
4974 		r = amdgpu_virt_reset_gpu(adev);
4975 	if (r)
4976 		return r;
4977 	amdgpu_irq_gpu_reset_resume_helper(adev);
4978 
4979 	/* some sw clean up VF needs to do before recover */
4980 	amdgpu_virt_post_reset(adev);
4981 
4982 	/* Resume IP prior to SMC */
4983 	r = amdgpu_device_ip_reinit_early_sriov(adev);
4984 	if (r)
4985 		goto error;
4986 
4987 	amdgpu_virt_init_data_exchange(adev);
4988 
4989 	r = amdgpu_device_fw_loading(adev);
4990 	if (r)
4991 		return r;
4992 
4993 	/* now we are okay to resume SMC/CP/SDMA */
4994 	r = amdgpu_device_ip_reinit_late_sriov(adev);
4995 	if (r)
4996 		goto error;
4997 
4998 	hive = amdgpu_get_xgmi_hive(adev);
4999 	/* Update PSP FW topology after reset */
5000 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5001 		r = amdgpu_xgmi_update_topology(hive, adev);
5002 
5003 	if (hive)
5004 		amdgpu_put_xgmi_hive(hive);
5005 
5006 	if (!r) {
5007 		r = amdgpu_ib_ring_tests(adev);
5008 
5009 		amdgpu_amdkfd_post_reset(adev);
5010 	}
5011 
5012 error:
5013 	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
5014 		amdgpu_inc_vram_lost(adev);
5015 		r = amdgpu_device_recover_vram(adev);
5016 	}
5017 	amdgpu_virt_release_full_gpu(adev, true);
5018 
5019 	if (AMDGPU_RETRY_SRIOV_RESET(r)) {
5020 		if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
5021 			retry_limit++;
5022 			goto retry;
5023 		} else
5024 			DRM_ERROR("GPU reset retry is beyond the retry limit\n");
5025 	}
5026 
5027 	return r;
5028 }
5029 
5030 /**
5031  * amdgpu_device_has_job_running - check if there is any job in mirror list
5032  *
5033  * @adev: amdgpu_device pointer
5034  *
5035  * check if there is any job in mirror list
5036  */
5037 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5038 {
5039 	int i;
5040 	struct drm_sched_job *job;
5041 
5042 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5043 		struct amdgpu_ring *ring = adev->rings[i];
5044 
5045 		if (!amdgpu_ring_sched_ready(ring))
5046 			continue;
5047 
5048 		spin_lock(&ring->sched.job_list_lock);
5049 		job = list_first_entry_or_null(&ring->sched.pending_list,
5050 					       struct drm_sched_job, list);
5051 		spin_unlock(&ring->sched.job_list_lock);
5052 		if (job)
5053 			return true;
5054 	}
5055 	return false;
5056 }
5057 
5058 /**
5059  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5060  *
5061  * @adev: amdgpu_device pointer
5062  *
5063  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5064  * a hung GPU.
5065  */
5066 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5067 {
5068 
5069 	if (amdgpu_gpu_recovery == 0)
5070 		goto disabled;
5071 
5072 	/* Skip soft reset check in fatal error mode */
5073 	if (!amdgpu_ras_is_poison_mode_supported(adev))
5074 		return true;
5075 
5076 	if (amdgpu_sriov_vf(adev))
5077 		return true;
5078 
5079 	if (amdgpu_gpu_recovery == -1) {
5080 		switch (adev->asic_type) {
5081 #ifdef CONFIG_DRM_AMDGPU_SI
5082 		case CHIP_VERDE:
5083 		case CHIP_TAHITI:
5084 		case CHIP_PITCAIRN:
5085 		case CHIP_OLAND:
5086 		case CHIP_HAINAN:
5087 #endif
5088 #ifdef CONFIG_DRM_AMDGPU_CIK
5089 		case CHIP_KAVERI:
5090 		case CHIP_KABINI:
5091 		case CHIP_MULLINS:
5092 #endif
5093 		case CHIP_CARRIZO:
5094 		case CHIP_STONEY:
5095 		case CHIP_CYAN_SKILLFISH:
5096 			goto disabled;
5097 		default:
5098 			break;
5099 		}
5100 	}
5101 
5102 	return true;
5103 
5104 disabled:
5105 		dev_info(adev->dev, "GPU recovery disabled.\n");
5106 		return false;
5107 }
5108 
5109 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5110 {
5111 	u32 i;
5112 	int ret = 0;
5113 
5114 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5115 
5116 	dev_info(adev->dev, "GPU mode1 reset\n");
5117 
5118 	/* disable BM */
5119 	pci_clear_master(adev->pdev);
5120 
5121 	amdgpu_device_cache_pci_state(adev->pdev);
5122 
5123 	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5124 		dev_info(adev->dev, "GPU smu mode1 reset\n");
5125 		ret = amdgpu_dpm_mode1_reset(adev);
5126 	} else {
5127 		dev_info(adev->dev, "GPU psp mode1 reset\n");
5128 		ret = psp_gpu_reset(adev);
5129 	}
5130 
5131 	if (ret)
5132 		goto mode1_reset_failed;
5133 
5134 	amdgpu_device_load_pci_state(adev->pdev);
5135 	ret = amdgpu_psp_wait_for_bootloader(adev);
5136 	if (ret)
5137 		goto mode1_reset_failed;
5138 
5139 	/* wait for asic to come out of reset */
5140 	for (i = 0; i < adev->usec_timeout; i++) {
5141 		u32 memsize = adev->nbio.funcs->get_memsize(adev);
5142 
5143 		if (memsize != 0xffffffff)
5144 			break;
5145 		udelay(1);
5146 	}
5147 
5148 	if (i >= adev->usec_timeout) {
5149 		ret = -ETIMEDOUT;
5150 		goto mode1_reset_failed;
5151 	}
5152 
5153 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5154 
5155 	return 0;
5156 
5157 mode1_reset_failed:
5158 	dev_err(adev->dev, "GPU mode1 reset failed\n");
5159 	return ret;
5160 }
5161 
5162 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5163 				 struct amdgpu_reset_context *reset_context)
5164 {
5165 	int i, r = 0;
5166 	struct amdgpu_job *job = NULL;
5167 	bool need_full_reset =
5168 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5169 
5170 	if (reset_context->reset_req_dev == adev)
5171 		job = reset_context->job;
5172 
5173 	if (amdgpu_sriov_vf(adev)) {
5174 		/* stop the data exchange thread */
5175 		amdgpu_virt_fini_data_exchange(adev);
5176 	}
5177 
5178 	amdgpu_fence_driver_isr_toggle(adev, true);
5179 
5180 	/* block all schedulers and reset given job's ring */
5181 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5182 		struct amdgpu_ring *ring = adev->rings[i];
5183 
5184 		if (!amdgpu_ring_sched_ready(ring))
5185 			continue;
5186 
5187 		/* Clear job fence from fence drv to avoid force_completion
5188 		 * leave NULL and vm flush fence in fence drv
5189 		 */
5190 		amdgpu_fence_driver_clear_job_fences(ring);
5191 
5192 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5193 		amdgpu_fence_driver_force_completion(ring);
5194 	}
5195 
5196 	amdgpu_fence_driver_isr_toggle(adev, false);
5197 
5198 	if (job && job->vm)
5199 		drm_sched_increase_karma(&job->base);
5200 
5201 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5202 	/* If reset handler not implemented, continue; otherwise return */
5203 	if (r == -EOPNOTSUPP)
5204 		r = 0;
5205 	else
5206 		return r;
5207 
5208 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5209 	if (!amdgpu_sriov_vf(adev)) {
5210 
5211 		if (!need_full_reset)
5212 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5213 
5214 		if (!need_full_reset && amdgpu_gpu_recovery &&
5215 		    amdgpu_device_ip_check_soft_reset(adev)) {
5216 			amdgpu_device_ip_pre_soft_reset(adev);
5217 			r = amdgpu_device_ip_soft_reset(adev);
5218 			amdgpu_device_ip_post_soft_reset(adev);
5219 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5220 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5221 				need_full_reset = true;
5222 			}
5223 		}
5224 
5225 		if (need_full_reset)
5226 			r = amdgpu_device_ip_suspend(adev);
5227 		if (need_full_reset)
5228 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5229 		else
5230 			clear_bit(AMDGPU_NEED_FULL_RESET,
5231 				  &reset_context->flags);
5232 	}
5233 
5234 	return r;
5235 }
5236 
5237 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5238 {
5239 	int i;
5240 
5241 	lockdep_assert_held(&adev->reset_domain->sem);
5242 
5243 	for (i = 0; i < adev->reset_info.num_regs; i++) {
5244 		adev->reset_info.reset_dump_reg_value[i] =
5245 			RREG32(adev->reset_info.reset_dump_reg_list[i]);
5246 
5247 		trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i],
5248 					     adev->reset_info.reset_dump_reg_value[i]);
5249 	}
5250 
5251 	return 0;
5252 }
5253 
5254 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5255 			 struct amdgpu_reset_context *reset_context)
5256 {
5257 	struct amdgpu_device *tmp_adev = NULL;
5258 	bool need_full_reset, skip_hw_reset, vram_lost = false;
5259 	int r = 0;
5260 
5261 	/* Try reset handler method first */
5262 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5263 				    reset_list);
5264 	amdgpu_reset_reg_dumps(tmp_adev);
5265 
5266 	reset_context->reset_device_list = device_list_handle;
5267 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5268 	/* If reset handler not implemented, continue; otherwise return */
5269 	if (r == -EOPNOTSUPP)
5270 		r = 0;
5271 	else
5272 		return r;
5273 
5274 	/* Reset handler not implemented, use the default method */
5275 	need_full_reset =
5276 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5277 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5278 
5279 	/*
5280 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
5281 	 * to allow proper links negotiation in FW (within 1 sec)
5282 	 */
5283 	if (!skip_hw_reset && need_full_reset) {
5284 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5285 			/* For XGMI run all resets in parallel to speed up the process */
5286 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5287 				tmp_adev->gmc.xgmi.pending_reset = false;
5288 				if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5289 					r = -EALREADY;
5290 			} else
5291 				r = amdgpu_asic_reset(tmp_adev);
5292 
5293 			if (r) {
5294 				dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5295 					 r, adev_to_drm(tmp_adev)->unique);
5296 				goto out;
5297 			}
5298 		}
5299 
5300 		/* For XGMI wait for all resets to complete before proceed */
5301 		if (!r) {
5302 			list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5303 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5304 					flush_work(&tmp_adev->xgmi_reset_work);
5305 					r = tmp_adev->asic_reset_res;
5306 					if (r)
5307 						break;
5308 				}
5309 			}
5310 		}
5311 	}
5312 
5313 	if (!r && amdgpu_ras_intr_triggered()) {
5314 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5315 			amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB);
5316 		}
5317 
5318 		amdgpu_ras_intr_cleared();
5319 	}
5320 
5321 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5322 		if (need_full_reset) {
5323 			/* post card */
5324 			amdgpu_ras_set_fed(tmp_adev, false);
5325 			r = amdgpu_device_asic_init(tmp_adev);
5326 			if (r) {
5327 				dev_warn(tmp_adev->dev, "asic atom init failed!");
5328 			} else {
5329 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5330 
5331 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
5332 				if (r)
5333 					goto out;
5334 
5335 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5336 
5337 				amdgpu_coredump(tmp_adev, vram_lost, reset_context);
5338 
5339 				if (vram_lost) {
5340 					DRM_INFO("VRAM is lost due to GPU reset!\n");
5341 					amdgpu_inc_vram_lost(tmp_adev);
5342 				}
5343 
5344 				r = amdgpu_device_fw_loading(tmp_adev);
5345 				if (r)
5346 					return r;
5347 
5348 				r = amdgpu_xcp_restore_partition_mode(
5349 					tmp_adev->xcp_mgr);
5350 				if (r)
5351 					goto out;
5352 
5353 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
5354 				if (r)
5355 					goto out;
5356 
5357 				if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5358 					amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5359 
5360 				if (vram_lost)
5361 					amdgpu_device_fill_reset_magic(tmp_adev);
5362 
5363 				/*
5364 				 * Add this ASIC as tracked as reset was already
5365 				 * complete successfully.
5366 				 */
5367 				amdgpu_register_gpu_instance(tmp_adev);
5368 
5369 				if (!reset_context->hive &&
5370 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5371 					amdgpu_xgmi_add_device(tmp_adev);
5372 
5373 				r = amdgpu_device_ip_late_init(tmp_adev);
5374 				if (r)
5375 					goto out;
5376 
5377 				drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5378 
5379 				/*
5380 				 * The GPU enters bad state once faulty pages
5381 				 * by ECC has reached the threshold, and ras
5382 				 * recovery is scheduled next. So add one check
5383 				 * here to break recovery if it indeed exceeds
5384 				 * bad page threshold, and remind user to
5385 				 * retire this GPU or setting one bigger
5386 				 * bad_page_threshold value to fix this once
5387 				 * probing driver again.
5388 				 */
5389 				if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5390 					/* must succeed. */
5391 					amdgpu_ras_resume(tmp_adev);
5392 				} else {
5393 					r = -EINVAL;
5394 					goto out;
5395 				}
5396 
5397 				/* Update PSP FW topology after reset */
5398 				if (reset_context->hive &&
5399 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5400 					r = amdgpu_xgmi_update_topology(
5401 						reset_context->hive, tmp_adev);
5402 			}
5403 		}
5404 
5405 out:
5406 		if (!r) {
5407 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5408 			r = amdgpu_ib_ring_tests(tmp_adev);
5409 			if (r) {
5410 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5411 				need_full_reset = true;
5412 				r = -EAGAIN;
5413 				goto end;
5414 			}
5415 		}
5416 
5417 		if (!r)
5418 			r = amdgpu_device_recover_vram(tmp_adev);
5419 		else
5420 			tmp_adev->asic_reset_res = r;
5421 	}
5422 
5423 end:
5424 	if (need_full_reset)
5425 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5426 	else
5427 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5428 	return r;
5429 }
5430 
5431 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5432 {
5433 
5434 	switch (amdgpu_asic_reset_method(adev)) {
5435 	case AMD_RESET_METHOD_MODE1:
5436 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5437 		break;
5438 	case AMD_RESET_METHOD_MODE2:
5439 		adev->mp1_state = PP_MP1_STATE_RESET;
5440 		break;
5441 	default:
5442 		adev->mp1_state = PP_MP1_STATE_NONE;
5443 		break;
5444 	}
5445 }
5446 
5447 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5448 {
5449 	amdgpu_vf_error_trans_all(adev);
5450 	adev->mp1_state = PP_MP1_STATE_NONE;
5451 }
5452 
5453 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5454 {
5455 	struct pci_dev *p = NULL;
5456 
5457 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5458 			adev->pdev->bus->number, 1);
5459 	if (p) {
5460 		pm_runtime_enable(&(p->dev));
5461 		pm_runtime_resume(&(p->dev));
5462 	}
5463 
5464 	pci_dev_put(p);
5465 }
5466 
5467 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5468 {
5469 	enum amd_reset_method reset_method;
5470 	struct pci_dev *p = NULL;
5471 	u64 expires;
5472 
5473 	/*
5474 	 * For now, only BACO and mode1 reset are confirmed
5475 	 * to suffer the audio issue without proper suspended.
5476 	 */
5477 	reset_method = amdgpu_asic_reset_method(adev);
5478 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5479 	     (reset_method != AMD_RESET_METHOD_MODE1))
5480 		return -EINVAL;
5481 
5482 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5483 			adev->pdev->bus->number, 1);
5484 	if (!p)
5485 		return -ENODEV;
5486 
5487 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5488 	if (!expires)
5489 		/*
5490 		 * If we cannot get the audio device autosuspend delay,
5491 		 * a fixed 4S interval will be used. Considering 3S is
5492 		 * the audio controller default autosuspend delay setting.
5493 		 * 4S used here is guaranteed to cover that.
5494 		 */
5495 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5496 
5497 	while (!pm_runtime_status_suspended(&(p->dev))) {
5498 		if (!pm_runtime_suspend(&(p->dev)))
5499 			break;
5500 
5501 		if (expires < ktime_get_mono_fast_ns()) {
5502 			dev_warn(adev->dev, "failed to suspend display audio\n");
5503 			pci_dev_put(p);
5504 			/* TODO: abort the succeeding gpu reset? */
5505 			return -ETIMEDOUT;
5506 		}
5507 	}
5508 
5509 	pm_runtime_disable(&(p->dev));
5510 
5511 	pci_dev_put(p);
5512 	return 0;
5513 }
5514 
5515 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5516 {
5517 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5518 
5519 #if defined(CONFIG_DEBUG_FS)
5520 	if (!amdgpu_sriov_vf(adev))
5521 		cancel_work(&adev->reset_work);
5522 #endif
5523 
5524 	if (adev->kfd.dev)
5525 		cancel_work(&adev->kfd.reset_work);
5526 
5527 	if (amdgpu_sriov_vf(adev))
5528 		cancel_work(&adev->virt.flr_work);
5529 
5530 	if (con && adev->ras_enabled)
5531 		cancel_work(&con->recovery_work);
5532 
5533 }
5534 
5535 /**
5536  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5537  *
5538  * @adev: amdgpu_device pointer
5539  * @job: which job trigger hang
5540  * @reset_context: amdgpu reset context pointer
5541  *
5542  * Attempt to reset the GPU if it has hung (all asics).
5543  * Attempt to do soft-reset or full-reset and reinitialize Asic
5544  * Returns 0 for success or an error on failure.
5545  */
5546 
5547 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5548 			      struct amdgpu_job *job,
5549 			      struct amdgpu_reset_context *reset_context)
5550 {
5551 	struct list_head device_list, *device_list_handle =  NULL;
5552 	bool job_signaled = false;
5553 	struct amdgpu_hive_info *hive = NULL;
5554 	struct amdgpu_device *tmp_adev = NULL;
5555 	int i, r = 0;
5556 	bool need_emergency_restart = false;
5557 	bool audio_suspended = false;
5558 
5559 	/*
5560 	 * Special case: RAS triggered and full reset isn't supported
5561 	 */
5562 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5563 
5564 	/*
5565 	 * Flush RAM to disk so that after reboot
5566 	 * the user can read log and see why the system rebooted.
5567 	 */
5568 	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5569 		amdgpu_ras_get_context(adev)->reboot) {
5570 		DRM_WARN("Emergency reboot.");
5571 
5572 		ksys_sync_helper();
5573 		emergency_restart();
5574 	}
5575 
5576 	dev_info(adev->dev, "GPU %s begin!\n",
5577 		need_emergency_restart ? "jobs stop":"reset");
5578 
5579 	if (!amdgpu_sriov_vf(adev))
5580 		hive = amdgpu_get_xgmi_hive(adev);
5581 	if (hive)
5582 		mutex_lock(&hive->hive_lock);
5583 
5584 	reset_context->job = job;
5585 	reset_context->hive = hive;
5586 	/*
5587 	 * Build list of devices to reset.
5588 	 * In case we are in XGMI hive mode, resort the device list
5589 	 * to put adev in the 1st position.
5590 	 */
5591 	INIT_LIST_HEAD(&device_list);
5592 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5593 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5594 			list_add_tail(&tmp_adev->reset_list, &device_list);
5595 			if (adev->shutdown)
5596 				tmp_adev->shutdown = true;
5597 		}
5598 		if (!list_is_first(&adev->reset_list, &device_list))
5599 			list_rotate_to_front(&adev->reset_list, &device_list);
5600 		device_list_handle = &device_list;
5601 	} else {
5602 		list_add_tail(&adev->reset_list, &device_list);
5603 		device_list_handle = &device_list;
5604 	}
5605 
5606 	/* We need to lock reset domain only once both for XGMI and single device */
5607 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5608 				    reset_list);
5609 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5610 
5611 	/* block all schedulers and reset given job's ring */
5612 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5613 
5614 		amdgpu_device_set_mp1_state(tmp_adev);
5615 
5616 		/*
5617 		 * Try to put the audio codec into suspend state
5618 		 * before gpu reset started.
5619 		 *
5620 		 * Due to the power domain of the graphics device
5621 		 * is shared with AZ power domain. Without this,
5622 		 * we may change the audio hardware from behind
5623 		 * the audio driver's back. That will trigger
5624 		 * some audio codec errors.
5625 		 */
5626 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5627 			audio_suspended = true;
5628 
5629 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5630 
5631 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5632 
5633 		if (!amdgpu_sriov_vf(tmp_adev))
5634 			amdgpu_amdkfd_pre_reset(tmp_adev);
5635 
5636 		/*
5637 		 * Mark these ASICs to be reseted as untracked first
5638 		 * And add them back after reset completed
5639 		 */
5640 		amdgpu_unregister_gpu_instance(tmp_adev);
5641 
5642 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5643 
5644 		/* disable ras on ALL IPs */
5645 		if (!need_emergency_restart &&
5646 		      amdgpu_device_ip_need_full_reset(tmp_adev))
5647 			amdgpu_ras_suspend(tmp_adev);
5648 
5649 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5650 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5651 
5652 			if (!amdgpu_ring_sched_ready(ring))
5653 				continue;
5654 
5655 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5656 
5657 			if (need_emergency_restart)
5658 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5659 		}
5660 		atomic_inc(&tmp_adev->gpu_reset_counter);
5661 	}
5662 
5663 	if (need_emergency_restart)
5664 		goto skip_sched_resume;
5665 
5666 	/*
5667 	 * Must check guilty signal here since after this point all old
5668 	 * HW fences are force signaled.
5669 	 *
5670 	 * job->base holds a reference to parent fence
5671 	 */
5672 	if (job && dma_fence_is_signaled(&job->hw_fence)) {
5673 		job_signaled = true;
5674 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5675 		goto skip_hw_reset;
5676 	}
5677 
5678 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
5679 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5680 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5681 		/*TODO Should we stop ?*/
5682 		if (r) {
5683 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5684 				  r, adev_to_drm(tmp_adev)->unique);
5685 			tmp_adev->asic_reset_res = r;
5686 		}
5687 
5688 		/*
5689 		 * Drop all pending non scheduler resets. Scheduler resets
5690 		 * were already dropped during drm_sched_stop
5691 		 */
5692 		amdgpu_device_stop_pending_resets(tmp_adev);
5693 	}
5694 
5695 	/* Actual ASIC resets if needed.*/
5696 	/* Host driver will handle XGMI hive reset for SRIOV */
5697 	if (amdgpu_sriov_vf(adev)) {
5698 		r = amdgpu_device_reset_sriov(adev, job ? false : true);
5699 		if (r)
5700 			adev->asic_reset_res = r;
5701 
5702 		/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5703 		if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
5704 			    IP_VERSION(9, 4, 2) ||
5705 		    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
5706 		    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5707 			amdgpu_ras_resume(adev);
5708 	} else {
5709 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5710 		if (r && r == -EAGAIN)
5711 			goto retry;
5712 	}
5713 
5714 skip_hw_reset:
5715 
5716 	/* Post ASIC reset for all devs .*/
5717 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5718 
5719 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5720 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5721 
5722 			if (!amdgpu_ring_sched_ready(ring))
5723 				continue;
5724 
5725 			drm_sched_start(&ring->sched, true);
5726 		}
5727 
5728 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5729 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5730 
5731 		if (tmp_adev->asic_reset_res)
5732 			r = tmp_adev->asic_reset_res;
5733 
5734 		tmp_adev->asic_reset_res = 0;
5735 
5736 		if (r) {
5737 			/* bad news, how to tell it to userspace ? */
5738 			dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5739 			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5740 		} else {
5741 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5742 			if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5743 				DRM_WARN("smart shift update failed\n");
5744 		}
5745 	}
5746 
5747 skip_sched_resume:
5748 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5749 		/* unlock kfd: SRIOV would do it separately */
5750 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5751 			amdgpu_amdkfd_post_reset(tmp_adev);
5752 
5753 		/* kfd_post_reset will do nothing if kfd device is not initialized,
5754 		 * need to bring up kfd here if it's not be initialized before
5755 		 */
5756 		if (!adev->kfd.init_complete)
5757 			amdgpu_amdkfd_device_init(adev);
5758 
5759 		if (audio_suspended)
5760 			amdgpu_device_resume_display_audio(tmp_adev);
5761 
5762 		amdgpu_device_unset_mp1_state(tmp_adev);
5763 
5764 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
5765 	}
5766 
5767 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5768 					    reset_list);
5769 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5770 
5771 	if (hive) {
5772 		mutex_unlock(&hive->hive_lock);
5773 		amdgpu_put_xgmi_hive(hive);
5774 	}
5775 
5776 	if (r)
5777 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5778 
5779 	atomic_set(&adev->reset_domain->reset_res, r);
5780 	return r;
5781 }
5782 
5783 /**
5784  * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5785  *
5786  * @adev: amdgpu_device pointer
5787  * @speed: pointer to the speed of the link
5788  * @width: pointer to the width of the link
5789  *
5790  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5791  * first physical partner to an AMD dGPU.
5792  * This will exclude any virtual switches and links.
5793  */
5794 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5795 					    enum pci_bus_speed *speed,
5796 					    enum pcie_link_width *width)
5797 {
5798 	struct pci_dev *parent = adev->pdev;
5799 
5800 	if (!speed || !width)
5801 		return;
5802 
5803 	*speed = PCI_SPEED_UNKNOWN;
5804 	*width = PCIE_LNK_WIDTH_UNKNOWN;
5805 
5806 	while ((parent = pci_upstream_bridge(parent))) {
5807 		/* skip upstream/downstream switches internal to dGPU*/
5808 		if (parent->vendor == PCI_VENDOR_ID_ATI)
5809 			continue;
5810 		*speed = pcie_get_speed_cap(parent);
5811 		*width = pcie_get_width_cap(parent);
5812 		break;
5813 	}
5814 }
5815 
5816 /**
5817  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5818  *
5819  * @adev: amdgpu_device pointer
5820  *
5821  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5822  * and lanes) of the slot the device is in. Handles APUs and
5823  * virtualized environments where PCIE config space may not be available.
5824  */
5825 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5826 {
5827 	struct pci_dev *pdev;
5828 	enum pci_bus_speed speed_cap, platform_speed_cap;
5829 	enum pcie_link_width platform_link_width;
5830 
5831 	if (amdgpu_pcie_gen_cap)
5832 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5833 
5834 	if (amdgpu_pcie_lane_cap)
5835 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5836 
5837 	/* covers APUs as well */
5838 	if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5839 		if (adev->pm.pcie_gen_mask == 0)
5840 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5841 		if (adev->pm.pcie_mlw_mask == 0)
5842 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5843 		return;
5844 	}
5845 
5846 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5847 		return;
5848 
5849 	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
5850 					&platform_link_width);
5851 
5852 	if (adev->pm.pcie_gen_mask == 0) {
5853 		/* asic caps */
5854 		pdev = adev->pdev;
5855 		speed_cap = pcie_get_speed_cap(pdev);
5856 		if (speed_cap == PCI_SPEED_UNKNOWN) {
5857 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5858 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5859 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5860 		} else {
5861 			if (speed_cap == PCIE_SPEED_32_0GT)
5862 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5863 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5864 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5865 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5866 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5867 			else if (speed_cap == PCIE_SPEED_16_0GT)
5868 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5869 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5870 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5871 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5872 			else if (speed_cap == PCIE_SPEED_8_0GT)
5873 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5874 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5875 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5876 			else if (speed_cap == PCIE_SPEED_5_0GT)
5877 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5878 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5879 			else
5880 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5881 		}
5882 		/* platform caps */
5883 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5884 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5885 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5886 		} else {
5887 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
5888 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5889 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5890 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5891 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5892 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5893 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5894 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5895 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5896 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5897 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5898 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5899 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5900 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5901 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5902 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5903 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5904 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5905 			else
5906 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5907 
5908 		}
5909 	}
5910 	if (adev->pm.pcie_mlw_mask == 0) {
5911 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5912 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5913 		} else {
5914 			switch (platform_link_width) {
5915 			case PCIE_LNK_X32:
5916 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5917 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5918 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5919 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5920 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5921 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5922 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5923 				break;
5924 			case PCIE_LNK_X16:
5925 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5926 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5927 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5928 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5929 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5930 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5931 				break;
5932 			case PCIE_LNK_X12:
5933 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5934 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5935 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5936 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5937 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5938 				break;
5939 			case PCIE_LNK_X8:
5940 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5941 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5942 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5943 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5944 				break;
5945 			case PCIE_LNK_X4:
5946 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5947 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5948 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5949 				break;
5950 			case PCIE_LNK_X2:
5951 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5952 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5953 				break;
5954 			case PCIE_LNK_X1:
5955 				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5956 				break;
5957 			default:
5958 				break;
5959 			}
5960 		}
5961 	}
5962 }
5963 
5964 /**
5965  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5966  *
5967  * @adev: amdgpu_device pointer
5968  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5969  *
5970  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5971  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5972  * @peer_adev.
5973  */
5974 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5975 				      struct amdgpu_device *peer_adev)
5976 {
5977 #ifdef CONFIG_HSA_AMD_P2P
5978 	uint64_t address_mask = peer_adev->dev->dma_mask ?
5979 		~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5980 	resource_size_t aper_limit =
5981 		adev->gmc.aper_base + adev->gmc.aper_size - 1;
5982 	bool p2p_access =
5983 		!adev->gmc.xgmi.connected_to_cpu &&
5984 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5985 
5986 	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5987 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5988 		!(adev->gmc.aper_base & address_mask ||
5989 		  aper_limit & address_mask));
5990 #else
5991 	return false;
5992 #endif
5993 }
5994 
5995 int amdgpu_device_baco_enter(struct drm_device *dev)
5996 {
5997 	struct amdgpu_device *adev = drm_to_adev(dev);
5998 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5999 
6000 	if (!amdgpu_device_supports_baco(dev))
6001 		return -ENOTSUPP;
6002 
6003 	if (ras && adev->ras_enabled &&
6004 	    adev->nbio.funcs->enable_doorbell_interrupt)
6005 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6006 
6007 	return amdgpu_dpm_baco_enter(adev);
6008 }
6009 
6010 int amdgpu_device_baco_exit(struct drm_device *dev)
6011 {
6012 	struct amdgpu_device *adev = drm_to_adev(dev);
6013 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6014 	int ret = 0;
6015 
6016 	if (!amdgpu_device_supports_baco(dev))
6017 		return -ENOTSUPP;
6018 
6019 	ret = amdgpu_dpm_baco_exit(adev);
6020 	if (ret)
6021 		return ret;
6022 
6023 	if (ras && adev->ras_enabled &&
6024 	    adev->nbio.funcs->enable_doorbell_interrupt)
6025 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6026 
6027 	if (amdgpu_passthrough(adev) &&
6028 	    adev->nbio.funcs->clear_doorbell_interrupt)
6029 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
6030 
6031 	return 0;
6032 }
6033 
6034 /**
6035  * amdgpu_pci_error_detected - Called when a PCI error is detected.
6036  * @pdev: PCI device struct
6037  * @state: PCI channel state
6038  *
6039  * Description: Called when a PCI error is detected.
6040  *
6041  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6042  */
6043 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6044 {
6045 	struct drm_device *dev = pci_get_drvdata(pdev);
6046 	struct amdgpu_device *adev = drm_to_adev(dev);
6047 	int i;
6048 
6049 	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6050 
6051 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
6052 		DRM_WARN("No support for XGMI hive yet...");
6053 		return PCI_ERS_RESULT_DISCONNECT;
6054 	}
6055 
6056 	adev->pci_channel_state = state;
6057 
6058 	switch (state) {
6059 	case pci_channel_io_normal:
6060 		return PCI_ERS_RESULT_CAN_RECOVER;
6061 	/* Fatal error, prepare for slot reset */
6062 	case pci_channel_io_frozen:
6063 		/*
6064 		 * Locking adev->reset_domain->sem will prevent any external access
6065 		 * to GPU during PCI error recovery
6066 		 */
6067 		amdgpu_device_lock_reset_domain(adev->reset_domain);
6068 		amdgpu_device_set_mp1_state(adev);
6069 
6070 		/*
6071 		 * Block any work scheduling as we do for regular GPU reset
6072 		 * for the duration of the recovery
6073 		 */
6074 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6075 			struct amdgpu_ring *ring = adev->rings[i];
6076 
6077 			if (!amdgpu_ring_sched_ready(ring))
6078 				continue;
6079 
6080 			drm_sched_stop(&ring->sched, NULL);
6081 		}
6082 		atomic_inc(&adev->gpu_reset_counter);
6083 		return PCI_ERS_RESULT_NEED_RESET;
6084 	case pci_channel_io_perm_failure:
6085 		/* Permanent error, prepare for device removal */
6086 		return PCI_ERS_RESULT_DISCONNECT;
6087 	}
6088 
6089 	return PCI_ERS_RESULT_NEED_RESET;
6090 }
6091 
6092 /**
6093  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6094  * @pdev: pointer to PCI device
6095  */
6096 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6097 {
6098 
6099 	DRM_INFO("PCI error: mmio enabled callback!!\n");
6100 
6101 	/* TODO - dump whatever for debugging purposes */
6102 
6103 	/* This called only if amdgpu_pci_error_detected returns
6104 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6105 	 * works, no need to reset slot.
6106 	 */
6107 
6108 	return PCI_ERS_RESULT_RECOVERED;
6109 }
6110 
6111 /**
6112  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6113  * @pdev: PCI device struct
6114  *
6115  * Description: This routine is called by the pci error recovery
6116  * code after the PCI slot has been reset, just before we
6117  * should resume normal operations.
6118  */
6119 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6120 {
6121 	struct drm_device *dev = pci_get_drvdata(pdev);
6122 	struct amdgpu_device *adev = drm_to_adev(dev);
6123 	int r, i;
6124 	struct amdgpu_reset_context reset_context;
6125 	u32 memsize;
6126 	struct list_head device_list;
6127 	struct amdgpu_hive_info *hive;
6128 	int hive_ras_recovery = 0;
6129 	struct amdgpu_ras *ras;
6130 
6131 	/* PCI error slot reset should be skipped During RAS recovery */
6132 	hive = amdgpu_get_xgmi_hive(adev);
6133 	if (hive) {
6134 		hive_ras_recovery = atomic_read(&hive->ras_recovery);
6135 		amdgpu_put_xgmi_hive(hive);
6136 	}
6137 	ras = amdgpu_ras_get_context(adev);
6138 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) &&
6139 		 ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
6140 		return PCI_ERS_RESULT_RECOVERED;
6141 
6142 	DRM_INFO("PCI error: slot reset callback!!\n");
6143 
6144 	memset(&reset_context, 0, sizeof(reset_context));
6145 
6146 	INIT_LIST_HEAD(&device_list);
6147 	list_add_tail(&adev->reset_list, &device_list);
6148 
6149 	/* wait for asic to come out of reset */
6150 	msleep(500);
6151 
6152 	/* Restore PCI confspace */
6153 	amdgpu_device_load_pci_state(pdev);
6154 
6155 	/* confirm  ASIC came out of reset */
6156 	for (i = 0; i < adev->usec_timeout; i++) {
6157 		memsize = amdgpu_asic_get_config_memsize(adev);
6158 
6159 		if (memsize != 0xffffffff)
6160 			break;
6161 		udelay(1);
6162 	}
6163 	if (memsize == 0xffffffff) {
6164 		r = -ETIME;
6165 		goto out;
6166 	}
6167 
6168 	reset_context.method = AMD_RESET_METHOD_NONE;
6169 	reset_context.reset_req_dev = adev;
6170 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6171 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6172 
6173 	adev->no_hw_access = true;
6174 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6175 	adev->no_hw_access = false;
6176 	if (r)
6177 		goto out;
6178 
6179 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
6180 
6181 out:
6182 	if (!r) {
6183 		if (amdgpu_device_cache_pci_state(adev->pdev))
6184 			pci_restore_state(adev->pdev);
6185 
6186 		DRM_INFO("PCIe error recovery succeeded\n");
6187 	} else {
6188 		DRM_ERROR("PCIe error recovery failed, err:%d", r);
6189 		amdgpu_device_unset_mp1_state(adev);
6190 		amdgpu_device_unlock_reset_domain(adev->reset_domain);
6191 	}
6192 
6193 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6194 }
6195 
6196 /**
6197  * amdgpu_pci_resume() - resume normal ops after PCI reset
6198  * @pdev: pointer to PCI device
6199  *
6200  * Called when the error recovery driver tells us that its
6201  * OK to resume normal operation.
6202  */
6203 void amdgpu_pci_resume(struct pci_dev *pdev)
6204 {
6205 	struct drm_device *dev = pci_get_drvdata(pdev);
6206 	struct amdgpu_device *adev = drm_to_adev(dev);
6207 	int i;
6208 
6209 
6210 	DRM_INFO("PCI error: resume callback!!\n");
6211 
6212 	/* Only continue execution for the case of pci_channel_io_frozen */
6213 	if (adev->pci_channel_state != pci_channel_io_frozen)
6214 		return;
6215 
6216 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6217 		struct amdgpu_ring *ring = adev->rings[i];
6218 
6219 		if (!amdgpu_ring_sched_ready(ring))
6220 			continue;
6221 
6222 		drm_sched_start(&ring->sched, true);
6223 	}
6224 
6225 	amdgpu_device_unset_mp1_state(adev);
6226 	amdgpu_device_unlock_reset_domain(adev->reset_domain);
6227 }
6228 
6229 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6230 {
6231 	struct drm_device *dev = pci_get_drvdata(pdev);
6232 	struct amdgpu_device *adev = drm_to_adev(dev);
6233 	int r;
6234 
6235 	r = pci_save_state(pdev);
6236 	if (!r) {
6237 		kfree(adev->pci_state);
6238 
6239 		adev->pci_state = pci_store_saved_state(pdev);
6240 
6241 		if (!adev->pci_state) {
6242 			DRM_ERROR("Failed to store PCI saved state");
6243 			return false;
6244 		}
6245 	} else {
6246 		DRM_WARN("Failed to save PCI state, err:%d\n", r);
6247 		return false;
6248 	}
6249 
6250 	return true;
6251 }
6252 
6253 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6254 {
6255 	struct drm_device *dev = pci_get_drvdata(pdev);
6256 	struct amdgpu_device *adev = drm_to_adev(dev);
6257 	int r;
6258 
6259 	if (!adev->pci_state)
6260 		return false;
6261 
6262 	r = pci_load_saved_state(pdev, adev->pci_state);
6263 
6264 	if (!r) {
6265 		pci_restore_state(pdev);
6266 	} else {
6267 		DRM_WARN("Failed to load PCI state, err:%d\n", r);
6268 		return false;
6269 	}
6270 
6271 	return true;
6272 }
6273 
6274 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6275 		struct amdgpu_ring *ring)
6276 {
6277 #ifdef CONFIG_X86_64
6278 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6279 		return;
6280 #endif
6281 	if (adev->gmc.xgmi.connected_to_cpu)
6282 		return;
6283 
6284 	if (ring && ring->funcs->emit_hdp_flush)
6285 		amdgpu_ring_emit_hdp_flush(ring);
6286 	else
6287 		amdgpu_asic_flush_hdp(adev, ring);
6288 }
6289 
6290 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6291 		struct amdgpu_ring *ring)
6292 {
6293 #ifdef CONFIG_X86_64
6294 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6295 		return;
6296 #endif
6297 	if (adev->gmc.xgmi.connected_to_cpu)
6298 		return;
6299 
6300 	amdgpu_asic_invalidate_hdp(adev, ring);
6301 }
6302 
6303 int amdgpu_in_reset(struct amdgpu_device *adev)
6304 {
6305 	return atomic_read(&adev->reset_domain->in_gpu_reset);
6306 }
6307 
6308 /**
6309  * amdgpu_device_halt() - bring hardware to some kind of halt state
6310  *
6311  * @adev: amdgpu_device pointer
6312  *
6313  * Bring hardware to some kind of halt state so that no one can touch it
6314  * any more. It will help to maintain error context when error occurred.
6315  * Compare to a simple hang, the system will keep stable at least for SSH
6316  * access. Then it should be trivial to inspect the hardware state and
6317  * see what's going on. Implemented as following:
6318  *
6319  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6320  *    clears all CPU mappings to device, disallows remappings through page faults
6321  * 2. amdgpu_irq_disable_all() disables all interrupts
6322  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6323  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6324  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6325  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6326  *    flush any in flight DMA operations
6327  */
6328 void amdgpu_device_halt(struct amdgpu_device *adev)
6329 {
6330 	struct pci_dev *pdev = adev->pdev;
6331 	struct drm_device *ddev = adev_to_drm(adev);
6332 
6333 	amdgpu_xcp_dev_unplug(adev);
6334 	drm_dev_unplug(ddev);
6335 
6336 	amdgpu_irq_disable_all(adev);
6337 
6338 	amdgpu_fence_driver_hw_fini(adev);
6339 
6340 	adev->no_hw_access = true;
6341 
6342 	amdgpu_device_unmap_mmio(adev);
6343 
6344 	pci_disable_device(pdev);
6345 	pci_wait_for_pending_transaction(pdev);
6346 }
6347 
6348 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6349 				u32 reg)
6350 {
6351 	unsigned long flags, address, data;
6352 	u32 r;
6353 
6354 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6355 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6356 
6357 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6358 	WREG32(address, reg * 4);
6359 	(void)RREG32(address);
6360 	r = RREG32(data);
6361 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6362 	return r;
6363 }
6364 
6365 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6366 				u32 reg, u32 v)
6367 {
6368 	unsigned long flags, address, data;
6369 
6370 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6371 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6372 
6373 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6374 	WREG32(address, reg * 4);
6375 	(void)RREG32(address);
6376 	WREG32(data, v);
6377 	(void)RREG32(data);
6378 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6379 }
6380 
6381 /**
6382  * amdgpu_device_switch_gang - switch to a new gang
6383  * @adev: amdgpu_device pointer
6384  * @gang: the gang to switch to
6385  *
6386  * Try to switch to a new gang.
6387  * Returns: NULL if we switched to the new gang or a reference to the current
6388  * gang leader.
6389  */
6390 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6391 					    struct dma_fence *gang)
6392 {
6393 	struct dma_fence *old = NULL;
6394 
6395 	do {
6396 		dma_fence_put(old);
6397 		rcu_read_lock();
6398 		old = dma_fence_get_rcu_safe(&adev->gang_submit);
6399 		rcu_read_unlock();
6400 
6401 		if (old == gang)
6402 			break;
6403 
6404 		if (!dma_fence_is_signaled(old))
6405 			return old;
6406 
6407 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6408 			 old, gang) != old);
6409 
6410 	dma_fence_put(old);
6411 	return NULL;
6412 }
6413 
6414 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6415 {
6416 	switch (adev->asic_type) {
6417 #ifdef CONFIG_DRM_AMDGPU_SI
6418 	case CHIP_HAINAN:
6419 #endif
6420 	case CHIP_TOPAZ:
6421 		/* chips with no display hardware */
6422 		return false;
6423 #ifdef CONFIG_DRM_AMDGPU_SI
6424 	case CHIP_TAHITI:
6425 	case CHIP_PITCAIRN:
6426 	case CHIP_VERDE:
6427 	case CHIP_OLAND:
6428 #endif
6429 #ifdef CONFIG_DRM_AMDGPU_CIK
6430 	case CHIP_BONAIRE:
6431 	case CHIP_HAWAII:
6432 	case CHIP_KAVERI:
6433 	case CHIP_KABINI:
6434 	case CHIP_MULLINS:
6435 #endif
6436 	case CHIP_TONGA:
6437 	case CHIP_FIJI:
6438 	case CHIP_POLARIS10:
6439 	case CHIP_POLARIS11:
6440 	case CHIP_POLARIS12:
6441 	case CHIP_VEGAM:
6442 	case CHIP_CARRIZO:
6443 	case CHIP_STONEY:
6444 		/* chips with display hardware */
6445 		return true;
6446 	default:
6447 		/* IP discovery */
6448 		if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6449 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6450 			return false;
6451 		return true;
6452 	}
6453 }
6454 
6455 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6456 		uint32_t inst, uint32_t reg_addr, char reg_name[],
6457 		uint32_t expected_value, uint32_t mask)
6458 {
6459 	uint32_t ret = 0;
6460 	uint32_t old_ = 0;
6461 	uint32_t tmp_ = RREG32(reg_addr);
6462 	uint32_t loop = adev->usec_timeout;
6463 
6464 	while ((tmp_ & (mask)) != (expected_value)) {
6465 		if (old_ != tmp_) {
6466 			loop = adev->usec_timeout;
6467 			old_ = tmp_;
6468 		} else
6469 			udelay(1);
6470 		tmp_ = RREG32(reg_addr);
6471 		loop--;
6472 		if (!loop) {
6473 			DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6474 				  inst, reg_name, (uint32_t)expected_value,
6475 				  (uint32_t)(tmp_ & (mask)));
6476 			ret = -ETIMEDOUT;
6477 			break;
6478 		}
6479 	}
6480 	return ret;
6481 }
6482