xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c (revision 543b7d2ca09b4b08d518d668ac2fdc361f231785)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/aperture.h>
30 #include <linux/power_supply.h>
31 #include <linux/kthread.h>
32 #include <linux/module.h>
33 #include <linux/console.h>
34 #include <linux/slab.h>
35 #include <linux/iommu.h>
36 #include <linux/pci.h>
37 #include <linux/pci-p2pdma.h>
38 #include <linux/apple-gmux.h>
39 #include <linux/nospec.h>
40 
41 #include <drm/drm_atomic_helper.h>
42 #include <drm/drm_client_event.h>
43 #include <drm/drm_crtc_helper.h>
44 #include <drm/drm_probe_helper.h>
45 #include <drm/amdgpu_drm.h>
46 #include <linux/device.h>
47 #include <linux/vgaarb.h>
48 #include <linux/vga_switcheroo.h>
49 #include <linux/efi.h>
50 #include "amdgpu.h"
51 #include "amdgpu_trace.h"
52 #include "amdgpu_i2c.h"
53 #include "atom.h"
54 #include "amdgpu_atombios.h"
55 #include "amdgpu_atomfirmware.h"
56 #include "amd_pcie.h"
57 #ifdef CONFIG_DRM_AMDGPU_SI
58 #include "si.h"
59 #endif
60 #ifdef CONFIG_DRM_AMDGPU_CIK
61 #include "cik.h"
62 #endif
63 #include "vi.h"
64 #include "soc15.h"
65 #include "nv.h"
66 #include "bif/bif_4_1_d.h"
67 #include <linux/firmware.h>
68 #include "amdgpu_vf_error.h"
69 
70 #include "amdgpu_amdkfd.h"
71 #include "amdgpu_ip.h"
72 #include "amdgpu_pm.h"
73 
74 #include "amdgpu_xgmi.h"
75 #include "amdgpu_ras.h"
76 #include "amdgpu_wb.h"
77 #include "amdgpu_ras_mgr.h"
78 #include "amdgpu_pmu.h"
79 #include "amdgpu_smu.h"
80 #include "amdgpu_fru_eeprom.h"
81 #include "amdgpu_reset.h"
82 #include "amdgpu_virt.h"
83 #include "amdgpu_dev_coredump.h"
84 
85 #include <linux/suspend.h>
86 #include <drm/task_barrier.h>
87 #include <linux/pm_runtime.h>
88 
89 #include <drm/drm_drv.h>
90 
91 #if IS_ENABLED(CONFIG_X86)
92 #include <asm/intel-family.h>
93 #include <asm/cpu_device_id.h>
94 #endif
95 
96 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
97 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
98 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
100 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
101 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
102 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
103 MODULE_FIRMWARE("amdgpu/cyan_skillfish_gpu_info.bin");
104 
105 #define AMDGPU_RESUME_MS		2000
106 #define AMDGPU_MAX_RETRY_LIMIT		2
107 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
108 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
109 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
110 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
111 
112 #define AMDGPU_VBIOS_SKIP (1U << 0)
113 #define AMDGPU_VBIOS_OPTIONAL (1U << 1)
114 
115 static const struct drm_driver amdgpu_kms_driver;
116 
117 const char *amdgpu_asic_name[] = {
118 	"TAHITI",
119 	"PITCAIRN",
120 	"VERDE",
121 	"OLAND",
122 	"HAINAN",
123 	"BONAIRE",
124 	"KAVERI",
125 	"KABINI",
126 	"HAWAII",
127 	"MULLINS",
128 	"TOPAZ",
129 	"TONGA",
130 	"FIJI",
131 	"CARRIZO",
132 	"STONEY",
133 	"POLARIS10",
134 	"POLARIS11",
135 	"POLARIS12",
136 	"VEGAM",
137 	"VEGA10",
138 	"VEGA12",
139 	"VEGA20",
140 	"RAVEN",
141 	"ARCTURUS",
142 	"RENOIR",
143 	"ALDEBARAN",
144 	"NAVI10",
145 	"CYAN_SKILLFISH",
146 	"NAVI14",
147 	"NAVI12",
148 	"SIENNA_CICHLID",
149 	"NAVY_FLOUNDER",
150 	"VANGOGH",
151 	"DIMGREY_CAVEFISH",
152 	"BEIGE_GOBY",
153 	"YELLOW_CARP",
154 	"IP DISCOVERY",
155 	"LAST",
156 };
157 
158 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM  - 1, 0)
159 /*
160  * Default init level where all blocks are expected to be initialized. This is
161  * the level of initialization expected by default and also after a full reset
162  * of the device.
163  */
164 struct amdgpu_init_level amdgpu_init_default = {
165 	.level = AMDGPU_INIT_LEVEL_DEFAULT,
166 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
167 };
168 
169 struct amdgpu_init_level amdgpu_init_recovery = {
170 	.level = AMDGPU_INIT_LEVEL_RESET_RECOVERY,
171 	.hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
172 };
173 
174 /*
175  * Minimal blocks needed to be initialized before a XGMI hive can be reset. This
176  * is used for cases like reset on initialization where the entire hive needs to
177  * be reset before first use.
178  */
179 struct amdgpu_init_level amdgpu_init_minimal_xgmi = {
180 	.level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI,
181 	.hwini_ip_block_mask =
182 		BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) |
183 		BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) |
184 		BIT(AMD_IP_BLOCK_TYPE_PSP)
185 };
186 
187 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev);
188 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev);
189 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev);
190 
191 static void amdgpu_device_load_switch_state(struct amdgpu_device *adev);
192 
193 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,
194 					     enum amd_ip_block_type block)
195 {
196 	return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;
197 }
198 
199 void amdgpu_set_init_level(struct amdgpu_device *adev,
200 			   enum amdgpu_init_lvl_id lvl)
201 {
202 	switch (lvl) {
203 	case AMDGPU_INIT_LEVEL_MINIMAL_XGMI:
204 		adev->init_lvl = &amdgpu_init_minimal_xgmi;
205 		break;
206 	case AMDGPU_INIT_LEVEL_RESET_RECOVERY:
207 		adev->init_lvl = &amdgpu_init_recovery;
208 		break;
209 	case AMDGPU_INIT_LEVEL_DEFAULT:
210 		fallthrough;
211 	default:
212 		adev->init_lvl = &amdgpu_init_default;
213 		break;
214 	}
215 }
216 
217 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
218 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
219 				     void *data);
220 
221 /**
222  * DOC: pcie_replay_count
223  *
224  * The amdgpu driver provides a sysfs API for reporting the total number
225  * of PCIe replays (NAKs).
226  * The file pcie_replay_count is used for this and returns the total
227  * number of replays as a sum of the NAKs generated and NAKs received.
228  */
229 
230 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
231 		struct device_attribute *attr, char *buf)
232 {
233 	struct drm_device *ddev = dev_get_drvdata(dev);
234 	struct amdgpu_device *adev = drm_to_adev(ddev);
235 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
236 
237 	return sysfs_emit(buf, "%llu\n", cnt);
238 }
239 
240 static DEVICE_ATTR(pcie_replay_count, 0444,
241 		amdgpu_device_get_pcie_replay_count, NULL);
242 
243 static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)
244 {
245 	int ret = 0;
246 
247 	if (amdgpu_nbio_is_replay_cnt_supported(adev))
248 		ret = sysfs_create_file(&adev->dev->kobj,
249 					&dev_attr_pcie_replay_count.attr);
250 
251 	if (!ret)
252 		ret = amdgpu_discovery_mem_reserved_info_sysfs_init(adev);
253 
254 	return ret;
255 }
256 
257 static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)
258 {
259 	if (amdgpu_nbio_is_replay_cnt_supported(adev))
260 		sysfs_remove_file(&adev->dev->kobj,
261 				  &dev_attr_pcie_replay_count.attr);
262 
263 	amdgpu_discovery_mem_reserved_info_sysfs_fini(adev);
264 }
265 
266 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
267 					  const struct bin_attribute *attr, char *buf,
268 					  loff_t ppos, size_t count)
269 {
270 	struct device *dev = kobj_to_dev(kobj);
271 	struct drm_device *ddev = dev_get_drvdata(dev);
272 	struct amdgpu_device *adev = drm_to_adev(ddev);
273 	ssize_t bytes_read;
274 
275 	switch (ppos) {
276 	case AMDGPU_SYS_REG_STATE_XGMI:
277 		bytes_read = amdgpu_asic_get_reg_state(
278 			adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
279 		break;
280 	case AMDGPU_SYS_REG_STATE_WAFL:
281 		bytes_read = amdgpu_asic_get_reg_state(
282 			adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
283 		break;
284 	case AMDGPU_SYS_REG_STATE_PCIE:
285 		bytes_read = amdgpu_asic_get_reg_state(
286 			adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
287 		break;
288 	case AMDGPU_SYS_REG_STATE_USR:
289 		bytes_read = amdgpu_asic_get_reg_state(
290 			adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
291 		break;
292 	case AMDGPU_SYS_REG_STATE_USR_1:
293 		bytes_read = amdgpu_asic_get_reg_state(
294 			adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
295 		break;
296 	default:
297 		return -EINVAL;
298 	}
299 
300 	return bytes_read;
301 }
302 
303 static const BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
304 		      AMDGPU_SYS_REG_STATE_END);
305 
306 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
307 {
308 	int ret;
309 
310 	if (!amdgpu_asic_get_reg_state_supported(adev))
311 		return 0;
312 
313 	ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
314 
315 	return ret;
316 }
317 
318 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
319 {
320 	if (!amdgpu_asic_get_reg_state_supported(adev))
321 		return;
322 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
323 }
324 
325 /**
326  * DOC: board_info
327  *
328  * The amdgpu driver provides a sysfs API for giving board related information.
329  * It provides the form factor information in the format
330  *
331  *   type : form factor
332  *
333  * Possible form factor values
334  *
335  * - "cem"		- PCIE CEM card
336  * - "oam"		- Open Compute Accelerator Module
337  * - "eam"		- Enhanced Accelerator Module
338  * - "unknown"	- Not known
339  *
340  */
341 
342 static ssize_t amdgpu_device_get_board_info(struct device *dev,
343 					    struct device_attribute *attr,
344 					    char *buf)
345 {
346 	struct drm_device *ddev = dev_get_drvdata(dev);
347 	struct amdgpu_device *adev = drm_to_adev(ddev);
348 	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
349 	const char *pkg;
350 
351 	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
352 		pkg_type = adev->smuio.funcs->get_pkg_type(adev);
353 
354 	switch (pkg_type) {
355 	case AMDGPU_PKG_TYPE_CEM:
356 		pkg = "cem";
357 		break;
358 	case AMDGPU_PKG_TYPE_OAM:
359 		pkg = "oam";
360 		break;
361 	case AMDGPU_PKG_TYPE_BB:
362 		pkg = "eam";
363 		break;
364 	default:
365 		pkg = "unknown";
366 		break;
367 	}
368 
369 	return sysfs_emit(buf, "%s : %s\n", "type", pkg);
370 }
371 
372 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
373 
374 static struct attribute *amdgpu_board_attrs[] = {
375 	&dev_attr_board_info.attr,
376 	NULL,
377 };
378 
379 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
380 					     struct attribute *attr, int n)
381 {
382 	struct device *dev = kobj_to_dev(kobj);
383 	struct drm_device *ddev = dev_get_drvdata(dev);
384 	struct amdgpu_device *adev = drm_to_adev(ddev);
385 
386 	if (adev->flags & AMD_IS_APU)
387 		return 0;
388 
389 	return attr->mode;
390 }
391 
392 static const struct attribute_group amdgpu_board_attrs_group = {
393 	.attrs = amdgpu_board_attrs,
394 	.is_visible = amdgpu_board_attrs_is_visible
395 };
396 
397 /**
398  * DOC: uma/carveout_options
399  *
400  * This is a read-only file that lists all available UMA allocation
401  * options and their corresponding indices. Example output::
402  *
403  *     $ cat uma/carveout_options
404  *     0: Minimum (512 MB)
405  *     1:  (1 GB)
406  *     2:  (2 GB)
407  *     3:  (4 GB)
408  *     4:  (6 GB)
409  *     5:  (8 GB)
410  *     6:  (12 GB)
411  *     7: Medium (16 GB)
412  *     8:  (24 GB)
413  *     9: High (32 GB)
414  */
415 static ssize_t carveout_options_show(struct device *dev,
416 				     struct device_attribute *attr,
417 				     char *buf)
418 {
419 	struct drm_device *ddev = dev_get_drvdata(dev);
420 	struct amdgpu_device *adev = drm_to_adev(ddev);
421 	struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
422 	uint32_t memory_carved;
423 	ssize_t size = 0;
424 
425 	if (!uma_info || !uma_info->num_entries)
426 		return -ENODEV;
427 
428 	for (int i = 0; i < uma_info->num_entries; i++) {
429 		memory_carved = uma_info->entries[i].memory_carved_mb;
430 		if (memory_carved >= SZ_1G/SZ_1M) {
431 			size += sysfs_emit_at(buf, size, "%d: %s (%u GB)\n",
432 					      i,
433 					      uma_info->entries[i].name,
434 					      memory_carved >> 10);
435 		} else {
436 			size += sysfs_emit_at(buf, size, "%d: %s (%u MB)\n",
437 					      i,
438 					      uma_info->entries[i].name,
439 					      memory_carved);
440 		}
441 	}
442 
443 	return size;
444 }
445 static DEVICE_ATTR_RO(carveout_options);
446 
447 /**
448  * DOC: uma/carveout
449  *
450  * This file is both readable and writable. When read, it shows the
451  * index of the current setting. Writing a valid index to this file
452  * allows users to change the UMA carveout size to the selected option
453  * on the next boot.
454  *
455  * The available options and their corresponding indices can be read
456  * from the uma/carveout_options file.
457  */
458 static ssize_t carveout_show(struct device *dev,
459 			     struct device_attribute *attr,
460 			     char *buf)
461 {
462 	struct drm_device *ddev = dev_get_drvdata(dev);
463 	struct amdgpu_device *adev = drm_to_adev(ddev);
464 
465 	return sysfs_emit(buf, "%u\n", adev->uma_info.uma_option_index);
466 }
467 
468 static ssize_t carveout_store(struct device *dev,
469 			      struct device_attribute *attr,
470 			      const char *buf, size_t count)
471 {
472 	struct drm_device *ddev = dev_get_drvdata(dev);
473 	struct amdgpu_device *adev = drm_to_adev(ddev);
474 	struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
475 	struct amdgpu_uma_carveout_option *opt;
476 	unsigned long val;
477 	uint8_t flags;
478 	int r;
479 
480 	r = kstrtoul(buf, 10, &val);
481 	if (r)
482 		return r;
483 
484 	if (val >= uma_info->num_entries)
485 		return -EINVAL;
486 
487 	val = array_index_nospec(val, uma_info->num_entries);
488 	opt = &uma_info->entries[val];
489 
490 	if (!(opt->flags & AMDGPU_UMA_FLAG_AUTO) &&
491 	    !(opt->flags & AMDGPU_UMA_FLAG_CUSTOM)) {
492 		drm_err_once(ddev, "Option %lu not supported due to lack of Custom/Auto flag", val);
493 		return -EINVAL;
494 	}
495 
496 	flags = opt->flags;
497 	flags &= ~((flags & AMDGPU_UMA_FLAG_AUTO) >> 1);
498 
499 	guard(mutex)(&uma_info->update_lock);
500 
501 	r = amdgpu_acpi_set_uma_allocation_size(adev, val, flags);
502 	if (r)
503 		return r;
504 
505 	uma_info->uma_option_index = val;
506 
507 	return count;
508 }
509 static DEVICE_ATTR_RW(carveout);
510 
511 static struct attribute *amdgpu_uma_attrs[] = {
512 	&dev_attr_carveout.attr,
513 	&dev_attr_carveout_options.attr,
514 	NULL
515 };
516 
517 const struct attribute_group amdgpu_uma_attr_group = {
518 	.name = "uma",
519 	.attrs = amdgpu_uma_attrs
520 };
521 
522 static void amdgpu_uma_sysfs_init(struct amdgpu_device *adev)
523 {
524 	int rc;
525 
526 	if (!(adev->flags & AMD_IS_APU))
527 		return;
528 
529 	if (!amdgpu_acpi_is_set_uma_allocation_size_supported())
530 		return;
531 
532 	rc = amdgpu_atomfirmware_get_uma_carveout_info(adev, &adev->uma_info);
533 	if (rc) {
534 		drm_dbg(adev_to_drm(adev),
535 			"Failed to parse UMA carveout info from VBIOS: %d\n", rc);
536 		goto out_info;
537 	}
538 
539 	mutex_init(&adev->uma_info.update_lock);
540 
541 	rc = devm_device_add_group(adev->dev, &amdgpu_uma_attr_group);
542 	if (rc) {
543 		drm_dbg(adev_to_drm(adev), "Failed to add UMA carveout sysfs interfaces %d\n", rc);
544 		goto out_attr;
545 	}
546 
547 	return;
548 
549 out_attr:
550 	mutex_destroy(&adev->uma_info.update_lock);
551 out_info:
552 	return;
553 }
554 
555 static void amdgpu_uma_sysfs_fini(struct amdgpu_device *adev)
556 {
557 	struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
558 
559 	if (!amdgpu_acpi_is_set_uma_allocation_size_supported())
560 		return;
561 
562 	mutex_destroy(&uma_info->update_lock);
563 	uma_info->num_entries = 0;
564 }
565 
566 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
567 
568 /**
569  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
570  *
571  * @adev: amdgpu device pointer
572  *
573  * Returns true if the device is a dGPU with ATPX power control,
574  * otherwise return false.
575  */
576 bool amdgpu_device_supports_px(struct amdgpu_device *adev)
577 {
578 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
579 		return true;
580 	return false;
581 }
582 
583 /**
584  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
585  *
586  * @adev: amdgpu device pointer
587  *
588  * Returns true if the device is a dGPU with ACPI power control,
589  * otherwise return false.
590  */
591 bool amdgpu_device_supports_boco(struct amdgpu_device *adev)
592 {
593 	if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
594 		return false;
595 
596 	/* BOCO needs a platform power resource (_PR3) to cut the power */
597 	return adev->has_pr3;
598 }
599 
600 /**
601  * amdgpu_device_supports_baco - Does the device support BACO
602  *
603  * @adev: amdgpu device pointer
604  *
605  * Return:
606  * 1 if the device supports BACO;
607  * 3 if the device supports MACO (only works if BACO is supported)
608  * otherwise return 0.
609  */
610 int amdgpu_device_supports_baco(struct amdgpu_device *adev)
611 {
612 	return amdgpu_asic_supports_baco(adev);
613 }
614 
615 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
616 {
617 	int bamaco_support;
618 
619 	adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
620 	if (pci_is_thunderbolt_attached(adev->pdev) ||
621 	    dev_is_removable(&adev->pdev->dev)) {
622 		dev_info(adev->dev,
623 			 "Runtime PM disabled for externally attached device\n");
624 		return;
625 	}
626 
627 	bamaco_support = amdgpu_device_supports_baco(adev);
628 
629 	switch (amdgpu_runtime_pm) {
630 	case 2:
631 		if (bamaco_support & MACO_SUPPORT) {
632 			adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
633 			dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
634 		} else if (bamaco_support == BACO_SUPPORT) {
635 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
636 			dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
637 		}
638 		break;
639 	case 1:
640 		if (bamaco_support & BACO_SUPPORT) {
641 			adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
642 			dev_info(adev->dev, "Forcing BACO for runtime pm\n");
643 		}
644 		break;
645 	case -1:
646 	case -2:
647 		if (amdgpu_device_supports_px(adev)) {
648 			/* enable PX as runtime mode */
649 			adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
650 			dev_info(adev->dev, "Using ATPX for runtime pm\n");
651 		} else if (amdgpu_device_supports_boco(adev)) {
652 			/* enable boco as runtime mode */
653 			adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
654 			dev_info(adev->dev, "Using BOCO for runtime pm\n");
655 		} else {
656 			if (!bamaco_support)
657 				goto no_runtime_pm;
658 
659 			switch (adev->asic_type) {
660 			case CHIP_VEGA20:
661 			case CHIP_ARCTURUS:
662 				/* BACO are not supported on vega20 and arctrus */
663 				break;
664 			case CHIP_VEGA10:
665 				/* enable BACO as runpm mode if noretry=0 */
666 				if (!adev->gmc.noretry && !amdgpu_passthrough(adev))
667 					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
668 				break;
669 			default:
670 				/* enable BACO as runpm mode on CI+ */
671 				if (!amdgpu_passthrough(adev))
672 					adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
673 				break;
674 			}
675 
676 			if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
677 				if (bamaco_support & MACO_SUPPORT) {
678 					adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
679 					dev_info(adev->dev, "Using BAMACO for runtime pm\n");
680 				} else {
681 					dev_info(adev->dev, "Using BACO for runtime pm\n");
682 				}
683 			}
684 		}
685 		break;
686 	case 0:
687 		dev_info(adev->dev, "runtime pm is manually disabled\n");
688 		break;
689 	default:
690 		break;
691 	}
692 
693 no_runtime_pm:
694 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
695 		dev_info(adev->dev, "Runtime PM not available\n");
696 }
697 /**
698  * amdgpu_device_supports_smart_shift - Is the device dGPU with
699  * smart shift support
700  *
701  * @adev: amdgpu device pointer
702  *
703  * Returns true if the device is a dGPU with Smart Shift support,
704  * otherwise returns false.
705  */
706 bool amdgpu_device_supports_smart_shift(struct amdgpu_device *adev)
707 {
708 	return (amdgpu_device_supports_boco(adev) &&
709 		amdgpu_acpi_is_power_shift_control_supported());
710 }
711 
712 /*
713  * VRAM access helper functions
714  */
715 
716 /**
717  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
718  *
719  * @adev: amdgpu_device pointer
720  * @pos: offset of the buffer in vram
721  * @buf: virtual address of the buffer in system memory
722  * @size: read/write size, sizeof(@buf) must > @size
723  * @write: true - write to vram, otherwise - read from vram
724  */
725 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
726 			     void *buf, size_t size, bool write)
727 {
728 	unsigned long flags;
729 	uint32_t hi = ~0, tmp = 0;
730 	uint32_t *data = buf;
731 	uint64_t last;
732 	int idx;
733 
734 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
735 		return;
736 
737 	if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) {
738 		dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n",
739 			pos, size);
740 		drm_dev_exit(idx);
741 		return;
742 	}
743 
744 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
745 	for (last = pos + size; pos < last; pos += 4) {
746 		tmp = pos >> 31;
747 
748 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
749 		if (tmp != hi) {
750 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
751 			hi = tmp;
752 		}
753 		if (write)
754 			WREG32_NO_KIQ(mmMM_DATA, *data++);
755 		else
756 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
757 	}
758 
759 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
760 	drm_dev_exit(idx);
761 }
762 
763 #ifdef CONFIG_64BIT
764 /*
765  * During early SR-IOV VF init, host-provided init data can live in FB before
766  * the normal VRAM aperture mapping is ready.  Use a temporary BAR0 mapping for
767  * reads only, and verify it matches the VRAM aperture when aperture information
768  * is already available.
769  */
770 static int amdgpu_device_read_fb_via_bar0(struct amdgpu_device *adev,
771 					  u64 offset, void *buf, size_t size)
772 {
773 	resource_size_t aper_base, aper_size, bar_start, bar_size, map_base;
774 	void __iomem *vram;
775 	size_t map_offset, map_size;
776 	unsigned long flags;
777 	u64 end;
778 
779 	if (!buf || !size)
780 		return -EINVAL;
781 
782 	if (!amdgpu_sriov_vf(adev))
783 		return -EINVAL;
784 
785 	flags = pci_resource_flags(adev->pdev, 0);
786 	if ((flags & IORESOURCE_UNSET) || !(flags & IORESOURCE_MEM))
787 		return -EINVAL;
788 
789 	bar_start = pci_resource_start(adev->pdev, 0);
790 	bar_size = pci_resource_len(adev->pdev, 0);
791 	if (!bar_size)
792 		return -ENODEV;
793 
794 	aper_base = adev->gmc.aper_base;
795 	aper_size = adev->gmc.visible_vram_size ? adev->gmc.visible_vram_size :
796 						  adev->gmc.aper_size;
797 
798 	if (aper_base || aper_size) {
799 		if (aper_base != bar_start || aper_size > bar_size)
800 			return -EINVAL;
801 	} else {
802 		aper_base = bar_start;
803 		aper_size = bar_size;
804 	}
805 
806 	if (check_add_overflow(offset, size, &end) || end > aper_size)
807 		return -EINVAL;
808 
809 	map_offset = offset_in_page(offset);
810 	map_base = aper_base + (offset & PAGE_MASK);
811 	map_size = PAGE_ALIGN(map_offset + size);
812 
813 	vram = ioremap_wc(map_base, map_size);
814 	if (!vram)
815 		return -ENOMEM;
816 
817 	memcpy_fromio(buf, (u8 __iomem *)vram + map_offset, size);
818 	iounmap(vram);
819 
820 	return 0;
821 }
822 #endif
823 
824 /**
825  * amdgpu_device_aper_access - access vram by vram aperture
826  *
827  * @adev: amdgpu_device pointer
828  * @pos: offset of the buffer in vram
829  * @buf: virtual address of the buffer in system memory
830  * @size: read/write size, sizeof(@buf) must > @size
831  * @write: true - write to vram, otherwise - read from vram
832  *
833  * The return value means how many bytes have been transferred.
834  */
835 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
836 				 void *buf, size_t size, bool write)
837 {
838 #ifdef CONFIG_64BIT
839 	void __iomem *addr;
840 	size_t count = 0;
841 	uint64_t last;
842 
843 	if (!adev->mman.aper_base_kaddr) {
844 		/* Writes still require the regular aperture/MM path. */
845 		if (!write && !amdgpu_device_read_fb_via_bar0(adev, pos, buf, size))
846 			return size;
847 		return 0;
848 	}
849 
850 	last = min(pos + size, adev->gmc.visible_vram_size);
851 	if (last > pos) {
852 		addr = adev->mman.aper_base_kaddr + pos;
853 		count = last - pos;
854 
855 		if (write) {
856 			memcpy_toio(addr, buf, count);
857 			/* Make sure HDP write cache flush happens without any reordering
858 			 * after the system memory contents are sent over PCIe device
859 			 */
860 			mb();
861 			amdgpu_device_flush_hdp(adev, NULL);
862 		} else {
863 			amdgpu_device_invalidate_hdp(adev, NULL);
864 			/* Make sure HDP read cache is invalidated before issuing a read
865 			 * to the PCIe device
866 			 */
867 			mb();
868 			memcpy_fromio(buf, addr, count);
869 		}
870 
871 	}
872 
873 	return count;
874 #else
875 	return 0;
876 #endif
877 }
878 
879 /**
880  * amdgpu_device_vram_access - read/write a buffer in vram
881  *
882  * @adev: amdgpu_device pointer
883  * @pos: offset of the buffer in vram
884  * @buf: virtual address of the buffer in system memory
885  * @size: read/write size, sizeof(@buf) must > @size
886  * @write: true - write to vram, otherwise - read from vram
887  */
888 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
889 			       void *buf, size_t size, bool write)
890 {
891 	size_t count;
892 
893 	/* try to using vram apreature to access vram first */
894 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
895 	size -= count;
896 	if (size) {
897 		/* using MM to access rest vram */
898 		pos += count;
899 		buf += count;
900 		amdgpu_device_mm_access(adev, pos, buf, size, write);
901 	}
902 }
903 
904 /*
905  * register access helper functions.
906  */
907 
908 /* Check if hw access should be skipped because of hotplug or device error */
909 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
910 {
911 	if (adev->no_hw_access)
912 		return true;
913 
914 #ifdef CONFIG_LOCKDEP
915 	/*
916 	 * This is a bit complicated to understand, so worth a comment. What we assert
917 	 * here is that the GPU reset is not running on another thread in parallel.
918 	 *
919 	 * For this we trylock the read side of the reset semaphore, if that succeeds
920 	 * we know that the reset is not running in parallel.
921 	 *
922 	 * If the trylock fails we assert that we are either already holding the read
923 	 * side of the lock or are the reset thread itself and hold the write side of
924 	 * the lock.
925 	 */
926 	if (in_task()) {
927 		if (down_read_trylock(&adev->reset_domain->sem))
928 			up_read(&adev->reset_domain->sem);
929 		else
930 			lockdep_assert_held(&adev->reset_domain->sem);
931 	}
932 #endif
933 	return false;
934 }
935 
936 /**
937  * amdgpu_device_get_rev_id - query device rev_id
938  *
939  * @adev: amdgpu_device pointer
940  *
941  * Return device rev_id
942  */
943 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
944 {
945 	return adev->nbio.funcs->get_rev_id(adev);
946 }
947 
948 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)
949 {
950 	if (amdgpu_virt_vram_is_spm(adev))
951 		return AMDGPU_VBIOS_OPTIONAL;
952 
953 	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
954 		return AMDGPU_VBIOS_SKIP;
955 
956 	if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))
957 		return AMDGPU_VBIOS_OPTIONAL;
958 
959 	return 0;
960 }
961 
962 /**
963  * amdgpu_device_asic_init - Wrapper for atom asic_init
964  *
965  * @adev: amdgpu_device pointer
966  *
967  * Does any asic specific work and then calls atom asic init.
968  */
969 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
970 {
971 	uint32_t flags;
972 	bool optional;
973 	int ret;
974 
975 	amdgpu_asic_pre_asic_init(adev);
976 	flags = amdgpu_device_get_vbios_flags(adev);
977 	optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP));
978 
979 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
980 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
981 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
982 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
983 		amdgpu_psp_wait_for_bootloader(adev);
984 		if (optional && !adev->bios)
985 			return 0;
986 
987 		ret = amdgpu_atomfirmware_asic_init(adev, true);
988 		return ret;
989 	} else {
990 		if (optional && !adev->bios)
991 			return 0;
992 
993 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
994 	}
995 
996 	return 0;
997 }
998 
999 /**
1000  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1001  *
1002  * @adev: amdgpu_device pointer
1003  *
1004  * Allocates a scratch page of VRAM for use by various things in the
1005  * driver.
1006  */
1007 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1008 {
1009 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1010 				       AMDGPU_GEM_DOMAIN_VRAM |
1011 				       AMDGPU_GEM_DOMAIN_GTT,
1012 				       &adev->mem_scratch.robj,
1013 				       &adev->mem_scratch.gpu_addr,
1014 				       (void **)&adev->mem_scratch.ptr);
1015 }
1016 
1017 /**
1018  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1019  *
1020  * @adev: amdgpu_device pointer
1021  *
1022  * Frees the VRAM scratch page.
1023  */
1024 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1025 {
1026 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1027 }
1028 
1029 /**
1030  * amdgpu_device_program_register_sequence - program an array of registers.
1031  *
1032  * @adev: amdgpu_device pointer
1033  * @registers: pointer to the register array
1034  * @array_size: size of the register array
1035  *
1036  * Programs an array or registers with and or masks.
1037  * This is a helper for setting golden registers.
1038  */
1039 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1040 					     const u32 *registers,
1041 					     const u32 array_size)
1042 {
1043 	u32 tmp, reg, and_mask, or_mask;
1044 	int i;
1045 
1046 	if (array_size % 3)
1047 		return;
1048 
1049 	for (i = 0; i < array_size; i += 3) {
1050 		reg = registers[i + 0];
1051 		and_mask = registers[i + 1];
1052 		or_mask = registers[i + 2];
1053 
1054 		if (and_mask == 0xffffffff) {
1055 			tmp = or_mask;
1056 		} else {
1057 			tmp = RREG32(reg);
1058 			tmp &= ~and_mask;
1059 			if (adev->family >= AMDGPU_FAMILY_AI)
1060 				tmp |= (or_mask & and_mask);
1061 			else
1062 				tmp |= or_mask;
1063 		}
1064 		WREG32(reg, tmp);
1065 	}
1066 }
1067 
1068 /**
1069  * amdgpu_device_pci_config_reset - reset the GPU
1070  *
1071  * @adev: amdgpu_device pointer
1072  *
1073  * Resets the GPU using the pci config reset sequence.
1074  * Only applicable to asics prior to vega10.
1075  */
1076 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1077 {
1078 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1079 }
1080 
1081 /**
1082  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1083  *
1084  * @adev: amdgpu_device pointer
1085  *
1086  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1087  */
1088 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1089 {
1090 	return pci_reset_function(adev->pdev);
1091 }
1092 
1093 /**
1094  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1095  *
1096  * @adev: amdgpu_device pointer
1097  *
1098  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1099  * to fail, but if any of the BARs is not accessible after the size we abort
1100  * driver loading by returning -ENODEV.
1101  */
1102 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1103 {
1104 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1105 	struct pci_bus *root;
1106 	struct resource *res;
1107 	int max_size, r;
1108 	unsigned int i;
1109 	u16 cmd;
1110 
1111 	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1112 		return 0;
1113 
1114 	/* Bypass for VF */
1115 	if (amdgpu_sriov_vf(adev))
1116 		return 0;
1117 
1118 	if (!amdgpu_rebar)
1119 		return 0;
1120 
1121 	/* resizing on Dell G5 SE platforms causes problems with runtime pm */
1122 	if ((amdgpu_runtime_pm != 0) &&
1123 	    adev->pdev->vendor == PCI_VENDOR_ID_ATI &&
1124 	    adev->pdev->device == 0x731f &&
1125 	    adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
1126 		return 0;
1127 
1128 	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1129 	if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1130 		dev_warn(
1131 			adev->dev,
1132 			"System can't access extended configuration space, please check!!\n");
1133 
1134 	/* skip if the bios has already enabled large BAR */
1135 	if (adev->gmc.real_vram_size &&
1136 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1137 		return 0;
1138 
1139 	/* Check if the root BUS has 64bit memory resources */
1140 	root = adev->pdev->bus;
1141 	while (root->parent)
1142 		root = root->parent;
1143 
1144 	pci_bus_for_each_resource(root, res, i) {
1145 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1146 		    res->start > 0x100000000ull)
1147 			break;
1148 	}
1149 
1150 	/* Trying to resize is pointless without a root hub window above 4GB */
1151 	if (!res)
1152 		return 0;
1153 
1154 	/* Limit the BAR size to what is available */
1155 	max_size = pci_rebar_get_max_size(adev->pdev, 0);
1156 	if (max_size < 0)
1157 		return 0;
1158 	rbar_size = min(max_size, rbar_size);
1159 
1160 	/* Disable memory decoding while we change the BAR addresses and size */
1161 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1162 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1163 			      cmd & ~PCI_COMMAND_MEMORY);
1164 
1165 	/* Tear down doorbell as resizing will release BARs */
1166 	amdgpu_doorbell_fini(adev);
1167 
1168 	r = pci_resize_resource(adev->pdev, 0, rbar_size,
1169 				(adev->asic_type >= CHIP_BONAIRE) ? 1 << 5
1170 								  : 1 << 2);
1171 	if (r == -ENOSPC)
1172 		dev_info(adev->dev,
1173 			 "Not enough PCI address space for a large BAR.");
1174 	else if (r && r != -ENOTSUPP)
1175 		dev_err(adev->dev, "Problem resizing BAR0 (%d).", r);
1176 
1177 	/* When the doorbell or fb BAR isn't available we have no chance of
1178 	 * using the device.
1179 	 */
1180 	r = amdgpu_doorbell_init(adev);
1181 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1182 		return -ENODEV;
1183 
1184 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1185 
1186 	return 0;
1187 }
1188 
1189 /*
1190  * GPU helpers function.
1191  */
1192 /**
1193  * amdgpu_device_need_post - check if the hw need post or not
1194  *
1195  * @adev: amdgpu_device pointer
1196  *
1197  * Check if the asic has been initialized (all asics) at driver startup
1198  * or post is needed if  hw reset is performed.
1199  * Returns true if need or false if not.
1200  */
1201 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1202 {
1203 	uint32_t reg, flags;
1204 
1205 	if (amdgpu_sriov_vf(adev))
1206 		return false;
1207 
1208 	flags = amdgpu_device_get_vbios_flags(adev);
1209 	if (flags & AMDGPU_VBIOS_SKIP)
1210 		return false;
1211 	if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)
1212 		return false;
1213 
1214 	if (amdgpu_passthrough(adev)) {
1215 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1216 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1217 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1218 		 * vpost executed for smc version below 22.15
1219 		 */
1220 		if (adev->asic_type == CHIP_FIJI) {
1221 			int err;
1222 			uint32_t fw_ver;
1223 
1224 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1225 			/* force vPost if error occurred */
1226 			if (err)
1227 				return true;
1228 
1229 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1230 			release_firmware(adev->pm.fw);
1231 			if (fw_ver < 0x00160e00)
1232 				return true;
1233 		}
1234 	}
1235 
1236 	/* Don't post if we need to reset whole hive on init */
1237 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
1238 		return false;
1239 
1240 	if (adev->has_hw_reset) {
1241 		adev->has_hw_reset = false;
1242 		return true;
1243 	}
1244 
1245 	/* bios scratch used on CIK+ */
1246 	if (adev->asic_type >= CHIP_BONAIRE)
1247 		return amdgpu_atombios_scratch_need_asic_init(adev);
1248 
1249 	/* check MEM_SIZE for older asics */
1250 	reg = amdgpu_asic_get_config_memsize(adev);
1251 
1252 	if ((reg != 0) && (reg != 0xffffffff))
1253 		return false;
1254 
1255 	return true;
1256 }
1257 
1258 /*
1259  * Check whether seamless boot is supported.
1260  *
1261  * So far we only support seamless boot on DCE 3.0 or later.
1262  * If users report that it works on older ASICS as well, we may
1263  * loosen this.
1264  */
1265 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1266 {
1267 	switch (amdgpu_seamless) {
1268 	case -1:
1269 		break;
1270 	case 1:
1271 		return true;
1272 	case 0:
1273 		return false;
1274 	default:
1275 		dev_err(adev->dev, "Invalid value for amdgpu.seamless: %d\n",
1276 			amdgpu_seamless);
1277 		return false;
1278 	}
1279 
1280 	if (!(adev->flags & AMD_IS_APU))
1281 		return false;
1282 
1283 	if (adev->mman.keep_stolen_vga_memory)
1284 		return false;
1285 
1286 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1287 }
1288 
1289 #if IS_ENABLED(CONFIG_X86)
1290 static const struct x86_cpu_id amdgpu_pcie_dynamic_switching_quirks[] = {
1291 	/*
1292 	 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1293 	 * don't support dynamic speed switching. Until we have confirmation from Intel
1294 	 * that a specific host supports it, it's safer that we keep it disabled for all.
1295 	 *
1296 	 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1297 	 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1298 	 */
1299 	X86_MATCH_VENDOR_FAM(INTEL, X86_FAMILY_ANY, NULL),
1300 	/*
1301 	 * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't
1302 	 * support PCIe dynamic speed switching.
1303 	 * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436
1304 	 */
1305 	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x17, 0x08, NULL),
1306 	{}
1307 };
1308 
1309 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1310 {
1311 	/* eGPU change speeds based on USB4 fabric conditions */
1312 	if (dev_is_removable(adev->dev))
1313 		return true;
1314 
1315 	/* Hosts have problems with dynamic speed switching */
1316 	if (x86_match_cpu(amdgpu_pcie_dynamic_switching_quirks))
1317 		return false;
1318 
1319 	return true;
1320 }
1321 #else
1322 static inline bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1323 {
1324 	return true;
1325 }
1326 #endif
1327 
1328 static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev)
1329 {
1330 	/* Enabling ASPM causes randoms hangs on Tahiti and Oland on Zen4.
1331 	 * It's unclear if this is a platform-specific or GPU-specific issue.
1332 	 * Disable ASPM on SI for the time being.
1333 	 */
1334 	if (adev->family == AMDGPU_FAMILY_SI ||
1335 		(!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI))
1336 		return true;
1337 
1338 #if IS_ENABLED(CONFIG_X86)
1339 	struct cpuinfo_x86 *c = &cpu_data(0);
1340 
1341 	if (c->x86_vendor == X86_VENDOR_INTEL) {
1342 		switch (c->x86_model) {
1343 		case VFM_MODEL(INTEL_ALDERLAKE):
1344 		case VFM_MODEL(INTEL_ALDERLAKE_L):
1345 		case VFM_MODEL(INTEL_RAPTORLAKE):
1346 		case VFM_MODEL(INTEL_RAPTORLAKE_P):
1347 		case VFM_MODEL(INTEL_RAPTORLAKE_S):
1348 		case VFM_MODEL(INTEL_TIGERLAKE):
1349 		case VFM_MODEL(INTEL_TIGERLAKE_L):
1350 			return true;
1351 		default:
1352 			return false;
1353 		}
1354 	} else {
1355 		return false;
1356 	}
1357 #else
1358 	return false;
1359 #endif
1360 }
1361 
1362 /**
1363  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1364  *
1365  * @adev: amdgpu_device pointer
1366  *
1367  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1368  * be set for this device.
1369  *
1370  * Returns true if it should be used or false if not.
1371  */
1372 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1373 {
1374 	struct pci_dev *aspm_pdev, *parent;
1375 	bool enabled;
1376 
1377 	switch (amdgpu_aspm) {
1378 	case -1:
1379 		break;
1380 	case 0:
1381 		return false;
1382 	case 1:
1383 		return true;
1384 	default:
1385 		return false;
1386 	}
1387 	if (adev->flags & AMD_IS_APU)
1388 		return false;
1389 	if (amdgpu_device_aspm_support_quirk(adev))
1390 		return false;
1391 
1392 	/*
1393 	 * pcie_aspm_enabled() checks the link between its argument and
1394 	 * the immediate upstream bridge. Use SWUS for dGPUs with an
1395 	 * internal switch so that this is the host-facing link.
1396 	 */
1397 	aspm_pdev = adev->link_dev;
1398 	parent = pci_upstream_bridge(aspm_pdev);
1399 	if (!parent) {
1400 		dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n",
1401 			pci_name(aspm_pdev));
1402 		return false;
1403 	}
1404 
1405 	enabled = pcie_aspm_enabled(aspm_pdev);
1406 	/* Report the exact link used for the automatic ASPM decision. */
1407 	dev_dbg(adev->dev, "ASPM: link %s <-> %s is %s\n",
1408 		pci_name(parent), pci_name(aspm_pdev),
1409 		enabled ? "enabled" : "disabled");
1410 
1411 	return enabled;
1412 }
1413 
1414 /* if we get transitioned to only one device, take VGA back */
1415 /**
1416  * amdgpu_device_vga_set_decode - enable/disable vga decode
1417  *
1418  * @data: driver private data (adev)
1419  * @state: enable/disable vga decode
1420  *
1421  * Enable/disable vga decode (all asics).
1422  * Returns VGA resource flags.
1423  */
1424 static unsigned int amdgpu_device_vga_set_decode(void *data, bool state)
1425 {
1426 	struct amdgpu_device *adev = data;
1427 
1428 	amdgpu_asic_set_vga_state(adev, state);
1429 	if (state)
1430 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1431 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1432 	else
1433 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1434 }
1435 
1436 /**
1437  * amdgpu_device_check_block_size - validate the vm block size
1438  *
1439  * @adev: amdgpu_device pointer
1440  *
1441  * Validates the vm block size specified via module parameter.
1442  * The vm block size defines number of bits in page table versus page directory,
1443  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1444  * page table and the remaining bits are in the page directory.
1445  */
1446 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1447 {
1448 	/* defines number of bits in page table versus page directory,
1449 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1450 	 * page table and the remaining bits are in the page directory
1451 	 */
1452 	if (amdgpu_vm_block_size == -1)
1453 		return;
1454 
1455 	if (amdgpu_vm_block_size < 9) {
1456 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1457 			 amdgpu_vm_block_size);
1458 		amdgpu_vm_block_size = -1;
1459 	}
1460 }
1461 
1462 /**
1463  * amdgpu_device_check_vm_size - validate the vm size
1464  *
1465  * @adev: amdgpu_device pointer
1466  *
1467  * Validates the vm size in GB specified via module parameter.
1468  * The VM size is the size of the GPU virtual memory space in GB.
1469  */
1470 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1471 {
1472 	/* no need to check the default value */
1473 	if (amdgpu_vm_size == -1)
1474 		return;
1475 
1476 	if (amdgpu_vm_size < 1) {
1477 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1478 			 amdgpu_vm_size);
1479 		amdgpu_vm_size = -1;
1480 	}
1481 }
1482 
1483 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1484 {
1485 	struct sysinfo si;
1486 	bool is_os_64 = (sizeof(void *) == 8);
1487 	uint64_t total_memory;
1488 	uint64_t dram_size_seven_GB = 0x1B8000000;
1489 	uint64_t dram_size_three_GB = 0xB8000000;
1490 
1491 	if (amdgpu_smu_memory_pool_size == 0)
1492 		return;
1493 
1494 	if (!is_os_64) {
1495 		dev_warn(adev->dev, "Not 64-bit OS, feature not supported\n");
1496 		goto def_value;
1497 	}
1498 	si_meminfo(&si);
1499 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1500 
1501 	if ((amdgpu_smu_memory_pool_size == 1) ||
1502 		(amdgpu_smu_memory_pool_size == 2)) {
1503 		if (total_memory < dram_size_three_GB)
1504 			goto def_value1;
1505 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1506 		(amdgpu_smu_memory_pool_size == 8)) {
1507 		if (total_memory < dram_size_seven_GB)
1508 			goto def_value1;
1509 	} else {
1510 		dev_warn(adev->dev, "Smu memory pool size not supported\n");
1511 		goto def_value;
1512 	}
1513 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1514 
1515 	return;
1516 
1517 def_value1:
1518 	dev_warn(adev->dev, "No enough system memory\n");
1519 def_value:
1520 	adev->pm.smu_prv_buffer_size = 0;
1521 }
1522 
1523 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1524 {
1525 	if (!(adev->flags & AMD_IS_APU) ||
1526 	    adev->asic_type < CHIP_RAVEN)
1527 		return 0;
1528 
1529 	switch (adev->asic_type) {
1530 	case CHIP_RAVEN:
1531 		if (adev->pdev->device == 0x15dd)
1532 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1533 		if (adev->pdev->device == 0x15d8)
1534 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1535 		break;
1536 	case CHIP_RENOIR:
1537 		if ((adev->pdev->device == 0x1636) ||
1538 		    (adev->pdev->device == 0x164c))
1539 			adev->apu_flags |= AMD_APU_IS_RENOIR;
1540 		else
1541 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1542 		break;
1543 	case CHIP_VANGOGH:
1544 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
1545 		break;
1546 	case CHIP_YELLOW_CARP:
1547 		break;
1548 	case CHIP_CYAN_SKILLFISH:
1549 		if ((adev->pdev->device == 0x13FE) ||
1550 		    (adev->pdev->device == 0x143F))
1551 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1552 		break;
1553 	default:
1554 		break;
1555 	}
1556 
1557 	return 0;
1558 }
1559 
1560 /**
1561  * amdgpu_device_check_arguments - validate module params
1562  *
1563  * @adev: amdgpu_device pointer
1564  *
1565  * Validates certain module parameters and updates
1566  * the associated values used by the driver (all asics).
1567  */
1568 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1569 {
1570 	int i;
1571 
1572 	if (amdgpu_sched_jobs < 4) {
1573 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1574 			 amdgpu_sched_jobs);
1575 		amdgpu_sched_jobs = 4;
1576 	} else if (!is_power_of_2(amdgpu_sched_jobs)) {
1577 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1578 			 amdgpu_sched_jobs);
1579 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1580 	}
1581 
1582 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1583 		/* gart size must be greater or equal to 32M */
1584 		dev_warn(adev->dev, "gart size (%d) too small\n",
1585 			 amdgpu_gart_size);
1586 		amdgpu_gart_size = -1;
1587 	}
1588 
1589 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1590 		/* gtt size must be greater or equal to 32M */
1591 		dev_warn(adev->dev, "gtt size (%d) too small\n",
1592 				 amdgpu_gtt_size);
1593 		amdgpu_gtt_size = -1;
1594 	}
1595 
1596 	/* valid range is between 4 and 9 inclusive */
1597 	if (amdgpu_vm_fragment_size != -1 &&
1598 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1599 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
1600 		amdgpu_vm_fragment_size = -1;
1601 	}
1602 
1603 	if (amdgpu_sched_hw_submission < 2) {
1604 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1605 			 amdgpu_sched_hw_submission);
1606 		amdgpu_sched_hw_submission = 2;
1607 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1608 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1609 			 amdgpu_sched_hw_submission);
1610 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1611 	}
1612 
1613 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1614 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1615 		amdgpu_reset_method = -1;
1616 	}
1617 
1618 	amdgpu_device_check_smu_prv_buffer_size(adev);
1619 
1620 	amdgpu_device_check_vm_size(adev);
1621 
1622 	amdgpu_device_check_block_size(adev);
1623 
1624 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1625 
1626 	for (i = 0; i < MAX_XCP; i++) {
1627 		switch (amdgpu_enforce_isolation) {
1628 		case -1:
1629 		case 0:
1630 		default:
1631 			/* disable */
1632 			adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE;
1633 			break;
1634 		case 1:
1635 			/* enable */
1636 			adev->enforce_isolation[i] =
1637 				AMDGPU_ENFORCE_ISOLATION_ENABLE;
1638 			break;
1639 		case 2:
1640 			/* enable legacy mode */
1641 			adev->enforce_isolation[i] =
1642 				AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY;
1643 			break;
1644 		case 3:
1645 			/* enable only process isolation without submitting cleaner shader */
1646 			adev->enforce_isolation[i] =
1647 				AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER;
1648 			break;
1649 		}
1650 	}
1651 
1652 	return 0;
1653 }
1654 
1655 /**
1656  * amdgpu_switcheroo_set_state - set switcheroo state
1657  *
1658  * @pdev: pci dev pointer
1659  * @state: vga_switcheroo state
1660  *
1661  * Callback for the switcheroo driver.  Suspends or resumes
1662  * the asics before or after it is powered up using ACPI methods.
1663  */
1664 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1665 					enum vga_switcheroo_state state)
1666 {
1667 	struct drm_device *dev = pci_get_drvdata(pdev);
1668 	int r;
1669 
1670 	if (amdgpu_device_supports_px(drm_to_adev(dev)) &&
1671 	    state == VGA_SWITCHEROO_OFF)
1672 		return;
1673 
1674 	if (state == VGA_SWITCHEROO_ON) {
1675 		pr_info("switched on\n");
1676 		/* don't suspend or resume card normally */
1677 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1678 
1679 		pci_set_power_state(pdev, PCI_D0);
1680 		amdgpu_device_load_pci_state(pdev);
1681 		r = pci_enable_device(pdev);
1682 		if (r)
1683 			dev_warn(&pdev->dev, "pci_enable_device failed (%d)\n",
1684 				 r);
1685 		amdgpu_device_resume(dev, true);
1686 
1687 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
1688 	} else {
1689 		dev_info(&pdev->dev, "switched off\n");
1690 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1691 		amdgpu_device_prepare(dev);
1692 		amdgpu_device_suspend(dev, true);
1693 		amdgpu_device_cache_pci_state(pdev);
1694 		/* Shut down the device */
1695 		pci_disable_device(pdev);
1696 		pci_set_power_state(pdev, PCI_D3cold);
1697 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1698 	}
1699 }
1700 
1701 /**
1702  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1703  *
1704  * @pdev: pci dev pointer
1705  *
1706  * Callback for the switcheroo driver.  Check of the switcheroo
1707  * state can be changed.
1708  * Returns true if the state can be changed, false if not.
1709  */
1710 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1711 {
1712 	struct drm_device *dev = pci_get_drvdata(pdev);
1713 
1714        /*
1715 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1716 	* locking inversion with the driver load path. And the access here is
1717 	* completely racy anyway. So don't bother with locking for now.
1718 	*/
1719 	return atomic_read(&dev->open_count) == 0;
1720 }
1721 
1722 static void amdgpu_switcheroo_pre_switch(struct pci_dev *pdev)
1723 {
1724 	struct drm_device *dev = pci_get_drvdata(pdev);
1725 
1726 	drm_client_dev_acquire_outputs(dev);
1727 }
1728 
1729 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1730 	.set_gpu_state = amdgpu_switcheroo_set_state,
1731 	.can_switch = amdgpu_switcheroo_can_switch,
1732 	.pre_switch = amdgpu_switcheroo_pre_switch,
1733 };
1734 
1735 /**
1736  * amdgpu_device_enable_virtual_display - enable virtual display feature
1737  *
1738  * @adev: amdgpu_device pointer
1739  *
1740  * Enabled the virtual display feature if the user has enabled it via
1741  * the module parameter virtual_display.  This feature provides a virtual
1742  * display hardware on headless boards or in virtualized environments.
1743  * This function parses and validates the configuration string specified by
1744  * the user and configures the virtual display configuration (number of
1745  * virtual connectors, crtcs, etc.) specified.
1746  */
1747 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1748 {
1749 	adev->enable_virtual_display = false;
1750 
1751 	if (amdgpu_virtual_display) {
1752 		const char *pci_address_name = pci_name(adev->pdev);
1753 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1754 
1755 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1756 		pciaddstr_tmp = pciaddstr;
1757 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1758 			pciaddname = strsep(&pciaddname_tmp, ",");
1759 			if (!strcmp("all", pciaddname)
1760 			    || !strcmp(pci_address_name, pciaddname)) {
1761 				long num_crtc;
1762 				int res = -1;
1763 
1764 				adev->enable_virtual_display = true;
1765 
1766 				if (pciaddname_tmp)
1767 					res = kstrtol(pciaddname_tmp, 10,
1768 						      &num_crtc);
1769 
1770 				if (!res) {
1771 					if (num_crtc < 1)
1772 						num_crtc = 1;
1773 					if (num_crtc > 6)
1774 						num_crtc = 6;
1775 					adev->mode_info.num_crtc = num_crtc;
1776 				} else {
1777 					adev->mode_info.num_crtc = 1;
1778 				}
1779 				break;
1780 			}
1781 		}
1782 
1783 		dev_info(
1784 			adev->dev,
1785 			"virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1786 			amdgpu_virtual_display, pci_address_name,
1787 			adev->enable_virtual_display, adev->mode_info.num_crtc);
1788 
1789 		kfree(pciaddstr);
1790 	}
1791 }
1792 
1793 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1794 {
1795 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1796 		adev->mode_info.num_crtc = 1;
1797 		adev->enable_virtual_display = true;
1798 		dev_info(adev->dev, "virtual_display:%d, num_crtc:%d\n",
1799 			 adev->enable_virtual_display,
1800 			 adev->mode_info.num_crtc);
1801 	}
1802 }
1803 
1804 /**
1805  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1806  *
1807  * @adev: amdgpu_device pointer
1808  *
1809  * Parses the asic configuration parameters specified in the gpu info
1810  * firmware and makes them available to the driver for use in configuring
1811  * the asic.
1812  * Returns 0 on success, -EINVAL on failure.
1813  */
1814 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1815 {
1816 	const char *chip_name;
1817 	int err;
1818 	const struct gpu_info_firmware_header_v1_0 *hdr;
1819 
1820 	adev->firmware.gpu_info_fw = NULL;
1821 
1822 	switch (adev->asic_type) {
1823 	default:
1824 		return 0;
1825 	case CHIP_VEGA10:
1826 		chip_name = "vega10";
1827 		break;
1828 	case CHIP_VEGA12:
1829 		chip_name = "vega12";
1830 		break;
1831 	case CHIP_RAVEN:
1832 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1833 			chip_name = "raven2";
1834 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1835 			chip_name = "picasso";
1836 		else
1837 			chip_name = "raven";
1838 		break;
1839 	case CHIP_ARCTURUS:
1840 		chip_name = "arcturus";
1841 		break;
1842 	case CHIP_NAVI12:
1843 		if (adev->discovery.bin)
1844 			return 0;
1845 		chip_name = "navi12";
1846 		break;
1847 	case CHIP_CYAN_SKILLFISH:
1848 		if (adev->discovery.bin)
1849 			return 0;
1850 		chip_name = "cyan_skillfish";
1851 		break;
1852 	}
1853 
1854 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,
1855 				   AMDGPU_UCODE_OPTIONAL,
1856 				   "amdgpu/%s_gpu_info.bin", chip_name);
1857 	if (err) {
1858 		dev_err(adev->dev,
1859 			"Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n",
1860 			chip_name);
1861 		goto out;
1862 	}
1863 
1864 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1865 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1866 
1867 	switch (hdr->version_major) {
1868 	case 1:
1869 	{
1870 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1871 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1872 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1873 
1874 		/*
1875 		 * Should be dropped when DAL no longer needs it.
1876 		 */
1877 		if (adev->asic_type == CHIP_NAVI12)
1878 			goto parse_soc_bounding_box;
1879 
1880 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1881 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1882 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1883 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1884 		adev->gfx.config.max_texture_channel_caches =
1885 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
1886 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1887 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1888 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1889 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1890 		adev->gfx.config.double_offchip_lds_buf =
1891 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1892 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1893 		adev->gfx.cu_info.max_waves_per_simd =
1894 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1895 		adev->gfx.cu_info.max_scratch_slots_per_cu =
1896 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1897 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1898 		if (hdr->version_minor >= 1) {
1899 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1900 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1901 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1902 			adev->gfx.config.num_sc_per_sh =
1903 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1904 			adev->gfx.config.num_packer_per_sc =
1905 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1906 		}
1907 
1908 parse_soc_bounding_box:
1909 		/*
1910 		 * soc bounding box info is not integrated in disocovery table,
1911 		 * we always need to parse it from gpu info firmware if needed.
1912 		 */
1913 		if (hdr->version_minor == 2) {
1914 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1915 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1916 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1917 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1918 		}
1919 		break;
1920 	}
1921 	default:
1922 		dev_err(adev->dev,
1923 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1924 		err = -EINVAL;
1925 		goto out;
1926 	}
1927 out:
1928 	return err;
1929 }
1930 
1931 static void amdgpu_uid_init(struct amdgpu_device *adev)
1932 {
1933 	/* Initialize the UID for the device */
1934 	adev->uid_info = kzalloc_obj(struct amdgpu_uid);
1935 	if (!adev->uid_info) {
1936 		dev_warn(adev->dev, "Failed to allocate memory for UID\n");
1937 		return;
1938 	}
1939 	adev->uid_info->adev = adev;
1940 }
1941 
1942 static void amdgpu_uid_fini(struct amdgpu_device *adev)
1943 {
1944 	/* Free the UID memory */
1945 	kfree(adev->uid_info);
1946 	adev->uid_info = NULL;
1947 }
1948 
1949 static void amdgpu_device_init_pcie_links(struct amdgpu_device *adev)
1950 {
1951 	struct pci_dev *swds, *swus;
1952 
1953 	adev->link_dev = adev->pdev;
1954 	adev->link_partner = pci_upstream_bridge(adev->link_dev);
1955 
1956 	/* Skip an upstream/downstream switch internal to the dGPU. */
1957 	swds = adev->link_partner;
1958 	if (!swds || swds->vendor != PCI_VENDOR_ID_ATI ||
1959 	    pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
1960 		return;
1961 
1962 	swus = pci_upstream_bridge(swds);
1963 	if (!swus ||
1964 	    (swus->vendor != PCI_VENDOR_ID_ATI &&
1965 	     swus->vendor != PCI_VENDOR_ID_AMD) ||
1966 	    pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
1967 		return;
1968 
1969 	adev->link_dev = swus;
1970 	adev->link_partner = pci_upstream_bridge(swus);
1971 }
1972 
1973 /**
1974  * amdgpu_device_ip_early_init - run early init for hardware IPs
1975  *
1976  * @adev: amdgpu_device pointer
1977  *
1978  * Early initialization pass for hardware IPs.  The hardware IPs that make
1979  * up each asic are discovered each IP's early_init callback is run.  This
1980  * is the first stage in initializing the asic.
1981  * Returns 0 on success, negative error code on failure.
1982  */
1983 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1984 {
1985 	struct amdgpu_ip_block *ip_block;
1986 	bool total, skip_bios, early_full_gpu_access = false;
1987 	uint32_t bios_flags;
1988 	int i, r;
1989 
1990 	amdgpu_device_enable_virtual_display(adev);
1991 
1992 	if (amdgpu_sriov_vf(adev)) {
1993 		/*
1994 		 * Legacy hosts do not provide init data before early init, so
1995 		 * keep the original early full GPU access request for them.  Newer
1996 		 * hosts publish the init data through VF FB, which lets us defer
1997 		 * full GPU access until after non-GPU early init work is done.
1998 		 */
1999 		early_full_gpu_access = (adev->virt.req_init_data_ver == 0);
2000 		if (early_full_gpu_access) {
2001 			r = amdgpu_virt_request_full_gpu(adev, true);
2002 			if (r)
2003 				return r;
2004 		}
2005 
2006 		r = amdgpu_virt_init_critical_region(adev);
2007 		if (r)
2008 			return r;
2009 	}
2010 
2011 	switch (adev->asic_type) {
2012 #ifdef CONFIG_DRM_AMDGPU_SI
2013 	case CHIP_VERDE:
2014 	case CHIP_TAHITI:
2015 	case CHIP_PITCAIRN:
2016 	case CHIP_OLAND:
2017 	case CHIP_HAINAN:
2018 		adev->family = AMDGPU_FAMILY_SI;
2019 		r = si_set_ip_blocks(adev);
2020 		if (r)
2021 			return r;
2022 		break;
2023 #endif
2024 #ifdef CONFIG_DRM_AMDGPU_CIK
2025 	case CHIP_BONAIRE:
2026 	case CHIP_HAWAII:
2027 	case CHIP_KAVERI:
2028 	case CHIP_KABINI:
2029 	case CHIP_MULLINS:
2030 		if (adev->flags & AMD_IS_APU)
2031 			adev->family = AMDGPU_FAMILY_KV;
2032 		else
2033 			adev->family = AMDGPU_FAMILY_CI;
2034 
2035 		r = cik_set_ip_blocks(adev);
2036 		if (r)
2037 			return r;
2038 		break;
2039 #endif
2040 	case CHIP_TOPAZ:
2041 	case CHIP_TONGA:
2042 	case CHIP_FIJI:
2043 	case CHIP_POLARIS10:
2044 	case CHIP_POLARIS11:
2045 	case CHIP_POLARIS12:
2046 	case CHIP_VEGAM:
2047 	case CHIP_CARRIZO:
2048 	case CHIP_STONEY:
2049 		if (adev->flags & AMD_IS_APU)
2050 			adev->family = AMDGPU_FAMILY_CZ;
2051 		else
2052 			adev->family = AMDGPU_FAMILY_VI;
2053 
2054 		r = vi_set_ip_blocks(adev);
2055 		if (r)
2056 			return r;
2057 		break;
2058 	default:
2059 		r = amdgpu_discovery_set_ip_blocks(adev);
2060 		if (r) {
2061 			adev->num_ip_blocks = 0;
2062 			return r;
2063 		}
2064 		break;
2065 	}
2066 
2067 	/* Check for IP version 9.4.3 with A0 hardware */
2068 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
2069 	    !amdgpu_device_get_rev_id(adev)) {
2070 		dev_err(adev->dev, "Unsupported A0 hardware\n");
2071 		return -ENODEV;	/* device unsupported - no device error */
2072 	}
2073 
2074 	if (amdgpu_has_atpx() &&
2075 	    (amdgpu_is_atpx_hybrid() ||
2076 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2077 	    ((adev->flags & AMD_IS_APU) == 0) &&
2078 	    !dev_is_removable(&adev->pdev->dev))
2079 		adev->flags |= AMD_IS_PX;
2080 
2081 	if (!(adev->flags & AMD_IS_APU)) {
2082 		adev->has_pr3 = adev->link_partner &&
2083 			pci_pr3_present(adev->link_partner);
2084 
2085 		if ((amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) &&
2086 		    !pci_resource_len(adev->pdev, 0))
2087 			adev->virt.caps |= AMDGPU_VIRT_VRAM_IS_SPM;
2088 	}
2089 
2090 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2091 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2092 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2093 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2094 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2095 	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2096 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2097 
2098 	adev->virt.is_xgmi_node_migrate_enabled = false;
2099 	if (amdgpu_sriov_vf(adev)) {
2100 		adev->virt.is_xgmi_node_migrate_enabled =
2101 			amdgpu_ip_version((adev), GC_HWIP, 0) == IP_VERSION(9, 4, 4);
2102 	}
2103 
2104 	total = true;
2105 	for (i = 0; i < adev->num_ip_blocks; i++) {
2106 		ip_block = &adev->ip_blocks[i];
2107 
2108 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2109 			dev_warn(adev->dev, "disabled ip block: %d <%s>\n", i,
2110 				 adev->ip_blocks[i].version->funcs->name);
2111 			adev->ip_blocks[i].status.valid = false;
2112 		} else if (ip_block->version->funcs->early_init) {
2113 			r = ip_block->version->funcs->early_init(ip_block);
2114 			if (r == -ENOENT) {
2115 				adev->ip_blocks[i].status.valid = false;
2116 			} else if (r) {
2117 				dev_err(adev->dev,
2118 					"early_init of IP block <%s> failed %d\n",
2119 					adev->ip_blocks[i].version->funcs->name,
2120 					r);
2121 				total = false;
2122 			} else {
2123 				adev->ip_blocks[i].status.valid = true;
2124 			}
2125 		} else {
2126 			adev->ip_blocks[i].status.valid = true;
2127 		}
2128 		/* get the vbios after the asic_funcs are set up */
2129 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2130 			r = amdgpu_device_parse_gpu_info_fw(adev);
2131 			if (r)
2132 				return r;
2133 
2134 			bios_flags = amdgpu_device_get_vbios_flags(adev);
2135 			skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP);
2136 			/* Read BIOS */
2137 			if (!skip_bios) {
2138 				bool optional =
2139 					!!(bios_flags & AMDGPU_VBIOS_OPTIONAL);
2140 				if (!amdgpu_get_bios(adev) && !optional)
2141 					return -EINVAL;
2142 
2143 				if (optional && !adev->bios)
2144 					dev_info(
2145 						adev->dev,
2146 						"VBIOS image optional, proceeding without VBIOS image");
2147 
2148 				if (adev->bios) {
2149 					r = amdgpu_atombios_init(adev);
2150 					if (r) {
2151 						dev_err(adev->dev,
2152 							"amdgpu_atombios_init failed\n");
2153 						amdgpu_vf_error_put(
2154 							adev,
2155 							AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL,
2156 							0, 0);
2157 						return r;
2158 					}
2159 				}
2160 			}
2161 
2162 			/*get pf2vf msg info at it's earliest time*/
2163 			if (amdgpu_sriov_vf(adev))
2164 				amdgpu_virt_init_data_exchange(adev);
2165 
2166 		}
2167 	}
2168 	if (!total)
2169 		return -ENODEV;
2170 
2171 	/* Request full GPU access only for the remaining SR-IOV init work. */
2172 	if (amdgpu_sriov_vf(adev) && !early_full_gpu_access) {
2173 		r = amdgpu_virt_request_full_gpu(adev, true);
2174 		if (r)
2175 			return r;
2176 	}
2177 
2178 	if (adev->gmc.xgmi.supported)
2179 		amdgpu_xgmi_early_init(adev);
2180 
2181 	if (amdgpu_is_multi_aid(adev))
2182 		amdgpu_uid_init(adev);
2183 	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
2184 	if (ip_block->status.valid != false)
2185 		amdgpu_amdkfd_device_probe(adev);
2186 
2187 	adev->cg_flags &= amdgpu_cg_mask;
2188 	adev->pg_flags &= amdgpu_pg_mask;
2189 
2190 	amdgpu_smu_early_init(adev);
2191 
2192 	return 0;
2193 }
2194 
2195 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2196 {
2197 	int i, r;
2198 
2199 	for (i = 0; i < adev->num_ip_blocks; i++) {
2200 		if (!adev->ip_blocks[i].status.sw)
2201 			continue;
2202 		if (adev->ip_blocks[i].status.hw)
2203 			continue;
2204 		if (!amdgpu_ip_member_of_hwini(
2205 			    adev, adev->ip_blocks[i].version->type))
2206 			continue;
2207 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2208 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2209 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2210 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2211 			if (r) {
2212 				dev_err(adev->dev,
2213 					"hw_init of IP block <%s> failed %d\n",
2214 					adev->ip_blocks[i].version->funcs->name,
2215 					r);
2216 				return r;
2217 			}
2218 			adev->ip_blocks[i].status.hw = true;
2219 		}
2220 	}
2221 
2222 	return 0;
2223 }
2224 
2225 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2226 {
2227 	int i, r;
2228 
2229 	for (i = 0; i < adev->num_ip_blocks; i++) {
2230 		if (!adev->ip_blocks[i].status.sw)
2231 			continue;
2232 		if (adev->ip_blocks[i].status.hw)
2233 			continue;
2234 		if (!amdgpu_ip_member_of_hwini(
2235 			    adev, adev->ip_blocks[i].version->type))
2236 			continue;
2237 		r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2238 		if (r) {
2239 			dev_err(adev->dev,
2240 				"hw_init of IP block <%s> failed %d\n",
2241 				adev->ip_blocks[i].version->funcs->name, r);
2242 			return r;
2243 		}
2244 		adev->ip_blocks[i].status.hw = true;
2245 	}
2246 
2247 	return 0;
2248 }
2249 
2250 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2251 {
2252 	int r = 0;
2253 	int i;
2254 	uint32_t smu_version;
2255 
2256 	if (adev->asic_type >= CHIP_VEGA10) {
2257 		for (i = 0; i < adev->num_ip_blocks; i++) {
2258 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2259 				continue;
2260 
2261 			if (!amdgpu_ip_member_of_hwini(adev,
2262 						       AMD_IP_BLOCK_TYPE_PSP))
2263 				break;
2264 
2265 			if (!adev->ip_blocks[i].status.sw)
2266 				continue;
2267 
2268 			/* no need to do the fw loading again if already done*/
2269 			if (adev->ip_blocks[i].status.hw == true)
2270 				break;
2271 
2272 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2273 				r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
2274 				if (r)
2275 					return r;
2276 			} else {
2277 				r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2278 				if (r) {
2279 					dev_err(adev->dev,
2280 						"hw_init of IP block <%s> failed %d\n",
2281 						adev->ip_blocks[i]
2282 							.version->funcs->name,
2283 						r);
2284 					return r;
2285 				}
2286 				adev->ip_blocks[i].status.hw = true;
2287 			}
2288 			break;
2289 		}
2290 	}
2291 
2292 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2293 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2294 
2295 	return r;
2296 }
2297 
2298 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2299 {
2300 	struct drm_sched_init_args args = {
2301 		.ops = &amdgpu_sched_ops,
2302 		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
2303 		.timeout_wq = adev->reset_domain->wq,
2304 		.dev = adev->dev,
2305 	};
2306 	long timeout;
2307 	int r, i;
2308 
2309 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2310 		struct amdgpu_ring *ring = adev->rings[i];
2311 
2312 		/* No need to setup the GPU scheduler for rings that don't need it */
2313 		if (!ring || ring->no_scheduler)
2314 			continue;
2315 
2316 		switch (ring->funcs->type) {
2317 		case AMDGPU_RING_TYPE_GFX:
2318 			timeout = adev->gfx_timeout;
2319 			break;
2320 		case AMDGPU_RING_TYPE_COMPUTE:
2321 			timeout = adev->compute_timeout;
2322 			break;
2323 		case AMDGPU_RING_TYPE_SDMA:
2324 			timeout = adev->sdma_timeout;
2325 			break;
2326 		default:
2327 			timeout = adev->video_timeout;
2328 			break;
2329 		}
2330 
2331 		args.timeout = timeout;
2332 		args.credit_limit = ring->num_hw_submission;
2333 		args.score = ring->sched_score;
2334 		args.name = ring->name;
2335 
2336 		r = drm_sched_init(&ring->sched, &args);
2337 		if (r) {
2338 			dev_err(adev->dev,
2339 				"Failed to create scheduler on ring %s.\n",
2340 				ring->name);
2341 			return r;
2342 		}
2343 		r = amdgpu_uvd_entity_init(adev, ring);
2344 		if (r) {
2345 			dev_err(adev->dev,
2346 				"Failed to create UVD scheduling entity on ring %s.\n",
2347 				ring->name);
2348 			return r;
2349 		}
2350 		r = amdgpu_vce_entity_init(adev, ring);
2351 		if (r) {
2352 			dev_err(adev->dev,
2353 				"Failed to create VCE scheduling entity on ring %s.\n",
2354 				ring->name);
2355 			return r;
2356 		}
2357 	}
2358 
2359 	if (adev->xcp_mgr)
2360 		amdgpu_xcp_update_partition_sched_list(adev);
2361 
2362 	return 0;
2363 }
2364 
2365 
2366 /**
2367  * amdgpu_device_ip_init - run init for hardware IPs
2368  *
2369  * @adev: amdgpu_device pointer
2370  *
2371  * Main initialization pass for hardware IPs.  The list of all the hardware
2372  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2373  * are run.  sw_init initializes the software state associated with each IP
2374  * and hw_init initializes the hardware associated with each IP.
2375  * Returns 0 on success, negative error code on failure.
2376  */
2377 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2378 {
2379 	bool init_badpage;
2380 	int i, r;
2381 
2382 	r = amdgpu_ras_init(adev);
2383 	if (r)
2384 		return r;
2385 
2386 	for (i = 0; i < adev->num_ip_blocks; i++) {
2387 		if (!adev->ip_blocks[i].status.valid)
2388 			continue;
2389 		if (adev->ip_blocks[i].version->funcs->sw_init) {
2390 			r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);
2391 			if (r) {
2392 				dev_err(adev->dev,
2393 					"sw_init of IP block <%s> failed %d\n",
2394 					adev->ip_blocks[i].version->funcs->name,
2395 					r);
2396 				goto init_failed;
2397 			}
2398 		}
2399 		adev->ip_blocks[i].status.sw = true;
2400 
2401 		if (!amdgpu_ip_member_of_hwini(
2402 			    adev, adev->ip_blocks[i].version->type))
2403 			continue;
2404 
2405 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2406 			/* need to do common hw init early so everything is set up for gmc */
2407 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2408 			if (r) {
2409 				dev_err(adev->dev, "hw_init %d failed %d\n", i,
2410 					r);
2411 				goto init_failed;
2412 			}
2413 			adev->ip_blocks[i].status.hw = true;
2414 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2415 			/* need to do gmc hw init early so we can allocate gpu mem */
2416 			/* Try to reserve bad pages early */
2417 			if (amdgpu_sriov_vf(adev))
2418 				amdgpu_virt_exchange_data(adev);
2419 
2420 			r = amdgpu_device_mem_scratch_init(adev);
2421 			if (r) {
2422 				dev_err(adev->dev,
2423 					"amdgpu_mem_scratch_init failed %d\n",
2424 					r);
2425 				goto init_failed;
2426 			}
2427 			r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2428 			if (r) {
2429 				dev_err(adev->dev, "hw_init %d failed %d\n", i,
2430 					r);
2431 				goto init_failed;
2432 			}
2433 			r = amdgpu_wb_init(adev);
2434 			if (r) {
2435 				dev_err(adev->dev,
2436 					"amdgpu_wb_init failed %d\n", r);
2437 				goto init_failed;
2438 			}
2439 			adev->ip_blocks[i].status.hw = true;
2440 
2441 			/* right after GMC hw init, we create CSA */
2442 			if (adev->gfx.mcbp) {
2443 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2444 							       AMDGPU_GEM_DOMAIN_VRAM |
2445 							       AMDGPU_GEM_DOMAIN_GTT,
2446 							       AMDGPU_CSA_SIZE);
2447 				if (r) {
2448 					dev_err(adev->dev,
2449 						"allocate CSA failed %d\n", r);
2450 					goto init_failed;
2451 				}
2452 			}
2453 
2454 			r = amdgpu_seq64_init(adev);
2455 			if (r) {
2456 				dev_err(adev->dev, "allocate seq64 failed %d\n",
2457 					r);
2458 				goto init_failed;
2459 			}
2460 		}
2461 	}
2462 
2463 	if (amdgpu_sriov_vf(adev))
2464 		amdgpu_virt_init_data_exchange(adev);
2465 
2466 	r = amdgpu_ib_pool_init(adev);
2467 	if (r) {
2468 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2469 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2470 		goto init_failed;
2471 	}
2472 
2473 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2474 	if (r)
2475 		goto init_failed;
2476 
2477 	r = amdgpu_device_ip_hw_init_phase1(adev);
2478 	if (r)
2479 		goto init_failed;
2480 
2481 	r = amdgpu_device_fw_loading(adev);
2482 	if (r)
2483 		goto init_failed;
2484 
2485 	r = amdgpu_device_ip_hw_init_phase2(adev);
2486 	if (r)
2487 		goto init_failed;
2488 
2489 	/*
2490 	 * retired pages will be loaded from eeprom and reserved here,
2491 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
2492 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2493 	 * for I2C communication which only true at this point.
2494 	 *
2495 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2496 	 * failure from bad gpu situation and stop amdgpu init process
2497 	 * accordingly. For other failed cases, it will still release all
2498 	 * the resource and print error message, rather than returning one
2499 	 * negative value to upper level.
2500 	 *
2501 	 * Note: theoretically, this should be called before all vram allocations
2502 	 * to protect retired page from abusing
2503 	 */
2504 	init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
2505 	r = amdgpu_ras_recovery_init(adev, init_badpage);
2506 	if (r)
2507 		goto init_failed;
2508 
2509 	/**
2510 	 * In case of XGMI grab extra reference for reset domain for this device
2511 	 */
2512 	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2513 	    adev->gmc.xgmi.supported) {
2514 		if (amdgpu_xgmi_add_device(adev) == 0) {
2515 			if (!amdgpu_sriov_vf(adev)) {
2516 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2517 
2518 				if (WARN_ON(!hive)) {
2519 					r = -ENOENT;
2520 					goto init_failed;
2521 				}
2522 
2523 				if (!hive->reset_domain ||
2524 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2525 					r = -ENOENT;
2526 					amdgpu_put_xgmi_hive(hive);
2527 					goto init_failed;
2528 				}
2529 
2530 				/* Drop the early temporary reset domain we created for device */
2531 				amdgpu_reset_put_reset_domain(adev->reset_domain);
2532 				adev->reset_domain = hive->reset_domain;
2533 				amdgpu_put_xgmi_hive(hive);
2534 			}
2535 		}
2536 	}
2537 
2538 	r = amdgpu_device_init_schedulers(adev);
2539 	if (r)
2540 		goto init_failed;
2541 
2542 	/* If SDMA is not brought up during hwini, the ttm buffer funcs enablement
2543 	 * is delayed after reset-on-init completes.
2544 	 */
2545 	if (amdgpu_ip_member_of_hwini(adev, AMD_IP_BLOCK_TYPE_SDMA))
2546 		amdgpu_ttm_enable_buffer_funcs(adev);
2547 
2548 	/* Don't init kfd if whole hive need to be reset during init */
2549 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
2550 		amdgpu_amdkfd_device_init(adev);
2551 	}
2552 
2553 	amdgpu_fru_get_product_info(adev);
2554 
2555 	r = amdgpu_cper_init(adev);
2556 
2557 init_failed:
2558 
2559 	return r;
2560 }
2561 
2562 /**
2563  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2564  *
2565  * @adev: amdgpu_device pointer
2566  *
2567  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2568  * this function before a GPU reset.  If the value is retained after a
2569  * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents.
2570  */
2571 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2572 {
2573 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2574 }
2575 
2576 /**
2577  * amdgpu_device_check_vram_lost - check if vram is valid
2578  *
2579  * @adev: amdgpu_device pointer
2580  *
2581  * Checks the reset magic value written to the gart pointer in VRAM.
2582  * The driver calls this after a GPU reset to see if the contents of
2583  * VRAM is lost or now.
2584  * returns true if vram is lost, false if not.
2585  */
2586 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2587 {
2588 	if (memcmp(adev->gart.ptr, adev->reset_magic,
2589 			AMDGPU_RESET_MAGIC_NUM))
2590 		return true;
2591 
2592 	if (!amdgpu_in_reset(adev))
2593 		return false;
2594 
2595 	/*
2596 	 * For all ASICs with baco/mode1 reset, the VRAM is
2597 	 * always assumed to be lost.
2598 	 */
2599 	switch (amdgpu_asic_reset_method(adev)) {
2600 	case AMD_RESET_METHOD_LEGACY:
2601 	case AMD_RESET_METHOD_LINK:
2602 	case AMD_RESET_METHOD_BACO:
2603 	case AMD_RESET_METHOD_MODE1:
2604 		return true;
2605 	default:
2606 		return false;
2607 	}
2608 }
2609 
2610 /**
2611  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2612  *
2613  * @adev: amdgpu_device pointer
2614  * @state: clockgating state (gate or ungate)
2615  *
2616  * The list of all the hardware IPs that make up the asic is walked and the
2617  * set_clockgating_state callbacks are run.
2618  * Late initialization pass enabling clockgating for hardware IPs.
2619  * Fini or suspend, pass disabling clockgating for hardware IPs.
2620  * Returns 0 on success, negative error code on failure.
2621  */
2622 
2623 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2624 			       enum amd_clockgating_state state)
2625 {
2626 	int i, j, r;
2627 
2628 	if (amdgpu_emu_mode == 1)
2629 		return 0;
2630 
2631 	for (j = 0; j < adev->num_ip_blocks; j++) {
2632 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2633 		if (!adev->ip_blocks[i].status.late_initialized)
2634 			continue;
2635 		if (!adev->ip_blocks[i].version)
2636 			continue;
2637 		/* skip CG for GFX, SDMA on S0ix */
2638 		if (adev->in_s0ix &&
2639 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2640 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2641 			continue;
2642 		/* skip CG for VCE/UVD, it's handled specially */
2643 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2644 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2645 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2646 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2647 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2648 			/* enable clockgating to save power */
2649 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i],
2650 										     state);
2651 			if (r) {
2652 				dev_err(adev->dev,
2653 					"set_clockgating_state(gate) of IP block <%s> failed %d\n",
2654 					adev->ip_blocks[i].version->funcs->name,
2655 					r);
2656 				return r;
2657 			}
2658 		}
2659 	}
2660 
2661 	return 0;
2662 }
2663 
2664 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2665 			       enum amd_powergating_state state)
2666 {
2667 	int i, j, r;
2668 
2669 	if (amdgpu_emu_mode == 1)
2670 		return 0;
2671 
2672 	for (j = 0; j < adev->num_ip_blocks; j++) {
2673 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2674 		if (!adev->ip_blocks[i].status.late_initialized)
2675 			continue;
2676 		if (!adev->ip_blocks[i].version)
2677 			continue;
2678 		/* skip PG for GFX, SDMA on S0ix */
2679 		if (adev->in_s0ix &&
2680 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2681 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2682 			continue;
2683 		/* skip CG for VCE/UVD, it's handled specially */
2684 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2685 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2686 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2687 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2688 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
2689 			/* enable powergating to save power */
2690 			r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i],
2691 											state);
2692 			if (r) {
2693 				dev_err(adev->dev,
2694 					"set_powergating_state(gate) of IP block <%s> failed %d\n",
2695 					adev->ip_blocks[i].version->funcs->name,
2696 					r);
2697 				return r;
2698 			}
2699 		}
2700 	}
2701 	return 0;
2702 }
2703 
2704 static int amdgpu_device_enable_mgpu_fan_boost(void)
2705 {
2706 	struct amdgpu_gpu_instance *gpu_ins;
2707 	struct amdgpu_device *adev;
2708 	int i, ret = 0;
2709 
2710 	mutex_lock(&mgpu_info.mutex);
2711 
2712 	/*
2713 	 * MGPU fan boost feature should be enabled
2714 	 * only when there are two or more dGPUs in
2715 	 * the system
2716 	 */
2717 	if (mgpu_info.num_dgpu < 2)
2718 		goto out;
2719 
2720 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
2721 		gpu_ins = &(mgpu_info.gpu_ins[i]);
2722 		adev = gpu_ins->adev;
2723 		if (!(adev->flags & AMD_IS_APU || amdgpu_sriov_multi_vf_mode(adev)) &&
2724 		    !gpu_ins->mgpu_fan_enabled) {
2725 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2726 			if (ret)
2727 				break;
2728 
2729 			gpu_ins->mgpu_fan_enabled = 1;
2730 		}
2731 	}
2732 
2733 out:
2734 	mutex_unlock(&mgpu_info.mutex);
2735 
2736 	return ret;
2737 }
2738 
2739 /**
2740  * amdgpu_device_ip_late_init - run late init for hardware IPs
2741  *
2742  * @adev: amdgpu_device pointer
2743  *
2744  * Late initialization pass for hardware IPs.  The list of all the hardware
2745  * IPs that make up the asic is walked and the late_init callbacks are run.
2746  * late_init covers any special initialization that an IP requires
2747  * after all of the have been initialized or something that needs to happen
2748  * late in the init process.
2749  * Returns 0 on success, negative error code on failure.
2750  */
2751 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2752 {
2753 	struct amdgpu_gpu_instance *gpu_instance;
2754 	int i = 0, r;
2755 
2756 	for (i = 0; i < adev->num_ip_blocks; i++) {
2757 		if (!adev->ip_blocks[i].status.hw)
2758 			continue;
2759 		if (adev->ip_blocks[i].version->funcs->late_init) {
2760 			r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
2761 			if (r) {
2762 				dev_err(adev->dev,
2763 					"late_init of IP block <%s> failed %d\n",
2764 					adev->ip_blocks[i].version->funcs->name,
2765 					r);
2766 				return r;
2767 			}
2768 		}
2769 		adev->ip_blocks[i].status.late_initialized = true;
2770 	}
2771 
2772 	r = amdgpu_ras_late_init(adev);
2773 	if (r) {
2774 		dev_err(adev->dev, "amdgpu_ras_late_init failed %d", r);
2775 		return r;
2776 	}
2777 
2778 	if (!amdgpu_reset_in_recovery(adev))
2779 		amdgpu_ras_set_error_query_ready(adev, true);
2780 
2781 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2782 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2783 
2784 	r = amdgpu_dpm_reapply_umd_profile(adev);
2785 	if (r) {
2786 		dev_err(adev->dev, "Failed to restore UMD profile state: %d\n", r);
2787 		return r;
2788 	}
2789 
2790 	amdgpu_device_fill_reset_magic(adev);
2791 
2792 	r = amdgpu_device_enable_mgpu_fan_boost();
2793 	if (r)
2794 		dev_err(adev->dev, "enable mgpu fan boost failed (%d).\n", r);
2795 
2796 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2797 	if (amdgpu_passthrough(adev) &&
2798 	    ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2799 	     adev->asic_type == CHIP_ALDEBARAN))
2800 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
2801 
2802 	if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported) {
2803 		mutex_lock(&mgpu_info.mutex);
2804 
2805 		/*
2806 		 * Reset device p-state to low as this was booted with high.
2807 		 *
2808 		 * This should be performed only after all devices from the same
2809 		 * hive get initialized.
2810 		 *
2811 		 * However, it's unknown how many device in the hive in advance.
2812 		 * As this is counted one by one during devices initializations.
2813 		 *
2814 		 * So, we wait for all XGMI interlinked devices initialized.
2815 		 * This may bring some delays as those devices may come from
2816 		 * different hives. But that should be OK.
2817 		 */
2818 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2819 			for (i = 0; i < mgpu_info.num_gpu; i++) {
2820 				gpu_instance = &(mgpu_info.gpu_ins[i]);
2821 				if (gpu_instance->adev->flags & AMD_IS_APU)
2822 					continue;
2823 
2824 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2825 						AMDGPU_XGMI_PSTATE_MIN);
2826 				if (r) {
2827 					dev_err(adev->dev,
2828 						"pstate setting failed (%d).\n",
2829 						r);
2830 					break;
2831 				}
2832 			}
2833 		}
2834 
2835 		mutex_unlock(&mgpu_info.mutex);
2836 	}
2837 
2838 	return 0;
2839 }
2840 
2841 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block)
2842 {
2843 	struct amdgpu_device *adev = ip_block->adev;
2844 	int r;
2845 
2846 	if (!ip_block->version->funcs->hw_fini) {
2847 		dev_err(adev->dev, "hw_fini of IP block <%s> not defined\n",
2848 			ip_block->version->funcs->name);
2849 	} else {
2850 		r = ip_block->version->funcs->hw_fini(ip_block);
2851 		/* XXX handle errors */
2852 		if (r) {
2853 			dev_dbg(adev->dev,
2854 				"hw_fini of IP block <%s> failed %d\n",
2855 				ip_block->version->funcs->name, r);
2856 		}
2857 	}
2858 
2859 	ip_block->status.hw = false;
2860 }
2861 
2862 /**
2863  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2864  *
2865  * @adev: amdgpu_device pointer
2866  *
2867  * For ASICs need to disable SMC first
2868  */
2869 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2870 {
2871 	int i;
2872 
2873 	if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
2874 		return;
2875 
2876 	for (i = 0; i < adev->num_ip_blocks; i++) {
2877 		if (!adev->ip_blocks[i].status.hw)
2878 			continue;
2879 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2880 			amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
2881 			break;
2882 		}
2883 	}
2884 }
2885 
2886 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2887 {
2888 	int i, r;
2889 
2890 	for (i = 0; i < adev->num_ip_blocks; i++) {
2891 		if (!adev->ip_blocks[i].version)
2892 			continue;
2893 		if (!adev->ip_blocks[i].version->funcs->early_fini)
2894 			continue;
2895 
2896 		r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);
2897 		if (r) {
2898 			dev_dbg(adev->dev,
2899 				"early_fini of IP block <%s> failed %d\n",
2900 				adev->ip_blocks[i].version->funcs->name, r);
2901 		}
2902 	}
2903 
2904 	amdgpu_amdkfd_suspend(adev, true);
2905 	amdgpu_amdkfd_teardown_processes(adev);
2906 	amdgpu_userq_suspend(adev);
2907 
2908 	/* Workaround for ASICs need to disable SMC first */
2909 	amdgpu_device_smu_fini_early(adev);
2910 
2911 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2912 		if (!adev->ip_blocks[i].status.hw)
2913 			continue;
2914 
2915 		amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
2916 	}
2917 
2918 	if (amdgpu_sriov_vf(adev)) {
2919 		if (amdgpu_virt_release_full_gpu(adev, false))
2920 			dev_err(adev->dev,
2921 				"failed to release exclusive mode on fini\n");
2922 	}
2923 
2924 	/*
2925 	 * Driver reload on the APU can fail due to firmware validation because
2926 	 * the PSP is always running, as it is shared across the whole SoC.
2927 	 * This same issue does not occur on dGPU because it has a mechanism
2928 	 * that checks whether the PSP is running. A solution for those issues
2929 	 * in the APU is to trigger a GPU reset, but this should be done during
2930 	 * the unload phase to avoid adding boot latency and screen flicker.
2931 	 * GFX V11 has GC block as default off IP. Every time AMDGPU driver sends
2932 	 * a request to PMFW to unload MP1, PMFW will put GC in reset and power down
2933 	 * the voltage. Hence, skipping reset for APUs with GFX V11 or later.
2934 	 */
2935 	if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu &&
2936 		amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 0, 0)) {
2937 		r = amdgpu_asic_reset(adev);
2938 		if (r)
2939 			dev_err(adev->dev, "asic reset on %s failed\n", __func__);
2940 	}
2941 
2942 	return 0;
2943 }
2944 
2945 /**
2946  * amdgpu_device_ip_fini - run fini for hardware IPs
2947  *
2948  * @adev: amdgpu_device pointer
2949  *
2950  * Main teardown pass for hardware IPs.  The list of all the hardware
2951  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2952  * are run.  hw_fini tears down the hardware associated with each IP
2953  * and sw_fini tears down any software state associated with each IP.
2954  * Returns 0 on success, negative error code on failure.
2955  */
2956 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2957 {
2958 	int i, r;
2959 
2960 	amdgpu_cper_fini(adev);
2961 
2962 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2963 		amdgpu_virt_release_ras_err_handler_data(adev);
2964 
2965 	if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported)
2966 		amdgpu_xgmi_remove_device(adev);
2967 
2968 	amdgpu_amdkfd_device_fini_sw(adev);
2969 
2970 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2971 		if (!adev->ip_blocks[i].status.sw)
2972 			continue;
2973 
2974 		if (!adev->ip_blocks[i].version)
2975 			continue;
2976 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2977 			amdgpu_ucode_free_bo(adev);
2978 			amdgpu_free_static_csa(&adev->virt.csa_obj);
2979 			amdgpu_wb_fini(adev);
2980 			amdgpu_device_mem_scratch_fini(adev);
2981 			amdgpu_ib_pool_fini(adev);
2982 			amdgpu_seq64_fini(adev);
2983 			amdgpu_doorbell_fini(adev);
2984 		}
2985 		if (adev->ip_blocks[i].version->funcs->sw_fini) {
2986 			r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);
2987 			/* XXX handle errors */
2988 			if (r) {
2989 				dev_dbg(adev->dev,
2990 					"sw_fini of IP block <%s> failed %d\n",
2991 					adev->ip_blocks[i].version->funcs->name,
2992 					r);
2993 			}
2994 		}
2995 		adev->ip_blocks[i].status.sw = false;
2996 		adev->ip_blocks[i].status.valid = false;
2997 	}
2998 
2999 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3000 		if (!adev->ip_blocks[i].status.late_initialized)
3001 			continue;
3002 		if (!adev->ip_blocks[i].version)
3003 			continue;
3004 		if (adev->ip_blocks[i].version->funcs->late_fini)
3005 			adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);
3006 		adev->ip_blocks[i].status.late_initialized = false;
3007 	}
3008 
3009 	amdgpu_ras_fini(adev);
3010 	amdgpu_uid_fini(adev);
3011 
3012 	return 0;
3013 }
3014 
3015 /**
3016  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3017  *
3018  * @work: work_struct.
3019  */
3020 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3021 {
3022 	struct amdgpu_device *adev =
3023 		container_of(work, struct amdgpu_device, delayed_init_work.work);
3024 	int r;
3025 
3026 	r = amdgpu_ib_ring_tests(adev);
3027 	if (r)
3028 		dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3029 }
3030 
3031 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3032 {
3033 	struct amdgpu_device *adev =
3034 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3035 
3036 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3037 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3038 
3039 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
3040 		adev->gfx.gfx_off_state = true;
3041 }
3042 
3043 /**
3044  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3045  *
3046  * @adev: amdgpu_device pointer
3047  *
3048  * Main suspend function for hardware IPs.  The list of all the hardware
3049  * IPs that make up the asic is walked, clockgating is disabled and the
3050  * suspend callbacks are run.  suspend puts the hardware and software state
3051  * in each IP into a state suitable for suspend.
3052  * Returns 0 on success, negative error code on failure.
3053  */
3054 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3055 {
3056 	int i, r, rec;
3057 
3058 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3059 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3060 
3061 	/*
3062 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
3063 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3064 	 * scenario. Add the missing df cstate disablement here.
3065 	 */
3066 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3067 		dev_warn(adev->dev, "Failed to disallow df cstate");
3068 
3069 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3070 		if (!adev->ip_blocks[i].status.valid)
3071 			continue;
3072 
3073 		/* displays are handled separately */
3074 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3075 			continue;
3076 
3077 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3078 		if (r)
3079 			goto unwind;
3080 	}
3081 
3082 	return 0;
3083 unwind:
3084 	rec = amdgpu_device_ip_resume_phase3(adev);
3085 	if (rec)
3086 		dev_err(adev->dev,
3087 			"amdgpu_device_ip_resume_phase3 failed during unwind: %d\n",
3088 			rec);
3089 
3090 	amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW);
3091 
3092 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3093 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3094 
3095 	return r;
3096 }
3097 
3098 /**
3099  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3100  *
3101  * @adev: amdgpu_device pointer
3102  *
3103  * Main suspend function for hardware IPs.  The list of all the hardware
3104  * IPs that make up the asic is walked, clockgating is disabled and the
3105  * suspend callbacks are run.  suspend puts the hardware and software state
3106  * in each IP into a state suitable for suspend.
3107  * Returns 0 on success, negative error code on failure.
3108  */
3109 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3110 {
3111 	int i, r, rec;
3112 
3113 	if (adev->in_s0ix)
3114 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3115 
3116 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3117 		if (!adev->ip_blocks[i].status.valid || !adev->ip_blocks[i].status.hw)
3118 			continue;
3119 		/* displays are handled in phase1 */
3120 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3121 			continue;
3122 		/* PSP lost connection when err_event_athub occurs */
3123 		if (amdgpu_ras_intr_triggered() &&
3124 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3125 			adev->ip_blocks[i].status.hw = false;
3126 			continue;
3127 		}
3128 
3129 		/* skip unnecessary suspend if we do not initialize them yet */
3130 		if (!amdgpu_ip_member_of_hwini(
3131 			    adev, adev->ip_blocks[i].version->type))
3132 			continue;
3133 
3134 		/* Since we skip suspend for S0i3, we need to cancel the delayed
3135 		 * idle work here as the suspend callback never gets called.
3136 		 */
3137 		if (adev->in_s0ix &&
3138 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX &&
3139 		    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))
3140 			cancel_delayed_work_sync(&adev->gfx.idle_work);
3141 		/* skip suspend of gfx/mes and psp for S0ix
3142 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3143 		 * like at runtime. PSP is also part of the always on hardware
3144 		 * so no need to suspend it.
3145 		 */
3146 		if (adev->in_s0ix &&
3147 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3148 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3149 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3150 			continue;
3151 
3152 		/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3153 		if (adev->in_s0ix &&
3154 		    (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3155 		     IP_VERSION(5, 0, 0)) &&
3156 		    (adev->ip_blocks[i].version->type ==
3157 		     AMD_IP_BLOCK_TYPE_SDMA))
3158 			continue;
3159 
3160 		/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3161 		 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3162 		 * from this location and RLC Autoload automatically also gets loaded
3163 		 * from here based on PMFW -> PSP message during re-init sequence.
3164 		 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3165 		 * the TMR and reload FWs again for IMU enabled APU ASICs.
3166 		 */
3167 		if (amdgpu_in_reset(adev) &&
3168 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3169 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3170 			continue;
3171 
3172 		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3173 		if (r)
3174 			goto unwind;
3175 
3176 		/* handle putting the SMC in the appropriate state */
3177 		if (!amdgpu_sriov_vf(adev)) {
3178 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3179 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3180 				if (r) {
3181 					dev_err(adev->dev,
3182 						"SMC failed to set mp1 state %d, %d\n",
3183 						adev->mp1_state, r);
3184 					goto unwind;
3185 				}
3186 			}
3187 		}
3188 	}
3189 
3190 	return 0;
3191 unwind:
3192 	/* suspend phase 2 = resume phase 1 + resume phase 2 */
3193 	rec = amdgpu_device_ip_resume_phase1(adev);
3194 	if (rec) {
3195 		dev_err(adev->dev,
3196 			"amdgpu_device_ip_resume_phase1 failed during unwind: %d\n",
3197 			rec);
3198 		return r;
3199 	}
3200 
3201 	rec = amdgpu_device_fw_loading(adev);
3202 	if (rec) {
3203 		dev_err(adev->dev,
3204 			"amdgpu_device_fw_loading failed during unwind: %d\n",
3205 			rec);
3206 		return r;
3207 	}
3208 
3209 	rec = amdgpu_device_ip_resume_phase2(adev);
3210 	if (rec) {
3211 		dev_err(adev->dev,
3212 			"amdgpu_device_ip_resume_phase2 failed during unwind: %d\n",
3213 			rec);
3214 		return r;
3215 	}
3216 
3217 	return r;
3218 }
3219 
3220 /**
3221  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3222  *
3223  * @adev: amdgpu_device pointer
3224  *
3225  * Main suspend function for hardware IPs.  The list of all the hardware
3226  * IPs that make up the asic is walked, clockgating is disabled and the
3227  * suspend callbacks are run.  suspend puts the hardware and software state
3228  * in each IP into a state suitable for suspend.
3229  * Returns 0 on success, negative error code on failure.
3230  */
3231 static int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3232 {
3233 	int r;
3234 
3235 	if (amdgpu_sriov_vf(adev)) {
3236 		amdgpu_virt_fini_data_exchange(adev);
3237 		amdgpu_virt_request_full_gpu(adev, false);
3238 	}
3239 
3240 	r = amdgpu_device_ip_suspend_phase1(adev);
3241 	if (r)
3242 		return r;
3243 	r = amdgpu_device_ip_suspend_phase2(adev);
3244 
3245 	if (amdgpu_sriov_vf(adev))
3246 		amdgpu_virt_release_full_gpu(adev, false);
3247 
3248 	return r;
3249 }
3250 
3251 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3252 {
3253 	int i, r;
3254 
3255 	static enum amd_ip_block_type ip_order[] = {
3256 		AMD_IP_BLOCK_TYPE_COMMON,
3257 		AMD_IP_BLOCK_TYPE_GMC,
3258 		AMD_IP_BLOCK_TYPE_PSP,
3259 		AMD_IP_BLOCK_TYPE_IH,
3260 	};
3261 
3262 	for (i = 0; i < adev->num_ip_blocks; i++) {
3263 		int j;
3264 		struct amdgpu_ip_block *block;
3265 
3266 		block = &adev->ip_blocks[i];
3267 		block->status.hw = false;
3268 
3269 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3270 
3271 			if (block->version->type != ip_order[j] ||
3272 				!block->status.valid)
3273 				continue;
3274 
3275 			r = block->version->funcs->hw_init(&adev->ip_blocks[i]);
3276 			if (r) {
3277 				dev_err(adev->dev, "RE-INIT-early: %s failed\n",
3278 					 block->version->funcs->name);
3279 				return r;
3280 			}
3281 			block->status.hw = true;
3282 		}
3283 	}
3284 
3285 	return 0;
3286 }
3287 
3288 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3289 {
3290 	struct amdgpu_ip_block *block;
3291 	int i, r = 0;
3292 
3293 	static enum amd_ip_block_type ip_order[] = {
3294 		AMD_IP_BLOCK_TYPE_SMC,
3295 		AMD_IP_BLOCK_TYPE_DCE,
3296 		AMD_IP_BLOCK_TYPE_GFX,
3297 		AMD_IP_BLOCK_TYPE_SDMA,
3298 		AMD_IP_BLOCK_TYPE_MES,
3299 		AMD_IP_BLOCK_TYPE_UVD,
3300 		AMD_IP_BLOCK_TYPE_VCE,
3301 		AMD_IP_BLOCK_TYPE_VCN,
3302 		AMD_IP_BLOCK_TYPE_JPEG
3303 	};
3304 
3305 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3306 		block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);
3307 
3308 		if (!block)
3309 			continue;
3310 
3311 		if (block->status.valid && !block->status.hw) {
3312 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) {
3313 				r = amdgpu_ip_block_resume(block);
3314 			} else {
3315 				r = block->version->funcs->hw_init(block);
3316 			}
3317 
3318 			if (r) {
3319 				dev_err(adev->dev, "RE-INIT-late: %s failed\n",
3320 					 block->version->funcs->name);
3321 				break;
3322 			}
3323 			block->status.hw = true;
3324 		}
3325 	}
3326 
3327 	return r;
3328 }
3329 
3330 /**
3331  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3332  *
3333  * @adev: amdgpu_device pointer
3334  *
3335  * First resume function for hardware IPs.  The list of all the hardware
3336  * IPs that make up the asic is walked and the resume callbacks are run for
3337  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3338  * after a suspend and updates the software state as necessary.  This
3339  * function is also used for restoring the GPU after a GPU reset.
3340  * Returns 0 on success, negative error code on failure.
3341  */
3342 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3343 {
3344 	int i, r;
3345 
3346 	for (i = 0; i < adev->num_ip_blocks; i++) {
3347 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3348 			continue;
3349 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3350 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3351 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3352 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3353 
3354 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3355 			if (r)
3356 				return r;
3357 		}
3358 	}
3359 
3360 	return 0;
3361 }
3362 
3363 /**
3364  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3365  *
3366  * @adev: amdgpu_device pointer
3367  *
3368  * Second resume function for hardware IPs.  The list of all the hardware
3369  * IPs that make up the asic is walked and the resume callbacks are run for
3370  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3371  * functional state after a suspend and updates the software state as
3372  * necessary.  This function is also used for restoring the GPU after a GPU
3373  * reset.
3374  * Returns 0 on success, negative error code on failure.
3375  */
3376 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3377 {
3378 	int i, r;
3379 
3380 	for (i = 0; i < adev->num_ip_blocks; i++) {
3381 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3382 			continue;
3383 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3384 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3385 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3386 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||
3387 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3388 			continue;
3389 		r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3390 		if (r)
3391 			return r;
3392 	}
3393 
3394 	return 0;
3395 }
3396 
3397 /**
3398  * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3399  *
3400  * @adev: amdgpu_device pointer
3401  *
3402  * Third resume function for hardware IPs.  The list of all the hardware
3403  * IPs that make up the asic is walked and the resume callbacks are run for
3404  * all DCE.  resume puts the hardware into a functional state after a suspend
3405  * and updates the software state as necessary.  This function is also used
3406  * for restoring the GPU after a GPU reset.
3407  *
3408  * Returns 0 on success, negative error code on failure.
3409  */
3410 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)
3411 {
3412 	int i, r;
3413 
3414 	for (i = 0; i < adev->num_ip_blocks; i++) {
3415 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3416 			continue;
3417 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
3418 			r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3419 			if (r)
3420 				return r;
3421 		}
3422 	}
3423 
3424 	return 0;
3425 }
3426 
3427 /**
3428  * amdgpu_device_ip_resume - run resume for hardware IPs
3429  *
3430  * @adev: amdgpu_device pointer
3431  *
3432  * Main resume function for hardware IPs.  The hardware IPs
3433  * are split into two resume functions because they are
3434  * also used in recovering from a GPU reset and some additional
3435  * steps need to be take between them.  In this case (S3/S4) they are
3436  * run sequentially.
3437  * Returns 0 on success, negative error code on failure.
3438  */
3439 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3440 {
3441 	int r;
3442 
3443 	r = amdgpu_device_ip_resume_phase1(adev);
3444 	if (r)
3445 		return r;
3446 
3447 	r = amdgpu_device_fw_loading(adev);
3448 	if (r)
3449 		return r;
3450 
3451 	r = amdgpu_device_ip_resume_phase2(adev);
3452 
3453 	amdgpu_ttm_enable_buffer_funcs(adev);
3454 
3455 	if (r)
3456 		return r;
3457 
3458 	amdgpu_fence_driver_hw_init(adev);
3459 
3460 	r = amdgpu_device_ip_resume_phase3(adev);
3461 
3462 	return r;
3463 }
3464 
3465 /**
3466  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3467  *
3468  * @adev: amdgpu_device pointer
3469  *
3470  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3471  */
3472 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3473 {
3474 	if (amdgpu_sriov_vf(adev)) {
3475 		if (adev->is_atom_fw) {
3476 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3477 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3478 		} else {
3479 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3480 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3481 		}
3482 
3483 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3484 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3485 	}
3486 }
3487 
3488 /**
3489  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3490  *
3491  * @pdev : pci device context
3492  * @asic_type: AMD asic type
3493  *
3494  * Check if there is DC (new modesetting infrastructre) support for an asic.
3495  * returns true if DC has support, false if not.
3496  */
3497 bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev,
3498 				       enum amd_asic_type asic_type)
3499 {
3500 	switch (asic_type) {
3501 #ifdef CONFIG_DRM_AMDGPU_SI
3502 	case CHIP_HAINAN:
3503 #endif
3504 	case CHIP_TOPAZ:
3505 		/* chips with no display hardware */
3506 		return false;
3507 #if defined(CONFIG_DRM_AMD_DC)
3508 	case CHIP_TAHITI:
3509 	case CHIP_PITCAIRN:
3510 	case CHIP_VERDE:
3511 	case CHIP_OLAND:
3512 		return amdgpu_dc != 0 && IS_ENABLED(CONFIG_DRM_AMD_DC_SI);
3513 	default:
3514 		return amdgpu_dc != 0;
3515 #else
3516 	default:
3517 		if (amdgpu_dc > 0)
3518 			dev_info_once(
3519 				&pdev->dev,
3520 				"Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3521 		return false;
3522 #endif
3523 	}
3524 }
3525 
3526 /**
3527  * amdgpu_device_has_dc_support - check if dc is supported
3528  *
3529  * @adev: amdgpu_device pointer
3530  *
3531  * Returns true for supported, false for not supported
3532  */
3533 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3534 {
3535 	if (adev->enable_virtual_display ||
3536 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3537 		return false;
3538 
3539 	return amdgpu_device_asic_has_dc_support(adev->pdev, adev->asic_type);
3540 }
3541 
3542 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3543 {
3544 	struct amdgpu_device *adev =
3545 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
3546 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3547 
3548 	/* It's a bug to not have a hive within this function */
3549 	if (WARN_ON(!hive))
3550 		return;
3551 
3552 	/*
3553 	 * Use task barrier to synchronize all xgmi reset works across the
3554 	 * hive. task_barrier_enter and task_barrier_exit will block
3555 	 * until all the threads running the xgmi reset works reach
3556 	 * those points. task_barrier_full will do both blocks.
3557 	 */
3558 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3559 
3560 		task_barrier_enter(&hive->tb);
3561 		adev->asic_reset_res = amdgpu_device_baco_enter(adev);
3562 
3563 		if (adev->asic_reset_res)
3564 			goto fail;
3565 
3566 		task_barrier_exit(&hive->tb);
3567 		adev->asic_reset_res = amdgpu_device_baco_exit(adev);
3568 
3569 		if (adev->asic_reset_res)
3570 			goto fail;
3571 
3572 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3573 	} else {
3574 
3575 		task_barrier_full(&hive->tb);
3576 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
3577 	}
3578 
3579 fail:
3580 	if (adev->asic_reset_res)
3581 		dev_warn(adev->dev,
3582 			 "ASIC reset failed with error, %d for drm dev, %s",
3583 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
3584 	amdgpu_put_xgmi_hive(hive);
3585 }
3586 
3587 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3588 {
3589 	char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH];
3590 	char *input = buf;
3591 	char *timeout_setting = NULL;
3592 	int index = 0;
3593 	long timeout;
3594 	int ret = 0;
3595 
3596 	/* By default timeout for all queues is 2 sec */
3597 	adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout =
3598 		adev->video_timeout = msecs_to_jiffies(2000);
3599 
3600 	if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH))
3601 		return 0;
3602 
3603 	/*
3604 	 * strsep() destructively modifies its input by replacing delimiters
3605 	 * with '\0'. Use a stack copy so the global module parameter buffer
3606 	 * remains intact for multi-GPU systems where this function is called
3607 	 * once per device.
3608 	 */
3609 	strscpy(buf, amdgpu_lockup_timeout, sizeof(buf));
3610 
3611 	while ((timeout_setting = strsep(&input, ",")) &&
3612 	       strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3613 		ret = kstrtol(timeout_setting, 0, &timeout);
3614 		if (ret)
3615 			return ret;
3616 
3617 		if (timeout == 0) {
3618 			index++;
3619 			continue;
3620 		} else if (timeout < 0) {
3621 			timeout = MAX_SCHEDULE_TIMEOUT;
3622 			dev_warn(adev->dev, "lockup timeout disabled");
3623 			add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3624 		} else {
3625 			timeout = msecs_to_jiffies(timeout);
3626 		}
3627 
3628 		switch (index++) {
3629 		case 0:
3630 			adev->gfx_timeout = timeout;
3631 			break;
3632 		case 1:
3633 			adev->compute_timeout = timeout;
3634 			break;
3635 		case 2:
3636 			adev->sdma_timeout = timeout;
3637 			break;
3638 		case 3:
3639 			adev->video_timeout = timeout;
3640 			break;
3641 		default:
3642 			break;
3643 		}
3644 	}
3645 
3646 	/* When only one value specified apply it to all queues. */
3647 	if (index == 1)
3648 		adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout =
3649 			adev->video_timeout = timeout;
3650 
3651 	return ret;
3652 }
3653 
3654 /**
3655  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3656  *
3657  * @adev: amdgpu_device pointer
3658  *
3659  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3660  */
3661 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3662 {
3663 	struct iommu_domain *domain;
3664 
3665 	domain = iommu_get_domain_for_dev(adev->dev);
3666 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3667 		adev->ram_is_direct_mapped = true;
3668 }
3669 
3670 #if defined(CONFIG_HSA_AMD_P2P)
3671 /**
3672  * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
3673  *
3674  * @adev: amdgpu_device pointer
3675  *
3676  * return if IOMMU remapping bar address
3677  */
3678 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)
3679 {
3680 	struct iommu_domain *domain;
3681 
3682 	domain = iommu_get_domain_for_dev(adev->dev);
3683 	if (domain && (domain->type == IOMMU_DOMAIN_DMA ||
3684 		domain->type ==	IOMMU_DOMAIN_DMA_FQ))
3685 		return true;
3686 
3687 	return false;
3688 }
3689 #endif
3690 
3691 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3692 {
3693 	if (amdgpu_mcbp == 1)
3694 		adev->gfx.mcbp = true;
3695 	else if (amdgpu_mcbp == 0)
3696 		adev->gfx.mcbp = false;
3697 
3698 	if (amdgpu_sriov_vf(adev))
3699 		adev->gfx.mcbp = true;
3700 
3701 	if (adev->gfx.mcbp)
3702 		dev_info(adev->dev, "MCBP is enabled\n");
3703 }
3704 
3705 static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev)
3706 {
3707 	int r;
3708 
3709 	r = amdgpu_atombios_sysfs_init(adev);
3710 	if (r)
3711 		drm_err(&adev->ddev,
3712 			"registering atombios sysfs failed (%d).\n", r);
3713 
3714 	r = amdgpu_pm_sysfs_init(adev);
3715 	if (r)
3716 		dev_err(adev->dev, "registering pm sysfs failed (%d).\n", r);
3717 
3718 	r = amdgpu_ucode_sysfs_init(adev);
3719 	if (r) {
3720 		adev->ucode_sysfs_en = false;
3721 		dev_err(adev->dev, "Creating firmware sysfs failed (%d).\n", r);
3722 	} else
3723 		adev->ucode_sysfs_en = true;
3724 
3725 	r = amdgpu_device_attr_sysfs_init(adev);
3726 	if (r)
3727 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
3728 
3729 	r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
3730 	if (r)
3731 		dev_err(adev->dev,
3732 			"Could not create amdgpu board attributes\n");
3733 
3734 	amdgpu_fru_sysfs_init(adev);
3735 	amdgpu_reg_state_sysfs_init(adev);
3736 	amdgpu_xcp_sysfs_init(adev);
3737 	amdgpu_uma_sysfs_init(adev);
3738 	amdgpu_ptl_sysfs_init(adev);
3739 	amdgpu_ualink_sysfs_init(adev);
3740 
3741 	return r;
3742 }
3743 
3744 static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)
3745 {
3746 	if (adev->pm.sysfs_initialized)
3747 		amdgpu_pm_sysfs_fini(adev);
3748 	if (adev->ucode_sysfs_en)
3749 		amdgpu_ucode_sysfs_fini(adev);
3750 
3751 	amdgpu_discovery_sysfs_fini(adev);
3752 	amdgpu_preempt_mgr_sysfs_fini(adev);
3753 
3754 	amdgpu_device_attr_sysfs_fini(adev);
3755 	amdgpu_fru_sysfs_fini(adev);
3756 
3757 	amdgpu_reg_state_sysfs_fini(adev);
3758 	amdgpu_xcp_sysfs_fini(adev);
3759 	amdgpu_uma_sysfs_fini(adev);
3760 	amdgpu_ptl_sysfs_fini(adev);
3761 	amdgpu_ualink_sysfs_fini(adev);
3762 }
3763 
3764 static bool
3765 amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
3766 {
3767 	return !pci_is_thunderbolt_attached(adev->pdev) &&
3768 	       (px || (!dev_is_removable(&adev->pdev->dev) &&
3769 		       apple_gmux_detect(NULL, NULL)));
3770 }
3771 
3772 /**
3773  * amdgpu_device_init - initialize the driver
3774  *
3775  * @adev: amdgpu_device pointer
3776  * @flags: driver flags
3777  *
3778  * Initializes the driver info and hw (all asics).
3779  * Returns 0 for success or an error on failure.
3780  * Called at driver startup.
3781  */
3782 int amdgpu_device_init(struct amdgpu_device *adev,
3783 		       uint32_t flags)
3784 {
3785 	struct pci_dev *pdev = adev->pdev;
3786 	int r, i;
3787 	bool px = false;
3788 	u32 max_MBps;
3789 	int tmp;
3790 
3791 	adev->shutdown = false;
3792 	adev->flags = flags;
3793 	amdgpu_device_init_pcie_links(adev);
3794 
3795 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3796 		adev->asic_type = amdgpu_force_asic_type;
3797 	else
3798 		adev->asic_type = flags & AMD_ASIC_MASK;
3799 
3800 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3801 	if (amdgpu_emu_mode == 1)
3802 		adev->usec_timeout *= 10;
3803 	adev->gmc.gart_size = 512 * 1024 * 1024;
3804 	adev->accel_working = false;
3805 	adev->num_rings = 0;
3806 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3807 	adev->mman.buffer_funcs = NULL;
3808 	adev->mman.num_buffer_funcs_scheds = 0;
3809 	adev->vm_manager.vm_pte_funcs = NULL;
3810 	adev->vm_manager.vm_pte_num_scheds = 0;
3811 	adev->gmc.gmc_funcs = NULL;
3812 	adev->harvest_ip_mask = 0x0;
3813 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3814 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3815 
3816 	amdgpu_reg_access_init(adev);
3817 
3818 	dev_info(
3819 		adev->dev,
3820 		"initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3821 		amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3822 		pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3823 
3824 	/* mutex initialization are all done here so we
3825 	 * can recall function without having locking issues
3826 	 */
3827 	mutex_init(&adev->firmware.mutex);
3828 	mutex_init(&adev->pm.mutex);
3829 	mutex_init(&adev->gfx.gpu_clock_mutex);
3830 	mutex_init(&adev->srbm_mutex);
3831 	mutex_init(&adev->gfx.pipe_reserve_mutex);
3832 	mutex_init(&adev->gfx.gfx_off_mutex);
3833 	mutex_init(&adev->gfx.partition_mutex);
3834 	mutex_init(&adev->grbm_idx_mutex);
3835 	mutex_init(&adev->mn_lock);
3836 	mutex_init(&adev->virt.vf_errors.lock);
3837 	hash_init(adev->mn_hash);
3838 	mutex_init(&adev->psp.mutex);
3839 	mutex_init(&adev->psp.ptl.mutex);
3840 	mutex_init(&adev->notifier_lock);
3841 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3842 	mutex_init(&adev->benchmark_mutex);
3843 	mutex_init(&adev->gfx.reset_sem_mutex);
3844 
3845 	/* Associate locks with lockdep classes for ordering validation */
3846 	amdgpu_lockdep_set_class(adev);
3847 	/* Initialize the mutex for cleaner shader isolation between GFX and compute processes */
3848 	mutex_init(&adev->enforce_isolation_mutex);
3849 	for (i = 0; i < MAX_XCP; ++i) {
3850 		adev->isolation[i].spearhead = dma_fence_get_stub();
3851 		amdgpu_sync_create(&adev->isolation[i].active);
3852 		amdgpu_sync_create(&adev->isolation[i].prev);
3853 	}
3854 	mutex_init(&adev->gfx.userq_sch_mutex);
3855 	mutex_init(&adev->gfx.workload_profile_mutex);
3856 	mutex_init(&adev->vcn.workload_profile_mutex);
3857 
3858 	spin_lock_init(&adev->irq.lock);
3859 
3860 	amdgpu_ip_map_init(adev);
3861 	amdgpu_early_init_rlc_reg_funcs(adev);
3862 	amdgpu_device_init_apu_flags(adev);
3863 
3864 	r = amdgpu_device_check_arguments(adev);
3865 	if (r)
3866 		return r;
3867 
3868 	spin_lock_init(&adev->mmio_idx_lock);
3869 	spin_lock_init(&adev->mm_stats.lock);
3870 	spin_lock_init(&adev->virt.rlcg_reg_lock);
3871 	spin_lock_init(&adev->wb.lock);
3872 	mutex_init(&adev->lsdma.lock);
3873 
3874 	INIT_LIST_HEAD(&adev->reset_list);
3875 
3876 	INIT_LIST_HEAD(&adev->ras_list);
3877 
3878 	INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3879 
3880 	xa_init_flags(&adev->userq_doorbell_xa, XA_FLAGS_LOCK_IRQ);
3881 
3882 	INIT_DELAYED_WORK(&adev->delayed_init_work,
3883 			  amdgpu_device_delayed_init_work_handler);
3884 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3885 			  amdgpu_device_delay_enable_gfx_off);
3886 	/*
3887 	 * Initialize the enforce_isolation work structures for each XCP
3888 	 * partition.  This work handler is responsible for enforcing shader
3889 	 * isolation on AMD GPUs.  It counts the number of emitted fences for
3890 	 * each GFX and compute ring.  If there are any fences, it schedules
3891 	 * the `enforce_isolation_work` to be run after a delay.  If there are
3892 	 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the
3893 	 * runqueue.
3894 	 */
3895 	for (i = 0; i < MAX_XCP; i++) {
3896 		INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,
3897 				  amdgpu_gfx_enforce_isolation_handler);
3898 		adev->gfx.enforce_isolation[i].adev = adev;
3899 		adev->gfx.enforce_isolation[i].xcp_id = i;
3900 	}
3901 
3902 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3903 
3904 	amdgpu_coredump_init(adev);
3905 
3906 	adev->gfx.gfx_off_req_count = 1;
3907 	adev->gfx.gfx_off_residency = 0;
3908 	adev->gfx.gfx_off_entrycount = 0;
3909 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3910 
3911 	atomic_set(&adev->throttling_logging_enabled, 1);
3912 	/*
3913 	 * If throttling continues, logging will be performed every minute
3914 	 * to avoid log flooding. "-1" is subtracted since the thermal
3915 	 * throttling interrupt comes every second. Thus, the total logging
3916 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3917 	 * for throttling interrupt) = 60 seconds.
3918 	 */
3919 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3920 
3921 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3922 
3923 	/* Registers mapping */
3924 	/* TODO: block userspace mapping of io register */
3925 	if (adev->asic_type >= CHIP_BONAIRE) {
3926 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3927 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3928 	} else {
3929 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3930 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3931 	}
3932 
3933 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3934 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3935 
3936 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3937 	if (!adev->rmmio)
3938 		return -ENOMEM;
3939 
3940 	dev_info(adev->dev, "register mmio base: 0x%08X\n",
3941 		 (uint32_t)adev->rmmio_base);
3942 	dev_info(adev->dev, "register mmio size: %u\n",
3943 		 (unsigned int)adev->rmmio_size);
3944 
3945 	/*
3946 	 * Reset domain needs to be present early, before XGMI hive discovered
3947 	 * (if any) and initialized to use reset sem and in_gpu reset flag
3948 	 * early on during init and before calling to RREG32.
3949 	 */
3950 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3951 	if (!adev->reset_domain)
3952 		return -ENOMEM;
3953 
3954 	/* detect hw virtualization here */
3955 	amdgpu_virt_init(adev);
3956 
3957 	amdgpu_device_get_pcie_info(adev);
3958 
3959 	r = amdgpu_device_get_job_timeout_settings(adev);
3960 	if (r) {
3961 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3962 		return r;
3963 	}
3964 
3965 	amdgpu_device_set_mcbp(adev);
3966 
3967 	/*
3968 	 * By default, use default mode where all blocks are expected to be
3969 	 * initialized. At present a 'swinit' of blocks is required to be
3970 	 * completed before the need for a different level is detected.
3971 	 */
3972 	amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
3973 
3974 	amdgpu_device_check_iommu_direct_map(adev);
3975 
3976 	/* early init functions */
3977 	r = amdgpu_device_ip_early_init(adev);
3978 	if (r)
3979 		return r;
3980 
3981 	/*
3982 	 * No need to remove conflicting FBs for non-display class devices.
3983 	 * This prevents the sysfb from being freed accidently.
3984 	 */
3985 	if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA ||
3986 	    (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) {
3987 		/* Get rid of things like offb */
3988 		r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);
3989 		if (r)
3990 			return r;
3991 	}
3992 
3993 	/* Enable TMZ based on IP_VERSION */
3994 	amdgpu_gmc_tmz_set(adev);
3995 
3996 	if (amdgpu_sriov_vf(adev) &&
3997 	    amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
3998 		/* VF MMIO access (except mailbox range) from CPU
3999 		 * will be blocked during sriov runtime
4000 		 */
4001 		adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
4002 
4003 	amdgpu_gmc_noretry_set(adev);
4004 	/* Need to get xgmi info early to decide the reset behavior*/
4005 	if (adev->gmc.xgmi.supported) {
4006 		if (adev->gfxhub.funcs &&
4007 		    adev->gfxhub.funcs->get_xgmi_info) {
4008 			r = adev->gfxhub.funcs->get_xgmi_info(adev);
4009 			if (r)
4010 				return r;
4011 		}
4012 	}
4013 
4014 	if (adev->gmc.xgmi.connected_to_cpu) {
4015 		if (adev->mmhub.funcs &&
4016 		    adev->mmhub.funcs->get_xgmi_info) {
4017 			r = adev->mmhub.funcs->get_xgmi_info(adev);
4018 			if (r)
4019 				return r;
4020 		}
4021 	}
4022 
4023 	/* enable PCIE atomic ops */
4024 	if (!amdgpu_sriov_vf(adev)) {
4025 		/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4026 		 * internal path natively support atomics, set have_atomics_support to true.
4027 		 */
4028 		if ((adev->flags & AMD_IS_APU &&
4029 		     amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) ||
4030 		    (adev->gmc.xgmi.connected_to_cpu &&
4031 		     amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) {
4032 			adev->have_atomics_support = true;
4033 		} else {
4034 			adev->have_atomics_support =
4035 				!pci_enable_atomic_ops_to_root(adev->pdev,
4036 						  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4037 						  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4038 		}
4039 	}
4040 
4041 	if (!adev->have_atomics_support)
4042 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4043 
4044 	/* doorbell bar mapping and doorbell index init*/
4045 	amdgpu_doorbell_init(adev);
4046 
4047 	if (amdgpu_emu_mode == 1) {
4048 		/* post the asic on emulation mode */
4049 		emu_soc_asic_init(adev);
4050 		goto fence_driver_init;
4051 	}
4052 
4053 	amdgpu_reset_init(adev);
4054 
4055 	/* detect if we are with an SRIOV vbios */
4056 	if (adev->bios)
4057 		amdgpu_device_detect_sriov_bios(adev);
4058 
4059 	/* check if we need to reset the asic
4060 	 *  E.g., driver was not cleanly unloaded previously, etc.
4061 	 */
4062 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4063 		if (adev->gmc.xgmi.num_physical_nodes) {
4064 			dev_info(adev->dev, "Pending hive reset.\n");
4065 			amdgpu_set_init_level(adev,
4066 					      AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
4067 		} else {
4068 				tmp = amdgpu_reset_method;
4069 				/* It should do a default reset when loading or reloading the driver,
4070 				 * regardless of the module parameter reset_method.
4071 				 */
4072 				amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4073 				r = amdgpu_asic_reset(adev);
4074 				amdgpu_reset_method = tmp;
4075 		}
4076 
4077 		if (r) {
4078 		  dev_err(adev->dev, "asic reset on init failed\n");
4079 		  goto failed;
4080 		}
4081 	}
4082 
4083 	/* Post card if necessary */
4084 	if (amdgpu_device_need_post(adev)) {
4085 		if (!adev->bios) {
4086 			dev_err(adev->dev, "no vBIOS found\n");
4087 			r = -EINVAL;
4088 			goto failed;
4089 		}
4090 		dev_info(adev->dev, "GPU posting now...\n");
4091 		r = amdgpu_device_asic_init(adev);
4092 		if (r) {
4093 			dev_err(adev->dev, "gpu post error!\n");
4094 			goto failed;
4095 		}
4096 	}
4097 
4098 	if (adev->bios) {
4099 		if (adev->is_atom_fw) {
4100 			/* Initialize clocks */
4101 			r = amdgpu_atomfirmware_get_clock_info(adev);
4102 			if (r) {
4103 				dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4104 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4105 				goto failed;
4106 			}
4107 		} else {
4108 			/* Initialize clocks */
4109 			r = amdgpu_atombios_get_clock_info(adev);
4110 			if (r) {
4111 				dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4112 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4113 				goto failed;
4114 			}
4115 			/* init i2c buses */
4116 			amdgpu_i2c_init(adev);
4117 		}
4118 	}
4119 
4120 fence_driver_init:
4121 	/* Fence driver */
4122 	r = amdgpu_fence_driver_sw_init(adev);
4123 	if (r) {
4124 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4125 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4126 		goto failed;
4127 	}
4128 
4129 	/* init the mode config */
4130 	drm_mode_config_init(adev_to_drm(adev));
4131 
4132 	r = amdgpu_device_ip_init(adev);
4133 	if (r) {
4134 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4135 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4136 		goto release_ras_con;
4137 	}
4138 
4139 	amdgpu_fence_driver_hw_init(adev);
4140 
4141 	dev_info(adev->dev,
4142 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4143 			adev->gfx.config.max_shader_engines,
4144 			adev->gfx.config.max_sh_per_se,
4145 			adev->gfx.config.max_cu_per_sh,
4146 			adev->gfx.cu_info.number);
4147 
4148 	adev->accel_working = true;
4149 
4150 	amdgpu_vm_check_compute_bug(adev);
4151 
4152 	/* Initialize the buffer migration limit. */
4153 	if (amdgpu_moverate >= 0)
4154 		max_MBps = amdgpu_moverate;
4155 	else
4156 		max_MBps = 8; /* Allow 8 MB/s. */
4157 	/* Get a log2 for easy divisions. */
4158 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4159 
4160 	/*
4161 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4162 	 * Otherwise the mgpu fan boost feature will be skipped due to the
4163 	 * gpu instance is counted less.
4164 	 */
4165 	amdgpu_register_gpu_instance(adev);
4166 
4167 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4168 	 * explicit gating rather than handling it automatically.
4169 	 */
4170 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4171 		r = amdgpu_device_ip_late_init(adev);
4172 		if (r) {
4173 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4174 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4175 			goto release_ras_con;
4176 		}
4177 		/* must succeed. */
4178 		amdgpu_ras_resume(adev);
4179 		queue_delayed_work(system_dfl_wq, &adev->delayed_init_work,
4180 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
4181 	}
4182 
4183 	if (amdgpu_sriov_vf(adev)) {
4184 		amdgpu_virt_release_full_gpu(adev, true);
4185 		flush_delayed_work(&adev->delayed_init_work);
4186 	}
4187 
4188 	/* Don't init kfd if whole hive need to be reset during init */
4189 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4190 		kgd2kfd_init_zone_device(adev);
4191 		kfd_update_svm_support_properties(adev);
4192 	}
4193 
4194 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
4195 		amdgpu_xgmi_reset_on_init(adev);
4196 
4197 	/*
4198 	 * Place those sysfs registering after `late_init`. As some of those
4199 	 * operations performed in `late_init` might affect the sysfs
4200 	 * interfaces creating.
4201 	 */
4202 	r = amdgpu_device_sys_interface_init(adev);
4203 
4204 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4205 		r = amdgpu_pmu_init(adev);
4206 	if (r)
4207 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4208 
4209 	/* Have stored pci confspace at hand for restore in sudden PCI error */
4210 	if (amdgpu_device_cache_pci_state(adev->pdev))
4211 		pci_restore_state(pdev);
4212 
4213 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4214 	/* this will fail for cards that aren't VGA class devices, just
4215 	 * ignore it
4216 	 */
4217 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4218 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode, adev);
4219 
4220 	px = amdgpu_device_supports_px(adev);
4221 
4222 	if (amdgpu_device_should_register_switcheroo(adev, px))
4223 		vga_switcheroo_register_client(adev->pdev,
4224 					       &amdgpu_switcheroo_ops, px);
4225 
4226 	if (px)
4227 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4228 
4229 	adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
4230 	r = register_pm_notifier(&adev->pm_nb);
4231 	if (r)
4232 		goto failed;
4233 
4234 	return 0;
4235 
4236 release_ras_con:
4237 	if (amdgpu_sriov_vf(adev))
4238 		amdgpu_virt_release_full_gpu(adev, true);
4239 
4240 	/* failed in exclusive mode due to timeout */
4241 	if (amdgpu_sriov_vf(adev) &&
4242 		!amdgpu_sriov_runtime(adev) &&
4243 		amdgpu_virt_mmio_blocked(adev) &&
4244 		!amdgpu_virt_wait_reset(adev)) {
4245 		dev_err(adev->dev, "VF exclusive mode timeout\n");
4246 		/* Don't send request since VF is inactive. */
4247 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4248 		adev->virt.ops = NULL;
4249 		r = -EAGAIN;
4250 	}
4251 	amdgpu_release_ras_context(adev);
4252 
4253 failed:
4254 	amdgpu_vf_error_trans_all(adev);
4255 
4256 	return r;
4257 }
4258 
4259 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4260 {
4261 
4262 	/* Clear all CPU mappings pointing to this device */
4263 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4264 
4265 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4266 	amdgpu_doorbell_fini(adev);
4267 
4268 	iounmap(adev->rmmio);
4269 	adev->rmmio = NULL;
4270 	if (adev->mman.aper_base_kaddr)
4271 		iounmap(adev->mman.aper_base_kaddr);
4272 	adev->mman.aper_base_kaddr = NULL;
4273 
4274 	/* Memory manager related */
4275 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4276 		arch_phys_wc_del(adev->gmc.vram_mtrr);
4277 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4278 	}
4279 }
4280 
4281 /**
4282  * amdgpu_device_fini_hw - tear down the driver
4283  *
4284  * @adev: amdgpu_device pointer
4285  *
4286  * Tear down the driver info (all asics).
4287  * Called at driver shutdown.
4288  */
4289 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4290 {
4291 	dev_info(adev->dev, "finishing device.\n");
4292 	flush_delayed_work(&adev->delayed_init_work);
4293 
4294 	if (adev->mman.initialized)
4295 		drain_workqueue(adev->mman.bdev.wq);
4296 
4297 	adev->shutdown = true;
4298 
4299 	unregister_pm_notifier(&adev->pm_nb);
4300 
4301 	/* make sure IB test finished before entering exclusive mode
4302 	 * to avoid preemption on IB test
4303 	 */
4304 	if (amdgpu_sriov_vf(adev)) {
4305 		amdgpu_virt_request_full_gpu(adev, false);
4306 		amdgpu_virt_fini_data_exchange(adev);
4307 	}
4308 
4309 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
4310 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
4311 
4312 	/* disable all interrupts */
4313 	amdgpu_irq_disable_all(adev);
4314 	if (adev->mode_info.mode_config_initialized) {
4315 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4316 			drm_helper_force_disable_all(adev_to_drm(adev));
4317 		else
4318 			drm_atomic_helper_shutdown(adev_to_drm(adev));
4319 	}
4320 	amdgpu_fence_driver_hw_fini(adev);
4321 
4322 	amdgpu_device_sys_interface_fini(adev);
4323 
4324 	/* disable ras feature must before hw fini */
4325 	amdgpu_ras_pre_fini(adev);
4326 
4327 	amdgpu_ttm_disable_buffer_funcs(adev);
4328 
4329 	/*
4330 	 * device went through surprise hotplug; we need to destroy topology
4331 	 * before ip_fini_early to prevent kfd locking refcount issues by calling
4332 	 * amdgpu_amdkfd_suspend()
4333 	 */
4334 	if (pci_dev_is_disconnected(adev->pdev)) {
4335 		/* Drain KFD processes while kfd.dev is still valid */
4336 		amdgpu_amdkfd_teardown_processes(adev);
4337 		amdgpu_amdkfd_device_fini_sw(adev);
4338 	}
4339 
4340 	amdgpu_coredump_fini(adev);
4341 	amdgpu_device_ip_fini_early(adev);
4342 
4343 	amdgpu_irq_fini_hw(adev);
4344 
4345 	if (adev->mman.initialized)
4346 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4347 
4348 	amdgpu_gart_dummy_page_fini(adev);
4349 
4350 	if (pci_dev_is_disconnected(adev->pdev))
4351 		amdgpu_device_unmap_mmio(adev);
4352 
4353 }
4354 
4355 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4356 {
4357 	int i;
4358 	bool px;
4359 
4360 	amdgpu_device_ip_fini(adev);
4361 	amdgpu_fence_driver_sw_fini(adev);
4362 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4363 	adev->accel_working = false;
4364 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4365 	for (i = 0; i < MAX_XCP; ++i) {
4366 		dma_fence_put(adev->isolation[i].spearhead);
4367 		amdgpu_sync_free(&adev->isolation[i].active);
4368 		amdgpu_sync_free(&adev->isolation[i].prev);
4369 	}
4370 
4371 	amdgpu_reset_fini(adev);
4372 
4373 	/* free i2c buses */
4374 	amdgpu_i2c_fini(adev);
4375 
4376 	if (adev->bios) {
4377 		if (amdgpu_emu_mode != 1)
4378 			amdgpu_atombios_fini(adev);
4379 		amdgpu_bios_release(adev);
4380 	}
4381 
4382 	kfree(adev->fru_info);
4383 	adev->fru_info = NULL;
4384 
4385 	kfree(adev->xcp_mgr);
4386 	adev->xcp_mgr = NULL;
4387 
4388 	px = amdgpu_device_supports_px(adev);
4389 
4390 	if (amdgpu_device_should_register_switcheroo(adev, px))
4391 		vga_switcheroo_unregister_client(adev->pdev);
4392 
4393 	if (px)
4394 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4395 
4396 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4397 		vga_client_unregister(adev->pdev);
4398 
4399 	if (adev->rmmio) {
4400 		iounmap(adev->rmmio);
4401 		adev->rmmio = NULL;
4402 	}
4403 
4404 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4405 		amdgpu_pmu_fini(adev);
4406 	if (adev->discovery.bin)
4407 		amdgpu_discovery_fini(adev);
4408 
4409 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4410 	adev->reset_domain = NULL;
4411 
4412 	kfree(adev->pci_state);
4413 	kfree(adev->pcie_reset_ctx.swds_pcistate);
4414 	kfree(adev->pcie_reset_ctx.swus_pcistate);
4415 }
4416 
4417 /**
4418  * amdgpu_device_evict_resources - evict device resources
4419  * @adev: amdgpu device object
4420  *
4421  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4422  * of the vram memory type. Mainly used for evicting device resources
4423  * at suspend time.
4424  *
4425  */
4426 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4427 {
4428 	int ret;
4429 
4430 	/* No need to evict vram on APUs unless going to S4 */
4431 	if (!adev->in_s4 && (adev->flags & AMD_IS_APU))
4432 		return 0;
4433 
4434 	/* No need to evict when going to S5 through S4 callbacks */
4435 	if (system_state == SYSTEM_POWER_OFF)
4436 		return 0;
4437 
4438 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4439 	if (ret) {
4440 		dev_warn(adev->dev, "evicting device resources failed\n");
4441 		return ret;
4442 	}
4443 
4444 	if (adev->in_s4) {
4445 		ret = ttm_device_prepare_hibernation(&adev->mman.bdev);
4446 		if (ret)
4447 			dev_err(adev->dev, "prepare hibernation failed, %d\n", ret);
4448 	}
4449 	return ret;
4450 }
4451 
4452 /*
4453  * Suspend & resume.
4454  */
4455 /**
4456  * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events
4457  * @nb: notifier block
4458  * @mode: suspend mode
4459  * @data: data
4460  *
4461  * This function is called when the system is about to suspend or hibernate.
4462  * It is used to set the appropriate flags so that eviction can be optimized
4463  * in the pm prepare callback.
4464  */
4465 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
4466 				     void *data)
4467 {
4468 	struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);
4469 
4470 	switch (mode) {
4471 	case PM_HIBERNATION_PREPARE:
4472 		adev->in_s4 = true;
4473 		break;
4474 	case PM_POST_HIBERNATION:
4475 		adev->in_s4 = false;
4476 		break;
4477 	}
4478 
4479 	return NOTIFY_DONE;
4480 }
4481 
4482 /**
4483  * amdgpu_device_prepare - prepare for device suspend
4484  *
4485  * @dev: drm dev pointer
4486  *
4487  * Prepare to put the hw in the suspend state (all asics).
4488  * Returns 0 for success or an error on failure.
4489  * Called at driver suspend.
4490  */
4491 int amdgpu_device_prepare(struct drm_device *dev)
4492 {
4493 	struct amdgpu_device *adev = drm_to_adev(dev);
4494 	int i, r;
4495 
4496 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4497 		return 0;
4498 
4499 	/* Evict the majority of BOs before starting suspend sequence */
4500 	r = amdgpu_device_evict_resources(adev);
4501 	if (r)
4502 		return r;
4503 
4504 	flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4505 
4506 	for (i = 0; i < adev->num_ip_blocks; i++) {
4507 		if (!adev->ip_blocks[i].status.valid)
4508 			continue;
4509 		if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4510 			continue;
4511 		r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);
4512 		if (r)
4513 			return r;
4514 	}
4515 
4516 	return 0;
4517 }
4518 
4519 /**
4520  * amdgpu_device_complete - complete power state transition
4521  *
4522  * @dev: drm dev pointer
4523  *
4524  * Undo the changes from amdgpu_device_prepare. This will be
4525  * called on all resume transitions, including those that failed.
4526  */
4527 void amdgpu_device_complete(struct drm_device *dev)
4528 {
4529 	struct amdgpu_device *adev = drm_to_adev(dev);
4530 	int i;
4531 
4532 	for (i = 0; i < adev->num_ip_blocks; i++) {
4533 		if (!adev->ip_blocks[i].status.valid)
4534 			continue;
4535 		if (!adev->ip_blocks[i].version->funcs->complete)
4536 			continue;
4537 		adev->ip_blocks[i].version->funcs->complete(&adev->ip_blocks[i]);
4538 	}
4539 }
4540 
4541 /**
4542  * amdgpu_device_suspend - initiate device suspend
4543  *
4544  * @dev: drm dev pointer
4545  * @notify_clients: notify in-kernel DRM clients
4546  *
4547  * Puts the hw in the suspend state (all asics).
4548  * Returns 0 for success or an error on failure.
4549  * Called at driver suspend.
4550  */
4551 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
4552 {
4553 	struct amdgpu_device *adev = drm_to_adev(dev);
4554 	int r, rec;
4555 
4556 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4557 		return 0;
4558 
4559 	adev->in_suspend = true;
4560 
4561 	if (amdgpu_sriov_vf(adev)) {
4562 		if (!adev->in_runpm)
4563 			amdgpu_amdkfd_suspend_process(adev);
4564 		amdgpu_virt_fini_data_exchange(adev);
4565 		r = amdgpu_virt_request_full_gpu(adev, false);
4566 		if (r)
4567 			return r;
4568 	}
4569 
4570 	r = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D3);
4571 	if (r)
4572 		goto unwind_sriov;
4573 
4574 	if (notify_clients)
4575 		drm_client_dev_suspend(adev_to_drm(adev));
4576 
4577 	cancel_delayed_work_sync(&adev->delayed_init_work);
4578 
4579 	amdgpu_ras_suspend(adev);
4580 
4581 	r = amdgpu_device_ip_suspend_phase1(adev);
4582 	if (r)
4583 		goto unwind_smartshift;
4584 
4585 	amdgpu_amdkfd_suspend(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4586 	r = amdgpu_userq_suspend(adev);
4587 	if (r)
4588 		goto unwind_ip_phase1;
4589 
4590 	r = amdgpu_device_evict_resources(adev);
4591 	if (r)
4592 		goto unwind_userq;
4593 
4594 	amdgpu_ttm_disable_buffer_funcs(adev);
4595 
4596 	amdgpu_fence_driver_hw_fini(adev);
4597 
4598 	r = amdgpu_device_ip_suspend_phase2(adev);
4599 	if (r)
4600 		goto unwind_evict;
4601 
4602 	if (amdgpu_sriov_vf(adev))
4603 		amdgpu_virt_release_full_gpu(adev, false);
4604 
4605 	return 0;
4606 
4607 unwind_evict:
4608 	amdgpu_ttm_enable_buffer_funcs(adev);
4609 	amdgpu_fence_driver_hw_init(adev);
4610 
4611 unwind_userq:
4612 	rec = amdgpu_userq_resume(adev);
4613 	if (rec) {
4614 		dev_warn(adev->dev, "failed to re-initialize user queues: %d\n", rec);
4615 		return r;
4616 	}
4617 	rec = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4618 	if (rec) {
4619 		dev_warn(adev->dev, "failed to re-initialize kfd: %d\n", rec);
4620 		return r;
4621 	}
4622 
4623 unwind_ip_phase1:
4624 	/* suspend phase 1 = resume phase 3 */
4625 	rec = amdgpu_device_ip_resume_phase3(adev);
4626 	if (rec) {
4627 		dev_warn(adev->dev, "failed to re-initialize IPs phase1: %d\n", rec);
4628 		return r;
4629 	}
4630 
4631 unwind_smartshift:
4632 	rec = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0);
4633 	if (rec) {
4634 		dev_warn(adev->dev, "failed to re-update smart shift: %d\n", rec);
4635 		return r;
4636 	}
4637 
4638 	if (notify_clients)
4639 		drm_client_dev_resume(adev_to_drm(adev));
4640 
4641 	amdgpu_ras_resume(adev);
4642 
4643 unwind_sriov:
4644 	if (amdgpu_sriov_vf(adev)) {
4645 		rec = amdgpu_virt_request_full_gpu(adev, true);
4646 		if (rec) {
4647 			dev_warn(adev->dev, "failed to reinitialize sriov: %d\n", rec);
4648 			return r;
4649 		}
4650 	}
4651 
4652 	adev->in_suspend = adev->in_s0ix = adev->in_s3 = false;
4653 
4654 	return r;
4655 }
4656 
4657 static inline int amdgpu_virt_resume(struct amdgpu_device *adev)
4658 {
4659 	int r;
4660 	unsigned int prev_physical_node_id = adev->gmc.xgmi.physical_node_id;
4661 
4662 	/* During VM resume, QEMU programming of VF MSIX table (register GFXMSIX_VECT0_ADDR_LO)
4663 	 * may not work. The access could be blocked by nBIF protection as VF isn't in
4664 	 * exclusive access mode. Exclusive access is enabled now, disable/enable MSIX
4665 	 * so that QEMU reprograms MSIX table.
4666 	 */
4667 	amdgpu_restore_msix(adev);
4668 
4669 	r = adev->gfxhub.funcs->get_xgmi_info(adev);
4670 	if (r)
4671 		return r;
4672 
4673 	dev_info(adev->dev, "xgmi node, old id %d, new id %d\n",
4674 		prev_physical_node_id, adev->gmc.xgmi.physical_node_id);
4675 
4676 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
4677 	adev->vm_manager.vram_base_offset +=
4678 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
4679 
4680 	return 0;
4681 }
4682 
4683 /**
4684  * amdgpu_device_resume - initiate device resume
4685  *
4686  * @dev: drm dev pointer
4687  * @notify_clients: notify in-kernel DRM clients
4688  *
4689  * Bring the hw back to operating state (all asics).
4690  * Returns 0 for success or an error on failure.
4691  * Called at driver resume.
4692  */
4693 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
4694 {
4695 	struct amdgpu_device *adev = drm_to_adev(dev);
4696 	int r = 0;
4697 
4698 	if (amdgpu_sriov_vf(adev)) {
4699 		r = amdgpu_virt_request_full_gpu(adev, true);
4700 		if (r)
4701 			return r;
4702 	}
4703 
4704 	if (amdgpu_virt_xgmi_migrate_enabled(adev)) {
4705 		r = amdgpu_virt_resume(adev);
4706 		if (r)
4707 			goto exit;
4708 	}
4709 
4710 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4711 		return 0;
4712 
4713 	if (adev->in_s0ix)
4714 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4715 
4716 	/* post card */
4717 	if (amdgpu_device_need_post(adev)) {
4718 		r = amdgpu_device_asic_init(adev);
4719 		if (r)
4720 			dev_err(adev->dev, "amdgpu asic init failed\n");
4721 	}
4722 
4723 	r = amdgpu_device_ip_resume(adev);
4724 
4725 	if (r) {
4726 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4727 		goto exit;
4728 	}
4729 
4730 	r = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4731 	if (r)
4732 		goto exit;
4733 
4734 	r = amdgpu_userq_resume(adev);
4735 	if (r)
4736 		goto exit;
4737 
4738 	r = amdgpu_device_ip_late_init(adev);
4739 	if (r)
4740 		goto exit;
4741 
4742 	queue_delayed_work(system_dfl_wq, &adev->delayed_init_work,
4743 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
4744 exit:
4745 	if (amdgpu_sriov_vf(adev)) {
4746 		amdgpu_virt_init_data_exchange(adev);
4747 		amdgpu_virt_release_full_gpu(adev, true);
4748 
4749 		if (!r && !adev->in_runpm)
4750 			r = amdgpu_amdkfd_resume_process(adev);
4751 	}
4752 
4753 	if (r)
4754 		return r;
4755 
4756 	/* Make sure IB tests flushed */
4757 	flush_delayed_work(&adev->delayed_init_work);
4758 
4759 	if (notify_clients)
4760 		drm_client_dev_resume(adev_to_drm(adev));
4761 
4762 	amdgpu_ras_resume(adev);
4763 
4764 	if (adev->mode_info.num_crtc) {
4765 		/*
4766 		 * Most of the connector probing functions try to acquire runtime pm
4767 		 * refs to ensure that the GPU is powered on when connector polling is
4768 		 * performed. Since we're calling this from a runtime PM callback,
4769 		 * trying to acquire rpm refs will cause us to deadlock.
4770 		 *
4771 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
4772 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
4773 		 */
4774 #ifdef CONFIG_PM
4775 		dev->dev->power.disable_depth++;
4776 #endif
4777 		if (!adev->dc_enabled)
4778 			drm_helper_hpd_irq_event(dev);
4779 		else
4780 			drm_kms_helper_hotplug_event(dev);
4781 #ifdef CONFIG_PM
4782 		dev->dev->power.disable_depth--;
4783 #endif
4784 	}
4785 
4786 	amdgpu_vram_mgr_clear_reset_blocks(adev);
4787 	adev->in_suspend = false;
4788 
4789 	if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0))
4790 		dev_warn(adev->dev, "smart shift update failed\n");
4791 
4792 	return 0;
4793 }
4794 
4795 /**
4796  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4797  *
4798  * @adev: amdgpu_device pointer
4799  * @reset_context: amdgpu reset context pointer
4800  *
4801  * do VF FLR and reinitialize Asic
4802  * return 0 means succeeded otherwise failed
4803  */
4804 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4805 				     struct amdgpu_reset_context *reset_context)
4806 {
4807 	int r;
4808 	struct amdgpu_hive_info *hive = NULL;
4809 
4810 	if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) {
4811 		if (!amdgpu_ras_get_fed_status(adev))
4812 			amdgpu_virt_ready_to_reset(adev);
4813 		amdgpu_virt_wait_reset(adev);
4814 		clear_bit(AMDGPU_HOST_FLR, &reset_context->flags);
4815 		r = amdgpu_virt_request_full_gpu(adev, true);
4816 	} else {
4817 		r = amdgpu_virt_reset_gpu(adev);
4818 	}
4819 	if (r)
4820 		return r;
4821 
4822 	amdgpu_ras_clear_err_state(adev);
4823 	amdgpu_irq_gpu_reset_resume_helper(adev);
4824 
4825 	/* some sw clean up VF needs to do before recover */
4826 	amdgpu_virt_post_reset(adev);
4827 
4828 	/* Resume IP prior to SMC */
4829 	r = amdgpu_device_ip_reinit_early_sriov(adev);
4830 	if (r)
4831 		return r;
4832 
4833 	amdgpu_virt_init_data_exchange(adev);
4834 
4835 	r = amdgpu_device_fw_loading(adev);
4836 	if (r)
4837 		return r;
4838 
4839 	/* now we are okay to resume SMC/CP/SDMA */
4840 	r = amdgpu_device_ip_reinit_late_sriov(adev);
4841 	if (r)
4842 		return r;
4843 
4844 	hive = amdgpu_get_xgmi_hive(adev);
4845 	/* Update PSP FW topology after reset */
4846 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4847 		r = amdgpu_xgmi_update_topology(hive, adev);
4848 	if (hive)
4849 		amdgpu_put_xgmi_hive(hive);
4850 	if (r)
4851 		return r;
4852 
4853 	r = amdgpu_ib_ring_tests(adev);
4854 	if (r)
4855 		return r;
4856 
4857 	if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)
4858 		amdgpu_inc_vram_lost(adev);
4859 
4860 	/* need to be called during full access so we can't do it later like
4861 	 * bare-metal does.
4862 	 */
4863 	amdgpu_amdkfd_post_reset(adev);
4864 	amdgpu_virt_release_full_gpu(adev, true);
4865 
4866 	/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
4867 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
4868 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
4869 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
4870 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
4871 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
4872 		amdgpu_ras_resume(adev);
4873 
4874 	amdgpu_virt_ras_telemetry_post_reset(adev);
4875 
4876 	return 0;
4877 }
4878 
4879 /**
4880  * amdgpu_device_has_job_running - check if there is any unfinished job
4881  *
4882  * @adev: amdgpu_device pointer
4883  *
4884  * check if there is any job running on the device when guest driver receives
4885  * FLR notification from host driver. If there are still jobs running, then
4886  * the guest driver will not respond the FLR reset. Instead, let the job hit
4887  * the timeout and guest driver then issue the reset request.
4888  */
4889 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4890 {
4891 	int i;
4892 
4893 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4894 		struct amdgpu_ring *ring = adev->rings[i];
4895 
4896 		if (!amdgpu_ring_sched_ready(ring))
4897 			continue;
4898 
4899 		if (amdgpu_fence_count_emitted(ring))
4900 			return true;
4901 	}
4902 	return false;
4903 }
4904 
4905 /**
4906  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4907  *
4908  * @adev: amdgpu_device pointer
4909  *
4910  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4911  * a hung GPU.
4912  */
4913 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4914 {
4915 
4916 	if (amdgpu_gpu_recovery == 0)
4917 		goto disabled;
4918 
4919 	/* Skip soft reset check in fatal error mode */
4920 	if (!amdgpu_ras_is_poison_mode_supported(adev))
4921 		return true;
4922 
4923 	if (amdgpu_sriov_vf(adev))
4924 		return true;
4925 
4926 	if (amdgpu_gpu_recovery == -1) {
4927 		switch (adev->asic_type) {
4928 #ifdef CONFIG_DRM_AMDGPU_SI
4929 		case CHIP_VERDE:
4930 		case CHIP_TAHITI:
4931 		case CHIP_PITCAIRN:
4932 		case CHIP_OLAND:
4933 		case CHIP_HAINAN:
4934 #endif
4935 #ifdef CONFIG_DRM_AMDGPU_CIK
4936 		case CHIP_KAVERI:
4937 		case CHIP_KABINI:
4938 		case CHIP_MULLINS:
4939 #endif
4940 		case CHIP_CARRIZO:
4941 		case CHIP_STONEY:
4942 		case CHIP_CYAN_SKILLFISH:
4943 			goto disabled;
4944 		default:
4945 			break;
4946 		}
4947 	}
4948 
4949 	return true;
4950 
4951 disabled:
4952 		dev_info(adev->dev, "GPU recovery disabled.\n");
4953 		return false;
4954 }
4955 
4956 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4957 {
4958 	u32 i;
4959 	int ret = 0;
4960 
4961 	if (adev->bios)
4962 		amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4963 
4964 	dev_info(adev->dev, "GPU mode1 reset\n");
4965 
4966 	/* Cache the state before bus master disable. The saved config space
4967 	 * values are used in other cases like restore after mode-2 reset.
4968 	 */
4969 	amdgpu_device_cache_pci_state(adev->pdev);
4970 
4971 	/* disable BM */
4972 	pci_clear_master(adev->pdev);
4973 
4974 	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4975 		dev_info(adev->dev, "GPU smu mode1 reset\n");
4976 		ret = amdgpu_dpm_mode1_reset(adev);
4977 	} else {
4978 		dev_info(adev->dev, "GPU psp mode1 reset\n");
4979 		ret = psp_gpu_reset(adev);
4980 	}
4981 
4982 	if (ret)
4983 		goto mode1_reset_failed;
4984 
4985 	/* enable mmio access after mode 1 reset completed */
4986 	adev->no_hw_access = false;
4987 
4988 	/* ensure no_hw_access is updated before we access hw */
4989 	smp_mb();
4990 
4991 	amdgpu_device_load_pci_state(adev->pdev);
4992 	ret = amdgpu_psp_wait_for_bootloader(adev);
4993 	if (ret)
4994 		goto mode1_reset_failed;
4995 
4996 	/* wait for asic to come out of reset */
4997 	for (i = 0; i < adev->usec_timeout; i++) {
4998 		u32 memsize = adev->nbio.funcs->get_memsize(adev);
4999 
5000 		if (memsize != 0xffffffff)
5001 			break;
5002 		udelay(1);
5003 	}
5004 
5005 	if (i >= adev->usec_timeout) {
5006 		ret = -ETIMEDOUT;
5007 		goto mode1_reset_failed;
5008 	}
5009 
5010 	if (adev->bios)
5011 		amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5012 
5013 	return 0;
5014 
5015 mode1_reset_failed:
5016 	dev_err(adev->dev, "GPU mode1 reset failed\n");
5017 	return ret;
5018 }
5019 
5020 int amdgpu_device_link_reset(struct amdgpu_device *adev)
5021 {
5022 	int ret = 0;
5023 
5024 	dev_info(adev->dev, "GPU link reset\n");
5025 
5026 	if (!amdgpu_reset_in_dpc(adev))
5027 		ret = amdgpu_dpm_link_reset(adev);
5028 
5029 	if (ret)
5030 		goto link_reset_failed;
5031 
5032 	ret = amdgpu_psp_wait_for_bootloader(adev);
5033 	if (ret)
5034 		goto link_reset_failed;
5035 
5036 	return 0;
5037 
5038 link_reset_failed:
5039 	dev_err(adev->dev, "GPU link reset failed\n");
5040 	return ret;
5041 }
5042 
5043 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5044 				 struct amdgpu_reset_context *reset_context)
5045 {
5046 	struct amdgpu_job *job = NULL;
5047 	struct dma_fence *fence = NULL;
5048 	struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
5049 	bool need_full_reset =
5050 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5051 	int i, r;
5052 
5053 	if (reset_context->reset_req_dev == adev)
5054 		job = reset_context->job;
5055 
5056 	if (amdgpu_sriov_vf(adev))
5057 		amdgpu_virt_pre_reset(adev);
5058 
5059 	amdgpu_fence_driver_isr_toggle(adev, true);
5060 
5061 	if (job)
5062 		fence = &job->hw_fence->base;
5063 
5064 	/* block all schedulers and reset given job's ring */
5065 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5066 		struct amdgpu_ring *ring = adev->rings[i];
5067 
5068 		if (!amdgpu_ring_sched_ready(ring))
5069 			continue;
5070 
5071 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5072 		amdgpu_fence_driver_force_completion(ring, fence);
5073 	}
5074 
5075 	/*
5076 	 * MES scheduler rings have no drm scheduler, so they are missed by the
5077 	 * loop above. Realign their polling fence too (one per XCC), otherwise the
5078 	 * first post-reset submission polls forever on a stale seq. sched.ready is
5079 	 * only set while the driver owns the ring.
5080 	 */
5081 	for (i = 0; i < AMDGPU_MAX_MES_INST_PIPES; i++) {
5082 		struct amdgpu_ring *mes_ring = &adev->mes.ring[i];
5083 
5084 		if (mes_ring->fence_drv.initialized && mes_ring->sched.ready)
5085 			amdgpu_fence_driver_force_completion(mes_ring, fence);
5086 	}
5087 
5088 	/*
5089 	 * KIQ rings are polling-fence/no_scheduler like MES, so realign their
5090 	 * fence too (one ring per XCC), otherwise the first post-reset KIQ
5091 	 * submission polls forever on a stale seq.
5092 	 */
5093 	for (i = 0; i < AMDGPU_MAX_GC_INSTANCES; i++) {
5094 		struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[i].ring;
5095 
5096 		if (kiq_ring->fence_drv.initialized && kiq_ring->sched.ready)
5097 			amdgpu_fence_driver_force_completion(kiq_ring, fence);
5098 	}
5099 
5100 	amdgpu_fence_driver_isr_toggle(adev, false);
5101 
5102 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5103 	/* If reset handler not implemented, continue; otherwise return */
5104 	if (r == -EOPNOTSUPP)
5105 		r = 0;
5106 	else
5107 		return r;
5108 
5109 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5110 	if (!amdgpu_sriov_vf(adev)) {
5111 		need_full_reset = true;
5112 
5113 		if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
5114 			dev_info(tmp_adev->dev, "Dumping IP State\n");
5115 			/* Trigger ip dump before we reset the asic */
5116 			for (i = 0; i < tmp_adev->num_ip_blocks; i++)
5117 				if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
5118 					tmp_adev->ip_blocks[i].version->funcs
5119 						->dump_ip_state((void *)&tmp_adev->ip_blocks[i]);
5120 			dev_info(tmp_adev->dev, "Dumping IP State Completed\n");
5121 		}
5122 
5123 		if (need_full_reset)
5124 			r = amdgpu_device_ip_suspend(adev);
5125 		if (need_full_reset)
5126 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5127 		else
5128 			clear_bit(AMDGPU_NEED_FULL_RESET,
5129 				  &reset_context->flags);
5130 	}
5131 
5132 	return r;
5133 }
5134 
5135 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
5136 {
5137 	struct list_head *device_list_handle;
5138 	bool full_reset, vram_lost = false;
5139 	struct amdgpu_device *tmp_adev;
5140 	int r, init_level;
5141 
5142 	device_list_handle = reset_context->reset_device_list;
5143 
5144 	if (!device_list_handle)
5145 		return -EINVAL;
5146 
5147 	full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5148 
5149 	/**
5150 	 * If it's reset on init, it's default init level, otherwise keep level
5151 	 * as recovery level.
5152 	 */
5153 	if (reset_context->method == AMD_RESET_METHOD_ON_INIT)
5154 			init_level = AMDGPU_INIT_LEVEL_DEFAULT;
5155 	else
5156 			init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY;
5157 
5158 	r = 0;
5159 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5160 		amdgpu_set_init_level(tmp_adev, init_level);
5161 		if (full_reset) {
5162 			/* post card */
5163 			amdgpu_reset_set_dpc_status(tmp_adev, false);
5164 			amdgpu_ras_clear_err_state(tmp_adev);
5165 			r = amdgpu_device_asic_init(tmp_adev);
5166 			if (r) {
5167 				dev_warn(tmp_adev->dev, "asic atom init failed!");
5168 			} else {
5169 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5170 
5171 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
5172 				if (r)
5173 					goto out;
5174 
5175 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5176 
5177 				if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
5178 					amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job);
5179 
5180 				if (vram_lost) {
5181 					dev_info(
5182 						tmp_adev->dev,
5183 						"VRAM is lost due to GPU reset!\n");
5184 					amdgpu_inc_vram_lost(tmp_adev);
5185 				}
5186 
5187 				r = amdgpu_device_fw_loading(tmp_adev);
5188 				if (r)
5189 					return r;
5190 
5191 				r = amdgpu_xcp_restore_partition_mode(
5192 					tmp_adev->xcp_mgr);
5193 				if (r)
5194 					goto out;
5195 
5196 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
5197 				if (r)
5198 					goto out;
5199 
5200 				r = amdgpu_device_ip_resume_phase3(tmp_adev);
5201 				if (r)
5202 					goto out;
5203 
5204 				if (vram_lost)
5205 					amdgpu_device_fill_reset_magic(tmp_adev);
5206 
5207 				/*
5208 				 * Add this ASIC as tracked as reset was already
5209 				 * complete successfully.
5210 				 */
5211 				amdgpu_register_gpu_instance(tmp_adev);
5212 
5213 				if (!reset_context->hive &&
5214 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5215 					amdgpu_xgmi_add_device(tmp_adev);
5216 
5217 				r = amdgpu_device_ip_late_init(tmp_adev);
5218 				if (r)
5219 					goto out;
5220 
5221 				r = amdgpu_userq_post_reset(tmp_adev, vram_lost);
5222 				if (r)
5223 					goto out;
5224 
5225 				drm_client_dev_resume(adev_to_drm(tmp_adev));
5226 
5227 				/*
5228 				 * The GPU enters bad state once faulty pages
5229 				 * by ECC has reached the threshold, and ras
5230 				 * recovery is scheduled next. So add one check
5231 				 * here to break recovery if it indeed exceeds
5232 				 * bad page threshold, and remind user to
5233 				 * retire this GPU or setting one bigger
5234 				 * bad_page_threshold value to fix this once
5235 				 * probing driver again.
5236 				 */
5237 				if (!amdgpu_ras_is_rma(tmp_adev)) {
5238 					/* must succeed. */
5239 					amdgpu_ras_resume(tmp_adev);
5240 				} else {
5241 					r = -EINVAL;
5242 					goto out;
5243 				}
5244 
5245 				/* Update PSP FW topology after reset */
5246 				if (reset_context->hive &&
5247 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5248 					r = amdgpu_xgmi_update_topology(
5249 						reset_context->hive, tmp_adev);
5250 			}
5251 		}
5252 
5253 out:
5254 		if (!r) {
5255 			/* IP init is complete now, set level as default */
5256 			amdgpu_set_init_level(tmp_adev,
5257 					      AMDGPU_INIT_LEVEL_DEFAULT);
5258 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5259 			r = amdgpu_ib_ring_tests(tmp_adev);
5260 			if (r) {
5261 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5262 				r = -EAGAIN;
5263 				goto end;
5264 			}
5265 		}
5266 
5267 		if (r)
5268 			tmp_adev->asic_reset_res = r;
5269 	}
5270 
5271 end:
5272 	return r;
5273 }
5274 
5275 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5276 			 struct amdgpu_reset_context *reset_context)
5277 {
5278 	struct amdgpu_device *tmp_adev = NULL;
5279 	bool need_full_reset, skip_hw_reset;
5280 	int r = 0;
5281 
5282 	/* Try reset handler method first */
5283 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5284 				    reset_list);
5285 
5286 	reset_context->reset_device_list = device_list_handle;
5287 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5288 	/* If reset handler not implemented, continue; otherwise return */
5289 	if (r == -EOPNOTSUPP)
5290 		r = 0;
5291 	else
5292 		return r;
5293 
5294 	/* Reset handler not implemented, use the default method */
5295 	need_full_reset =
5296 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5297 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5298 
5299 	/*
5300 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
5301 	 * to allow proper links negotiation in FW (within 1 sec)
5302 	 */
5303 	if (!skip_hw_reset && need_full_reset) {
5304 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5305 			/* For XGMI run all resets in parallel to speed up the process */
5306 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5307 				if (!queue_work(system_dfl_wq,
5308 						&tmp_adev->xgmi_reset_work))
5309 					r = -EALREADY;
5310 			} else
5311 				r = amdgpu_asic_reset(tmp_adev);
5312 
5313 			if (r) {
5314 				dev_err(tmp_adev->dev,
5315 					"ASIC reset failed with error, %d for drm dev, %s",
5316 					r, adev_to_drm(tmp_adev)->unique);
5317 				goto out;
5318 			}
5319 		}
5320 
5321 		/* For XGMI wait for all resets to complete before proceed */
5322 		if (!r) {
5323 			list_for_each_entry(tmp_adev, device_list_handle,
5324 					    reset_list) {
5325 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5326 					flush_work(&tmp_adev->xgmi_reset_work);
5327 					r = tmp_adev->asic_reset_res;
5328 					if (r)
5329 						break;
5330 				}
5331 			}
5332 		}
5333 	}
5334 
5335 	if (!r && amdgpu_ras_intr_triggered()) {
5336 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5337 			amdgpu_ras_reset_error_count(tmp_adev,
5338 						     AMDGPU_RAS_BLOCK__MMHUB);
5339 		}
5340 
5341 		amdgpu_ras_intr_cleared();
5342 	}
5343 
5344 	r = amdgpu_device_reinit_after_reset(reset_context);
5345 	if (r == -EAGAIN)
5346 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5347 	else
5348 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5349 
5350 out:
5351 	return r;
5352 }
5353 
5354 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5355 {
5356 
5357 	switch (amdgpu_asic_reset_method(adev)) {
5358 	case AMD_RESET_METHOD_MODE1:
5359 	case AMD_RESET_METHOD_LINK:
5360 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5361 		break;
5362 	case AMD_RESET_METHOD_MODE2:
5363 		adev->mp1_state = PP_MP1_STATE_RESET;
5364 		break;
5365 	default:
5366 		adev->mp1_state = PP_MP1_STATE_NONE;
5367 		break;
5368 	}
5369 }
5370 
5371 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5372 {
5373 	amdgpu_vf_error_trans_all(adev);
5374 	adev->mp1_state = PP_MP1_STATE_NONE;
5375 }
5376 
5377 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5378 {
5379 	struct pci_dev *p = NULL;
5380 
5381 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5382 			adev->pdev->bus->number, 1);
5383 	if (p) {
5384 		pm_runtime_enable(&(p->dev));
5385 		pm_runtime_resume(&(p->dev));
5386 	}
5387 
5388 	pci_dev_put(p);
5389 }
5390 
5391 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5392 {
5393 	enum amd_reset_method reset_method;
5394 	struct pci_dev *p = NULL;
5395 	u64 expires;
5396 
5397 	/*
5398 	 * For now, only BACO and mode1 reset are confirmed
5399 	 * to suffer the audio issue without proper suspended.
5400 	 */
5401 	reset_method = amdgpu_asic_reset_method(adev);
5402 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5403 	     (reset_method != AMD_RESET_METHOD_MODE1))
5404 		return -EINVAL;
5405 
5406 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5407 			adev->pdev->bus->number, 1);
5408 	if (!p)
5409 		return -ENODEV;
5410 
5411 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5412 	if (!expires)
5413 		/*
5414 		 * If we cannot get the audio device autosuspend delay,
5415 		 * a fixed 4S interval will be used. Considering 3S is
5416 		 * the audio controller default autosuspend delay setting.
5417 		 * 4S used here is guaranteed to cover that.
5418 		 */
5419 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5420 
5421 	while (!pm_runtime_status_suspended(&(p->dev))) {
5422 		if (!pm_runtime_suspend(&(p->dev)))
5423 			break;
5424 
5425 		if (expires < ktime_get_mono_fast_ns()) {
5426 			dev_warn(adev->dev, "failed to suspend display audio\n");
5427 			pci_dev_put(p);
5428 			/* TODO: abort the succeeding gpu reset? */
5429 			return -ETIMEDOUT;
5430 		}
5431 	}
5432 
5433 	pm_runtime_disable(&(p->dev));
5434 
5435 	pci_dev_put(p);
5436 	return 0;
5437 }
5438 
5439 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5440 {
5441 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5442 
5443 #if defined(CONFIG_DEBUG_FS)
5444 	if (!amdgpu_sriov_vf(adev))
5445 		cancel_work(&adev->reset_work);
5446 #endif
5447 	amdgpu_userq_mgr_cancel_reset_work(adev);
5448 
5449 	if (adev->kfd.dev)
5450 		cancel_work(&adev->kfd.reset_work);
5451 
5452 	if (amdgpu_sriov_vf(adev))
5453 		cancel_work(&adev->virt.flr_work);
5454 
5455 	if (con && adev->ras_enabled)
5456 		cancel_work(&con->recovery_work);
5457 
5458 }
5459 
5460 static int amdgpu_device_health_check(struct list_head *device_list_handle)
5461 {
5462 	struct amdgpu_device *tmp_adev;
5463 	int ret = 0;
5464 
5465 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5466 		ret |= amdgpu_device_bus_status_check(tmp_adev);
5467 	}
5468 
5469 	return ret;
5470 }
5471 
5472 static void amdgpu_device_recovery_prepare(struct amdgpu_device *adev,
5473 					  struct list_head *device_list,
5474 					  struct amdgpu_hive_info *hive)
5475 {
5476 	struct amdgpu_device *tmp_adev = NULL;
5477 
5478 	/*
5479 	 * Build list of devices to reset.
5480 	 * In case we are in XGMI hive mode, resort the device list
5481 	 * to put adev in the 1st position.
5482 	 */
5483 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
5484 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5485 			list_add_tail(&tmp_adev->reset_list, device_list);
5486 			if (adev->shutdown)
5487 				tmp_adev->shutdown = true;
5488 		}
5489 		if (!list_is_first(&adev->reset_list, device_list))
5490 			list_rotate_to_front(&adev->reset_list, device_list);
5491 	} else {
5492 		list_add_tail(&adev->reset_list, device_list);
5493 	}
5494 }
5495 
5496 static void amdgpu_device_recovery_get_reset_lock(struct amdgpu_device *adev,
5497 						  struct list_head *device_list)
5498 {
5499 	struct amdgpu_device *tmp_adev = NULL;
5500 
5501 	if (list_empty(device_list))
5502 		return;
5503 	tmp_adev =
5504 		list_first_entry(device_list, struct amdgpu_device, reset_list);
5505 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5506 }
5507 
5508 static void amdgpu_device_recovery_put_reset_lock(struct amdgpu_device *adev,
5509 						  struct list_head *device_list)
5510 {
5511 	struct amdgpu_device *tmp_adev = NULL;
5512 
5513 	if (list_empty(device_list))
5514 		return;
5515 	tmp_adev =
5516 		list_first_entry(device_list, struct amdgpu_device, reset_list);
5517 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5518 }
5519 
5520 static void amdgpu_device_halt_activities(struct amdgpu_device *adev,
5521 					  struct amdgpu_job *job,
5522 					  struct amdgpu_reset_context *reset_context,
5523 					  struct list_head *device_list,
5524 					  struct amdgpu_hive_info *hive,
5525 					  bool need_emergency_restart)
5526 {
5527 	struct amdgpu_device *tmp_adev = NULL;
5528 	int i;
5529 
5530 	/* Stop job submission first, the preparation below can take seconds */
5531 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5532 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5533 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5534 
5535 			if (!amdgpu_ring_sched_ready(ring))
5536 				continue;
5537 
5538 			drm_sched_wqueue_stop(&ring->sched);
5539 
5540 			if (need_emergency_restart)
5541 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5542 		}
5543 	}
5544 
5545 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5546 		amdgpu_device_set_mp1_state(tmp_adev);
5547 
5548 		/*
5549 		 * Try to put the audio codec into suspend state
5550 		 * before gpu reset started.
5551 		 *
5552 		 * Due to the power domain of the graphics device
5553 		 * is shared with AZ power domain. Without this,
5554 		 * we may change the audio hardware from behind
5555 		 * the audio driver's back. That will trigger
5556 		 * some audio codec errors.
5557 		 */
5558 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5559 			tmp_adev->pcie_reset_ctx.audio_suspended = true;
5560 
5561 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5562 
5563 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5564 
5565 		amdgpu_amdkfd_pre_reset(tmp_adev, reset_context);
5566 
5567 		/*
5568 		 * Mark these ASICs to be reset as untracked first
5569 		 * And add them back after reset completed
5570 		 */
5571 		amdgpu_unregister_gpu_instance(tmp_adev);
5572 
5573 		drm_client_dev_suspend(adev_to_drm(tmp_adev));
5574 
5575 		/* disable ras on ALL IPs */
5576 		if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev))
5577 			amdgpu_ras_suspend(tmp_adev);
5578 
5579 		amdgpu_userq_pre_reset(tmp_adev);
5580 		atomic_inc(&tmp_adev->gpu_reset_counter);
5581 	}
5582 }
5583 
5584 static int amdgpu_device_asic_reset(struct amdgpu_device *adev,
5585 			      struct list_head *device_list,
5586 			      struct amdgpu_reset_context *reset_context)
5587 {
5588 	struct amdgpu_device *tmp_adev = NULL;
5589 	int retry_limit = AMDGPU_MAX_RETRY_LIMIT;
5590 	int r = 0;
5591 
5592 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
5593 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5594 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5595 		/*TODO Should we stop ?*/
5596 		if (r) {
5597 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5598 				  r, adev_to_drm(tmp_adev)->unique);
5599 			tmp_adev->asic_reset_res = r;
5600 		}
5601 	}
5602 
5603 	/* Actual ASIC resets if needed.*/
5604 	/* Host driver will handle XGMI hive reset for SRIOV */
5605 	if (amdgpu_sriov_vf(adev)) {
5606 
5607 		/* Bail out of reset early */
5608 		if (amdgpu_ras_is_rma(adev))
5609 			return -ENODEV;
5610 
5611 		if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
5612 			dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
5613 			amdgpu_ras_set_fed(adev, true);
5614 			set_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5615 		}
5616 
5617 		r = amdgpu_device_reset_sriov(adev, reset_context);
5618 		if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) {
5619 			amdgpu_virt_release_full_gpu(adev, true);
5620 			goto retry;
5621 		}
5622 		if (r)
5623 			adev->asic_reset_res = r;
5624 	} else {
5625 		r = amdgpu_do_asic_reset(device_list, reset_context);
5626 		if (r && r == -EAGAIN)
5627 			goto retry;
5628 	}
5629 
5630 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5631 		/*
5632 		 * Drop any pending non scheduler resets queued before reset is done.
5633 		 * Any reset scheduled after this point would be valid. Scheduler resets
5634 		 * were already dropped during drm_sched_stop and no new ones can come
5635 		 * in before drm_sched_start.
5636 		 */
5637 		amdgpu_device_stop_pending_resets(tmp_adev);
5638 	}
5639 
5640 	return r;
5641 }
5642 
5643 static int amdgpu_device_sched_resume(struct list_head *device_list,
5644 			      struct amdgpu_reset_context *reset_context,
5645 			      bool   job_signaled)
5646 {
5647 	struct amdgpu_device *tmp_adev = NULL;
5648 	int i, r = 0;
5649 
5650 	/* Post ASIC reset for all devs .*/
5651 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5652 
5653 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5654 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5655 
5656 			if (!amdgpu_ring_sched_ready(ring))
5657 				continue;
5658 
5659 			drm_sched_wqueue_start(&ring->sched);
5660 		}
5661 
5662 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5663 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5664 
5665 		if (tmp_adev->asic_reset_res) {
5666 			/* bad news, how to tell it to userspace ?
5667 			 * for ras error, we should report GPU bad status instead of
5668 			 * reset failure
5669 			 */
5670 			if (reset_context->src != AMDGPU_RESET_SRC_RAS ||
5671 			    !amdgpu_ras_eeprom_check_err_threshold(tmp_adev))
5672 				dev_info(
5673 					tmp_adev->dev,
5674 					"GPU reset(%d) failed with error %d\n",
5675 					atomic_read(
5676 						&tmp_adev->gpu_reset_counter),
5677 					tmp_adev->asic_reset_res);
5678 			amdgpu_vf_error_put(tmp_adev,
5679 					    AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0,
5680 					    tmp_adev->asic_reset_res);
5681 			if (!r)
5682 				r = tmp_adev->asic_reset_res;
5683 			tmp_adev->asic_reset_res = 0;
5684 		} else {
5685 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n",
5686 				 atomic_read(&tmp_adev->gpu_reset_counter));
5687 			if (amdgpu_acpi_smart_shift_update(tmp_adev,
5688 							   AMDGPU_SS_DEV_D0))
5689 				dev_warn(tmp_adev->dev,
5690 					 "smart shift update failed\n");
5691 		}
5692 	}
5693 
5694 	return r;
5695 }
5696 
5697 static void amdgpu_device_gpu_resume(struct amdgpu_device *adev,
5698 			      struct list_head *device_list,
5699 			      bool   need_emergency_restart)
5700 {
5701 	struct amdgpu_device *tmp_adev = NULL;
5702 
5703 	list_for_each_entry(tmp_adev, device_list, reset_list) {
5704 		/* unlock kfd: SRIOV would do it separately */
5705 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5706 			amdgpu_amdkfd_post_reset(tmp_adev);
5707 
5708 		/* kfd_post_reset will do nothing if kfd device is not initialized,
5709 		 * need to bring up kfd here if it's not be initialized before
5710 		 */
5711 		if (!adev->kfd.init_complete)
5712 			amdgpu_amdkfd_device_init(adev);
5713 
5714 		if (tmp_adev->pcie_reset_ctx.audio_suspended)
5715 			amdgpu_device_resume_display_audio(tmp_adev);
5716 
5717 		amdgpu_device_unset_mp1_state(tmp_adev);
5718 
5719 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
5720 
5721 	}
5722 }
5723 
5724 
5725 /**
5726  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5727  *
5728  * @adev: amdgpu_device pointer
5729  * @job: which job trigger hang
5730  * @reset_context: amdgpu reset context pointer
5731  *
5732  * Attempt to reset the GPU if it has hung (all asics).
5733  * Attempt to do soft-reset or full-reset and reinitialize Asic
5734  * Returns 0 for success or an error on failure.
5735  */
5736 
5737 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5738 			      struct amdgpu_job *job,
5739 			      struct amdgpu_reset_context *reset_context)
5740 {
5741 	struct list_head device_list;
5742 	bool job_signaled = false;
5743 	struct amdgpu_hive_info *hive = NULL;
5744 	int r = 0;
5745 	bool need_emergency_restart = false;
5746 	/* save the pasid here as the job may be freed before the end of the reset */
5747 	int pasid = job ? job->pasid : -EINVAL;
5748 
5749 	/*
5750 	 * If it reaches here because of hang/timeout and a RAS error is
5751 	 * detected at the same time, let RAS recovery take care of it.
5752 	 */
5753 	if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) &&
5754 	    !amdgpu_sriov_vf(adev) &&
5755 	    reset_context->src != AMDGPU_RESET_SRC_RAS) {
5756 		dev_dbg(adev->dev,
5757 			"Gpu recovery from source: %d yielding to RAS error recovery handling",
5758 			reset_context->src);
5759 		return 0;
5760 	}
5761 
5762 	/*
5763 	 * Special case: RAS triggered and full reset isn't supported
5764 	 */
5765 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5766 
5767 	/*
5768 	 * Flush RAM to disk so that after reboot
5769 	 * the user can read log and see why the system rebooted.
5770 	 */
5771 	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5772 		amdgpu_ras_get_context(adev)->reboot) {
5773 		dev_warn(adev->dev, "Emergency reboot.");
5774 
5775 		ksys_sync_helper();
5776 		emergency_restart();
5777 	}
5778 
5779 	dev_info(adev->dev, "GPU %s begin!. Source:  %d\n",
5780 		 need_emergency_restart ? "jobs stop" : "reset",
5781 		 reset_context->src);
5782 
5783 	if (!amdgpu_sriov_vf(adev))
5784 		hive = amdgpu_get_xgmi_hive(adev);
5785 	if (hive)
5786 		mutex_lock(&hive->hive_lock);
5787 
5788 	reset_context->job = job;
5789 	reset_context->hive = hive;
5790 	INIT_LIST_HEAD(&device_list);
5791 
5792 	amdgpu_device_recovery_prepare(adev, &device_list, hive);
5793 
5794 	if (!amdgpu_sriov_vf(adev)) {
5795 		r = amdgpu_device_health_check(&device_list);
5796 		if (r)
5797 			goto end_reset;
5798 	}
5799 
5800 	/* Cannot be called after locking reset domain */
5801 	amdgpu_ras_pre_reset(adev, &device_list);
5802 
5803 	/* We need to lock reset domain only once both for XGMI and single device */
5804 	amdgpu_device_recovery_get_reset_lock(adev, &device_list);
5805 
5806 	/* unmap all the mappings of doorbell and framebuffer to prevent user space from
5807 	 * accessing them
5808 	 */
5809 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
5810 	amdgpu_amdkfd_clear_kfd_mapping(adev);
5811 
5812 	amdgpu_device_halt_activities(adev, job, reset_context, &device_list,
5813 				      hive, need_emergency_restart);
5814 	if (need_emergency_restart) {
5815 		amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart);
5816 		goto reset_unlock;
5817 	}
5818 	/*
5819 	 * Must check guilty signal here since after this point all old
5820 	 * HW fences are force signaled.
5821 	 *
5822 	 * job->base holds a reference to parent fence
5823 	 */
5824 	if (job && (dma_fence_get_status(&job->hw_fence->base) > 0)) {
5825 		job_signaled = true;
5826 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5827 		goto post_resume;
5828 	}
5829 
5830 	r = amdgpu_device_asic_reset(adev, &device_list, reset_context);
5831 	if (r)
5832 		goto reset_unlock;
5833 post_resume:
5834 	amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart);
5835 	r = amdgpu_device_sched_resume(&device_list, reset_context, job_signaled);
5836 	if (r)
5837 		goto reset_unlock;
5838 reset_unlock:
5839 	amdgpu_device_recovery_put_reset_lock(adev, &device_list);
5840 	amdgpu_ras_post_reset(adev, &device_list);
5841 end_reset:
5842 	if (hive) {
5843 		mutex_unlock(&hive->hive_lock);
5844 		amdgpu_put_xgmi_hive(hive);
5845 	}
5846 
5847 	if (r)
5848 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5849 
5850 	atomic_set(&adev->reset_domain->reset_res, r);
5851 
5852 	if (!r) {
5853 		struct amdgpu_task_info *ti = NULL;
5854 
5855 		/*
5856 		 * The job may already be freed at this point via the sched tdr workqueue so
5857 		 * use the cached pasid.
5858 		 */
5859 		if (pasid >= 0)
5860 			ti = amdgpu_vm_get_task_info_pasid(adev, pasid);
5861 
5862 		drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE,
5863 				     ti ? &ti->task : NULL);
5864 
5865 		amdgpu_vm_put_task_info(ti);
5866 	}
5867 
5868 	return r;
5869 }
5870 
5871 /**
5872  * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5873  *
5874  * @adev: amdgpu_device pointer
5875  * @speed: pointer to the speed of the link
5876  * @width: pointer to the width of the link
5877  *
5878  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5879  * first physical partner to an AMD dGPU.
5880  * This will exclude any virtual switches and links.
5881  */
5882 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5883 					    enum pci_bus_speed *speed,
5884 					    enum pcie_link_width *width)
5885 {
5886 	if (!speed || !width)
5887 		return;
5888 
5889 	*speed = PCI_SPEED_UNKNOWN;
5890 	*width = PCIE_LNK_WIDTH_UNKNOWN;
5891 
5892 	if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
5893 		if (adev->link_partner) {
5894 			*speed = pcie_get_speed_cap(adev->link_partner);
5895 			*width = pcie_get_width_cap(adev->link_partner);
5896 		}
5897 	} else {
5898 		/* use the current speeds rather than max if switching is not supported */
5899 		pcie_bandwidth_available(adev->pdev, NULL, speed, width);
5900 	}
5901 }
5902 
5903 /**
5904  * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU
5905  *
5906  * @adev: amdgpu_device pointer
5907  * @speed: pointer to the speed of the link
5908  * @width: pointer to the width of the link
5909  *
5910  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5911  * AMD dGPU which may be a virtual upstream bridge.
5912  */
5913 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,
5914 					enum pci_bus_speed *speed,
5915 					enum pcie_link_width *width)
5916 {
5917 	if (!speed || !width)
5918 		return;
5919 
5920 	*speed = pcie_get_speed_cap(adev->link_dev);
5921 	*width = pcie_get_width_cap(adev->link_dev);
5922 }
5923 
5924 /**
5925  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5926  *
5927  * @adev: amdgpu_device pointer
5928  *
5929  * Fetches and stores in the driver the PCIE capabilities (gen speed
5930  * and lanes) of the slot the device is in. Handles APUs and
5931  * virtualized environments where PCIE config space may not be available.
5932  */
5933 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5934 {
5935 	enum pci_bus_speed speed_cap, platform_speed_cap;
5936 	enum pcie_link_width platform_link_width, link_width;
5937 
5938 	if (amdgpu_pcie_gen_cap)
5939 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5940 
5941 	if (amdgpu_pcie_lane_cap)
5942 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5943 
5944 	/* covers APUs as well */
5945 	if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5946 		if (adev->pm.pcie_gen_mask == 0)
5947 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5948 		if (adev->pm.pcie_mlw_mask == 0)
5949 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5950 		return;
5951 	}
5952 
5953 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5954 		return;
5955 
5956 	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
5957 					&platform_link_width);
5958 	amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width);
5959 
5960 	if (adev->pm.pcie_gen_mask == 0) {
5961 		/* asic caps */
5962 		if (speed_cap == PCI_SPEED_UNKNOWN) {
5963 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5964 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5965 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5966 		} else {
5967 			if (speed_cap == PCIE_SPEED_32_0GT)
5968 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5969 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5970 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5971 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5972 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5973 			else if (speed_cap == PCIE_SPEED_16_0GT)
5974 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5975 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5976 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5977 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5978 			else if (speed_cap == PCIE_SPEED_8_0GT)
5979 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5980 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5981 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5982 			else if (speed_cap == PCIE_SPEED_5_0GT)
5983 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5984 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5985 			else
5986 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5987 		}
5988 		/* platform caps */
5989 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5990 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5991 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5992 		} else {
5993 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
5994 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5995 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5996 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5997 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5998 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5999 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
6000 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6001 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6002 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
6003 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
6004 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
6005 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6006 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
6007 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
6008 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
6009 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
6010 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
6011 			else
6012 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
6013 
6014 		}
6015 	}
6016 	if (adev->pm.pcie_mlw_mask == 0) {
6017 		/* asic caps */
6018 		if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6019 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK;
6020 		} else {
6021 			switch (link_width) {
6022 			case PCIE_LNK_X32:
6023 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 |
6024 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6025 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6026 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6027 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6028 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6029 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6030 				break;
6031 			case PCIE_LNK_X16:
6032 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6033 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6034 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6035 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6036 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6037 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6038 				break;
6039 			case PCIE_LNK_X12:
6040 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6041 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6042 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6043 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6044 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6045 				break;
6046 			case PCIE_LNK_X8:
6047 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6048 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6049 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6050 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6051 				break;
6052 			case PCIE_LNK_X4:
6053 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6054 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6055 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6056 				break;
6057 			case PCIE_LNK_X2:
6058 				adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6059 							   CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6060 				break;
6061 			case PCIE_LNK_X1:
6062 				adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1;
6063 				break;
6064 			default:
6065 				break;
6066 			}
6067 		}
6068 		/* platform caps */
6069 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6070 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
6071 		} else {
6072 			switch (platform_link_width) {
6073 			case PCIE_LNK_X32:
6074 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
6075 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6076 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6077 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6078 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6079 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6080 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6081 				break;
6082 			case PCIE_LNK_X16:
6083 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6084 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6085 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6086 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6087 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6088 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6089 				break;
6090 			case PCIE_LNK_X12:
6091 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6092 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6093 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6094 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6095 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6096 				break;
6097 			case PCIE_LNK_X8:
6098 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6099 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6100 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6101 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6102 				break;
6103 			case PCIE_LNK_X4:
6104 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6105 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6106 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6107 				break;
6108 			case PCIE_LNK_X2:
6109 				adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6110 							   CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6111 				break;
6112 			case PCIE_LNK_X1:
6113 				adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
6114 				break;
6115 			default:
6116 				break;
6117 			}
6118 		}
6119 	}
6120 }
6121 
6122 /**
6123  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6124  *
6125  * @adev: amdgpu_device pointer
6126  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6127  *
6128  * Return true if @peer_adev can access (DMA) @adev through the PCIe
6129  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6130  * @peer_adev.
6131  */
6132 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6133 				      struct amdgpu_device *peer_adev)
6134 {
6135 #ifdef CONFIG_HSA_AMD_P2P
6136 	bool p2p_access =
6137 		!adev->gmc.xgmi.connected_to_cpu &&
6138 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6139 	if (!p2p_access)
6140 		dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",
6141 			pci_name(peer_adev->pdev));
6142 
6143 	bool is_large_bar = adev->gmc.visible_vram_size &&
6144 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size;
6145 	bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev);
6146 
6147 	if (!p2p_addressable) {
6148 		uint64_t address_mask = peer_adev->dev->dma_mask ?
6149 			~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6150 		resource_size_t aper_limit =
6151 			adev->gmc.aper_base + adev->gmc.aper_size - 1;
6152 
6153 		p2p_addressable = !(adev->gmc.aper_base & address_mask ||
6154 				     aper_limit & address_mask);
6155 	}
6156 	return pcie_p2p && is_large_bar && p2p_access && p2p_addressable;
6157 #else
6158 	return false;
6159 #endif
6160 }
6161 
6162 int amdgpu_device_baco_enter(struct amdgpu_device *adev)
6163 {
6164 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6165 
6166 	if (!amdgpu_device_supports_baco(adev))
6167 		return -ENOTSUPP;
6168 
6169 	if (ras && adev->ras_enabled &&
6170 	    adev->nbio.funcs->enable_doorbell_interrupt)
6171 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6172 
6173 	return amdgpu_dpm_baco_enter(adev);
6174 }
6175 
6176 int amdgpu_device_baco_exit(struct amdgpu_device *adev)
6177 {
6178 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6179 	int ret = 0;
6180 
6181 	if (!amdgpu_device_supports_baco(adev))
6182 		return -ENOTSUPP;
6183 
6184 	ret = amdgpu_dpm_baco_exit(adev);
6185 	if (ret)
6186 		return ret;
6187 
6188 	if (ras && adev->ras_enabled &&
6189 	    adev->nbio.funcs->enable_doorbell_interrupt)
6190 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6191 
6192 	if (amdgpu_passthrough(adev) && adev->nbio.funcs &&
6193 	    adev->nbio.funcs->clear_doorbell_interrupt)
6194 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
6195 
6196 	return 0;
6197 }
6198 
6199 /**
6200  * amdgpu_pci_error_detected - Called when a PCI error is detected.
6201  * @pdev: PCI device struct
6202  * @state: PCI channel state
6203  *
6204  * Description: Called when a PCI error is detected.
6205  *
6206  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6207  */
6208 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6209 {
6210 	struct drm_device *dev = pci_get_drvdata(pdev);
6211 	struct amdgpu_device *adev = drm_to_adev(dev);
6212 	struct amdgpu_hive_info *hive __free(xgmi_put_hive) =
6213 		amdgpu_get_xgmi_hive(adev);
6214 	struct amdgpu_reset_context reset_context;
6215 	struct list_head device_list;
6216 
6217 	dev_info(adev->dev, "PCI error: detected callback!!\n");
6218 
6219 	adev->pci_channel_state = state;
6220 
6221 	switch (state) {
6222 	case pci_channel_io_normal:
6223 		dev_info(adev->dev, "pci_channel_io_normal: state(%d)!!\n", state);
6224 		return PCI_ERS_RESULT_CAN_RECOVER;
6225 	case pci_channel_io_frozen:
6226 		/* Fatal error, prepare for slot reset */
6227 		dev_info(adev->dev, "pci_channel_io_frozen: state(%d)!!\n", state);
6228 		if (hive) {
6229 			/* Hive devices should be able to support FW based
6230 			 * link reset on other devices, if not return.
6231 			 */
6232 			if (!amdgpu_dpm_is_link_reset_supported(adev)) {
6233 				dev_warn(adev->dev,
6234 					 "No support for XGMI hive yet...\n");
6235 				return PCI_ERS_RESULT_DISCONNECT;
6236 			}
6237 			/* Set dpc status only if device is part of hive
6238 			 * Non-hive devices should be able to recover after
6239 			 * link reset.
6240 			 */
6241 			amdgpu_reset_set_dpc_status(adev, true);
6242 
6243 			mutex_lock(&hive->hive_lock);
6244 		} else {
6245 			if (amdgpu_device_bus_status_check(adev))
6246 				amdgpu_reset_set_dpc_status(adev, true);
6247 		}
6248 		memset(&reset_context, 0, sizeof(reset_context));
6249 		INIT_LIST_HEAD(&device_list);
6250 
6251 		amdgpu_device_recovery_prepare(adev, &device_list, hive);
6252 		amdgpu_device_recovery_get_reset_lock(adev, &device_list);
6253 		amdgpu_device_halt_activities(adev, NULL, &reset_context, &device_list,
6254 					      hive, false);
6255 		if (hive)
6256 			mutex_unlock(&hive->hive_lock);
6257 		return PCI_ERS_RESULT_NEED_RESET;
6258 	case pci_channel_io_perm_failure:
6259 		/* Permanent error, prepare for device removal */
6260 		dev_info(adev->dev, "pci_channel_io_perm_failure: state(%d)!!\n", state);
6261 		return PCI_ERS_RESULT_DISCONNECT;
6262 	}
6263 
6264 	return PCI_ERS_RESULT_NEED_RESET;
6265 }
6266 
6267 /**
6268  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6269  * @pdev: pointer to PCI device
6270  */
6271 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6272 {
6273 	struct drm_device *dev = pci_get_drvdata(pdev);
6274 	struct amdgpu_device *adev = drm_to_adev(dev);
6275 
6276 	dev_info(adev->dev, "PCI error: mmio enabled callback!!\n");
6277 
6278 	/* TODO - dump whatever for debugging purposes */
6279 
6280 	/* This called only if amdgpu_pci_error_detected returns
6281 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6282 	 * works, no need to reset slot.
6283 	 */
6284 
6285 	return PCI_ERS_RESULT_RECOVERED;
6286 }
6287 
6288 /**
6289  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6290  * @pdev: PCI device struct
6291  *
6292  * Description: This routine is called by the pci error recovery
6293  * code after the PCI slot has been reset, just before we
6294  * should resume normal operations.
6295  */
6296 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6297 {
6298 	struct drm_device *dev = pci_get_drvdata(pdev);
6299 	struct amdgpu_device *adev = drm_to_adev(dev);
6300 	struct amdgpu_reset_context reset_context;
6301 	struct amdgpu_device *tmp_adev;
6302 	struct amdgpu_hive_info *hive;
6303 	struct list_head device_list;
6304 	struct pci_dev *link_dev;
6305 	int r = 0, i, timeout;
6306 	u32 memsize;
6307 	u16 status;
6308 
6309 	dev_info(adev->dev, "PCI error: slot reset callback!!\n");
6310 
6311 	memset(&reset_context, 0, sizeof(reset_context));
6312 	INIT_LIST_HEAD(&device_list);
6313 	hive = amdgpu_get_xgmi_hive(adev);
6314 	if (hive) {
6315 		mutex_lock(&hive->hive_lock);
6316 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
6317 			list_add_tail(&tmp_adev->reset_list, &device_list);
6318 	} else {
6319 		list_add_tail(&adev->reset_list, &device_list);
6320 	}
6321 
6322 	if (adev->pcie_reset_ctx.swus)
6323 		link_dev = adev->pcie_reset_ctx.swus;
6324 	else
6325 		link_dev = adev->pdev;
6326 	/* wait for asic to come out of reset, timeout = 10s */
6327 	timeout = 10000;
6328 	do {
6329 		usleep_range(10000, 10500);
6330 		r = pci_read_config_word(link_dev, PCI_VENDOR_ID, &status);
6331 		timeout -= 10;
6332 	} while (timeout > 0 && (status != PCI_VENDOR_ID_ATI) &&
6333 		 (status != PCI_VENDOR_ID_AMD));
6334 
6335 	if ((status != PCI_VENDOR_ID_ATI) && (status != PCI_VENDOR_ID_AMD)) {
6336 		r = -ETIME;
6337 		goto out;
6338 	}
6339 
6340 	amdgpu_device_load_switch_state(adev);
6341 	/* Restore PCI confspace */
6342 	amdgpu_device_load_pci_state(pdev);
6343 
6344 	/* confirm  ASIC came out of reset */
6345 	for (i = 0; i < adev->usec_timeout; i++) {
6346 		memsize = amdgpu_asic_get_config_memsize(adev);
6347 
6348 		if (memsize != 0xffffffff)
6349 			break;
6350 		udelay(1);
6351 	}
6352 	if (memsize == 0xffffffff) {
6353 		r = -ETIME;
6354 		goto out;
6355 	}
6356 
6357 	reset_context.method = AMD_RESET_METHOD_NONE;
6358 	reset_context.reset_req_dev = adev;
6359 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6360 	set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
6361 
6362 	if (hive) {
6363 		reset_context.hive = hive;
6364 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
6365 			tmp_adev->pcie_reset_ctx.in_link_reset = true;
6366 	} else {
6367 		adev->pcie_reset_ctx.in_link_reset = true;
6368 		set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6369 	}
6370 
6371 	r = amdgpu_device_asic_reset(adev, &device_list, &reset_context);
6372 out:
6373 	if (!r) {
6374 		if (amdgpu_device_cache_pci_state(adev->pdev))
6375 			pci_restore_state(adev->pdev);
6376 		dev_info(adev->dev, "PCIe error recovery succeeded\n");
6377 	} else {
6378 		dev_err(adev->dev, "PCIe error recovery failed, err:%d\n", r);
6379 		if (hive) {
6380 			list_for_each_entry(tmp_adev, &device_list, reset_list)
6381 				amdgpu_device_unset_mp1_state(tmp_adev);
6382 		}
6383 		amdgpu_device_recovery_put_reset_lock(adev, &device_list);
6384 	}
6385 
6386 	if (hive) {
6387 		mutex_unlock(&hive->hive_lock);
6388 		amdgpu_put_xgmi_hive(hive);
6389 	}
6390 
6391 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6392 }
6393 
6394 /**
6395  * amdgpu_pci_resume() - resume normal ops after PCI reset
6396  * @pdev: pointer to PCI device
6397  *
6398  * Called when the error recovery driver tells us that its
6399  * OK to resume normal operation.
6400  */
6401 void amdgpu_pci_resume(struct pci_dev *pdev)
6402 {
6403 	struct drm_device *dev = pci_get_drvdata(pdev);
6404 	struct amdgpu_device *adev = drm_to_adev(dev);
6405 	struct list_head device_list;
6406 	struct amdgpu_hive_info *hive = NULL;
6407 	struct amdgpu_device *tmp_adev = NULL;
6408 
6409 	dev_info(adev->dev, "PCI error: resume callback!!\n");
6410 
6411 	/* Only continue execution for the case of pci_channel_io_frozen */
6412 	if (adev->pci_channel_state != pci_channel_io_frozen)
6413 		return;
6414 
6415 	INIT_LIST_HEAD(&device_list);
6416 
6417 	hive = amdgpu_get_xgmi_hive(adev);
6418 	if (hive) {
6419 		mutex_lock(&hive->hive_lock);
6420 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
6421 			tmp_adev->pcie_reset_ctx.in_link_reset = false;
6422 			list_add_tail(&tmp_adev->reset_list, &device_list);
6423 		}
6424 	} else {
6425 		adev->pcie_reset_ctx.in_link_reset = false;
6426 		list_add_tail(&adev->reset_list, &device_list);
6427 	}
6428 	amdgpu_device_sched_resume(&device_list, NULL, NULL);
6429 	amdgpu_device_gpu_resume(adev, &device_list, false);
6430 	amdgpu_device_recovery_put_reset_lock(adev, &device_list);
6431 
6432 	if (hive) {
6433 		mutex_unlock(&hive->hive_lock);
6434 		amdgpu_put_xgmi_hive(hive);
6435 	}
6436 }
6437 
6438 static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev)
6439 {
6440 	struct pci_dev *swus, *swds;
6441 	int r;
6442 
6443 	swus = adev->link_dev != adev->pdev ? adev->link_dev : NULL;
6444 	if (!swus)
6445 		return;
6446 	swds = pci_upstream_bridge(adev->pdev);
6447 
6448 	/* If already saved, return */
6449 	if (adev->pcie_reset_ctx.swus)
6450 		return;
6451 	/* Upstream bridge is ATI, assume it's SWUS/DS architecture */
6452 	r = pci_save_state(swds);
6453 	if (r)
6454 		return;
6455 	adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(swds);
6456 
6457 	r = pci_save_state(swus);
6458 	if (r)
6459 		return;
6460 	adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(swus);
6461 
6462 	adev->pcie_reset_ctx.swus = swus;
6463 }
6464 
6465 static void amdgpu_device_load_switch_state(struct amdgpu_device *adev)
6466 {
6467 	struct pci_dev *pdev;
6468 	int r;
6469 
6470 	if (!adev->pcie_reset_ctx.swds_pcistate ||
6471 	    !adev->pcie_reset_ctx.swus_pcistate)
6472 		return;
6473 
6474 	pdev = adev->pcie_reset_ctx.swus;
6475 	r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate);
6476 	if (!r) {
6477 		pci_restore_state(pdev);
6478 	} else {
6479 		dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r);
6480 		return;
6481 	}
6482 
6483 	pdev = pci_upstream_bridge(adev->pdev);
6484 	r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate);
6485 	if (!r)
6486 		pci_restore_state(pdev);
6487 	else
6488 		dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r);
6489 }
6490 
6491 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6492 {
6493 	struct drm_device *dev = pci_get_drvdata(pdev);
6494 	struct amdgpu_device *adev = drm_to_adev(dev);
6495 	int r;
6496 
6497 	if (amdgpu_sriov_vf(adev))
6498 		return false;
6499 
6500 	r = pci_save_state(pdev);
6501 	if (!r) {
6502 		kfree(adev->pci_state);
6503 
6504 		adev->pci_state = pci_store_saved_state(pdev);
6505 
6506 		if (!adev->pci_state) {
6507 			dev_err(adev->dev, "Failed to store PCI saved state");
6508 			return false;
6509 		}
6510 	} else {
6511 		dev_warn(adev->dev, "Failed to save PCI state, err:%d\n", r);
6512 		return false;
6513 	}
6514 
6515 	amdgpu_device_cache_switch_state(adev);
6516 
6517 	return true;
6518 }
6519 
6520 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6521 {
6522 	struct drm_device *dev = pci_get_drvdata(pdev);
6523 	struct amdgpu_device *adev = drm_to_adev(dev);
6524 	int r;
6525 
6526 	if (!adev->pci_state)
6527 		return false;
6528 
6529 	r = pci_load_saved_state(pdev, adev->pci_state);
6530 
6531 	if (!r) {
6532 		pci_restore_state(pdev);
6533 	} else {
6534 		dev_warn(adev->dev, "Failed to load PCI state, err:%d\n", r);
6535 		return false;
6536 	}
6537 
6538 	return true;
6539 }
6540 
6541 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6542 		struct amdgpu_ring *ring)
6543 {
6544 #ifdef CONFIG_X86_64
6545 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6546 		return;
6547 #endif
6548 	if (adev->gmc.xgmi.connected_to_cpu)
6549 		return;
6550 
6551 	if (ring && ring->funcs->emit_hdp_flush) {
6552 		amdgpu_ring_emit_hdp_flush(ring);
6553 		return;
6554 	}
6555 
6556 	if (!ring && amdgpu_sriov_runtime(adev)) {
6557 		if (!amdgpu_kiq_hdp_flush(adev))
6558 			return;
6559 	}
6560 
6561 	amdgpu_hdp_flush(adev, ring);
6562 }
6563 
6564 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6565 		struct amdgpu_ring *ring)
6566 {
6567 #ifdef CONFIG_X86_64
6568 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6569 		return;
6570 #endif
6571 	if (adev->gmc.xgmi.connected_to_cpu)
6572 		return;
6573 
6574 	amdgpu_hdp_invalidate(adev, ring);
6575 }
6576 
6577 int amdgpu_in_reset(struct amdgpu_device *adev)
6578 {
6579 	return atomic_read(&adev->reset_domain->in_gpu_reset);
6580 }
6581 
6582 /**
6583  * amdgpu_device_halt() - bring hardware to some kind of halt state
6584  *
6585  * @adev: amdgpu_device pointer
6586  *
6587  * Bring hardware to some kind of halt state so that no one can touch it
6588  * any more. It will help to maintain error context when error occurred.
6589  * Compare to a simple hang, the system will keep stable at least for SSH
6590  * access. Then it should be trivial to inspect the hardware state and
6591  * see what's going on. Implemented as following:
6592  *
6593  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6594  *    clears all CPU mappings to device, disallows remappings through page faults
6595  * 2. amdgpu_irq_disable_all() disables all interrupts
6596  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6597  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6598  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6599  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6600  *    flush any in flight DMA operations
6601  */
6602 void amdgpu_device_halt(struct amdgpu_device *adev)
6603 {
6604 	struct pci_dev *pdev = adev->pdev;
6605 	struct drm_device *ddev = adev_to_drm(adev);
6606 
6607 	amdgpu_xcp_dev_unplug(adev);
6608 	drm_dev_unplug(ddev);
6609 
6610 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
6611 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
6612 
6613 	amdgpu_irq_disable_all(adev);
6614 
6615 	amdgpu_fence_driver_hw_fini(adev);
6616 
6617 	adev->no_hw_access = true;
6618 
6619 	amdgpu_device_unmap_mmio(adev);
6620 
6621 	pci_disable_device(pdev);
6622 	pci_wait_for_pending_transaction(pdev);
6623 }
6624 
6625 /**
6626  * amdgpu_device_get_gang - return a reference to the current gang
6627  * @adev: amdgpu_device pointer
6628  *
6629  * Returns: A new reference to the current gang leader.
6630  */
6631 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)
6632 {
6633 	struct dma_fence *fence;
6634 
6635 	rcu_read_lock();
6636 	fence = dma_fence_get_rcu_safe(&adev->gang_submit);
6637 	rcu_read_unlock();
6638 	return fence;
6639 }
6640 
6641 /**
6642  * amdgpu_device_switch_gang - switch to a new gang
6643  * @adev: amdgpu_device pointer
6644  * @gang: the gang to switch to
6645  *
6646  * Try to switch to a new gang.
6647  * Returns: NULL if we switched to the new gang or a reference to the current
6648  * gang leader.
6649  */
6650 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6651 					    struct dma_fence *gang)
6652 {
6653 	struct dma_fence *old = NULL;
6654 
6655 	dma_fence_get(gang);
6656 	do {
6657 		dma_fence_put(old);
6658 		old = amdgpu_device_get_gang(adev);
6659 		if (old == gang)
6660 			break;
6661 
6662 		if (!dma_fence_is_signaled(old)) {
6663 			dma_fence_put(gang);
6664 			return old;
6665 		}
6666 
6667 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6668 			 old, gang) != old);
6669 
6670 	/*
6671 	 * Drop it once for the exchanged reference in adev and once for the
6672 	 * thread local reference acquired in amdgpu_device_get_gang().
6673 	 */
6674 	dma_fence_put(old);
6675 	dma_fence_put(old);
6676 	return NULL;
6677 }
6678 
6679 /**
6680  * amdgpu_device_enforce_isolation - enforce HW isolation
6681  * @adev: the amdgpu device pointer
6682  * @ring: the HW ring the job is supposed to run on
6683  * @job: the job which is about to be pushed to the HW ring
6684  *
6685  * Makes sure that only one client at a time can use the GFX block.
6686  * Returns: The dependency to wait on before the job can be pushed to the HW.
6687  * The function is called multiple times until NULL is returned.
6688  */
6689 struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,
6690 						  struct amdgpu_ring *ring,
6691 						  struct amdgpu_job *job)
6692 {
6693 	struct drm_sched_fence *f = job->base.s_fence;
6694 	struct amdgpu_isolation *isolation;
6695 	struct dma_fence *dep;
6696 	void *owner;
6697 	int r;
6698 
6699 	/*
6700 	 * For now enforce isolation only for the GFX block since we only need
6701 	 * the cleaner shader on those rings.
6702 	 */
6703 	if (ring->funcs->type != AMDGPU_RING_TYPE_GFX &&
6704 	    ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
6705 		return NULL;
6706 
6707 	isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ?
6708 				     0 : ring->xcp_id];
6709 
6710 	/*
6711 	 * All submissions where enforce isolation is false are handled as if
6712 	 * they come from a single client. Use ~0l as the owner to distinct it
6713 	 * from kernel submissions where the owner is NULL.
6714 	 */
6715 	owner = job->enforce_isolation ? f->owner : (void *)~0l;
6716 
6717 	mutex_lock(&adev->enforce_isolation_mutex);
6718 
6719 	/*
6720 	 * The "spearhead" submission is the first one which changes the
6721 	 * ownership to its client. We always need to wait for it to be
6722 	 * pushed to the HW before proceeding with anything.
6723 	 */
6724 	if (&f->scheduled != isolation->spearhead &&
6725 	    !dma_fence_is_signaled(isolation->spearhead)) {
6726 		dep = isolation->spearhead;
6727 		goto out_grab_ref;
6728 	}
6729 
6730 	if (isolation->owner != owner) {
6731 
6732 		/*
6733 		 * Wait for any gang to be assembled before switching to a
6734 		 * different owner or otherwise we could deadlock the
6735 		 * submissions.
6736 		 */
6737 		if (!job->gang_submit) {
6738 			dep = amdgpu_device_get_gang(adev);
6739 			if (!dma_fence_is_signaled(dep))
6740 				goto out_return_dep;
6741 			dma_fence_put(dep);
6742 		}
6743 
6744 		dma_fence_put(isolation->spearhead);
6745 		isolation->spearhead = dma_fence_get(&f->scheduled);
6746 		amdgpu_sync_move(&isolation->active, &isolation->prev);
6747 		trace_amdgpu_isolation(isolation->owner, owner);
6748 		isolation->owner = owner;
6749 	}
6750 
6751 	/*
6752 	 * Specifying the ring here helps to pipeline submissions even when
6753 	 * isolation is enabled. If that is not desired for testing NULL can be
6754 	 * used instead of the ring to enforce a CPU round trip while switching
6755 	 * between clients.
6756 	 */
6757 	dep = amdgpu_sync_peek_fence(&isolation->prev, ring);
6758 	r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT);
6759 	if (r)
6760 		dev_warn(adev->dev, "OOM tracking isolation\n");
6761 
6762 out_grab_ref:
6763 	dma_fence_get(dep);
6764 out_return_dep:
6765 	mutex_unlock(&adev->enforce_isolation_mutex);
6766 	return dep;
6767 }
6768 
6769 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6770 {
6771 	switch (adev->asic_type) {
6772 #ifdef CONFIG_DRM_AMDGPU_SI
6773 	case CHIP_HAINAN:
6774 #endif
6775 	case CHIP_TOPAZ:
6776 		/* chips with no display hardware */
6777 		return false;
6778 #ifdef CONFIG_DRM_AMDGPU_SI
6779 	case CHIP_TAHITI:
6780 	case CHIP_PITCAIRN:
6781 	case CHIP_VERDE:
6782 	case CHIP_OLAND:
6783 #endif
6784 #ifdef CONFIG_DRM_AMDGPU_CIK
6785 	case CHIP_BONAIRE:
6786 	case CHIP_HAWAII:
6787 	case CHIP_KAVERI:
6788 	case CHIP_KABINI:
6789 	case CHIP_MULLINS:
6790 #endif
6791 	case CHIP_TONGA:
6792 	case CHIP_FIJI:
6793 	case CHIP_POLARIS10:
6794 	case CHIP_POLARIS11:
6795 	case CHIP_POLARIS12:
6796 	case CHIP_VEGAM:
6797 	case CHIP_CARRIZO:
6798 	case CHIP_STONEY:
6799 		/* chips with display hardware */
6800 		return true;
6801 	default:
6802 		/* IP discovery */
6803 		if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6804 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6805 			return false;
6806 		return true;
6807 	}
6808 }
6809 
6810 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
6811 {
6812 	ssize_t size = 0;
6813 
6814 	if (!ring || !ring->adev)
6815 		return size;
6816 
6817 	if (amdgpu_device_should_recover_gpu(ring->adev))
6818 		size |= AMDGPU_RESET_TYPE_FULL;
6819 
6820 	if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
6821 	    !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
6822 		size |= AMDGPU_RESET_TYPE_SOFT_RECOVERY;
6823 
6824 	return size;
6825 }
6826 
6827 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
6828 {
6829 	ssize_t size = 0;
6830 
6831 	if (supported_reset == 0) {
6832 		size += sysfs_emit_at(buf, size, "unsupported");
6833 		size += sysfs_emit_at(buf, size, "\n");
6834 		return size;
6835 
6836 	}
6837 
6838 	if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RECOVERY)
6839 		size += sysfs_emit_at(buf, size, "soft_recovery ");
6840 
6841 	if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
6842 		size += sysfs_emit_at(buf, size, "queue ");
6843 
6844 	if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE)
6845 		size += sysfs_emit_at(buf, size, "pipe ");
6846 
6847 	if (supported_reset & AMDGPU_RESET_TYPE_FULL)
6848 		size += sysfs_emit_at(buf, size, "full ");
6849 
6850 	size += sysfs_emit_at(buf, size, "\n");
6851 	return size;
6852 }
6853 
6854 void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
6855 			   enum amdgpu_uid_type type, uint8_t inst,
6856 			   uint64_t uid)
6857 {
6858 	if (!uid_info)
6859 		return;
6860 
6861 	if (type >= AMDGPU_UID_TYPE_MAX) {
6862 		dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n",
6863 			     type);
6864 		return;
6865 	}
6866 
6867 	if (inst >= AMDGPU_UID_INST_MAX) {
6868 		dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n",
6869 			     inst);
6870 		return;
6871 	}
6872 
6873 	if (uid_info->uid[type][inst] != 0) {
6874 		dev_warn_once(
6875 			uid_info->adev->dev,
6876 			"Overwriting existing UID %llu for type %d instance %d\n",
6877 			uid_info->uid[type][inst], type, inst);
6878 	}
6879 
6880 	uid_info->uid[type][inst] = uid;
6881 }
6882 
6883 u64 amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
6884 			  enum amdgpu_uid_type type, uint8_t inst)
6885 {
6886 	if (!uid_info)
6887 		return 0;
6888 
6889 	if (type >= AMDGPU_UID_TYPE_MAX) {
6890 		dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n",
6891 			     type);
6892 		return 0;
6893 	}
6894 
6895 	if (inst >= AMDGPU_UID_INST_MAX) {
6896 		dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n",
6897 			     inst);
6898 		return 0;
6899 	}
6900 
6901 	return uid_info->uid[type][inst];
6902 }
6903