xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/debugfs.h>
25 #include <linux/list.h>
26 #include <linux/module.h>
27 #include <linux/uaccess.h>
28 #include <linux/reboot.h>
29 #include <linux/syscalls.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/list_sort.h>
32 
33 #include "amdgpu.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_atomfirmware.h"
36 #include "amdgpu_xgmi.h"
37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
38 #include "nbio_v4_3.h"
39 #include "nbif_v6_3_1.h"
40 #include "nbio_v7_9.h"
41 #include "atom.h"
42 #include "amdgpu_reset.h"
43 #include "amdgpu_psp.h"
44 
45 #ifdef CONFIG_X86_MCE_AMD
46 #include <asm/mce.h>
47 
48 static bool notifier_registered;
49 #endif
50 static const char *RAS_FS_NAME = "ras";
51 
52 const char *ras_error_string[] = {
53 	"none",
54 	"parity",
55 	"single_correctable",
56 	"multi_uncorrectable",
57 	"poison",
58 };
59 
60 const char *ras_block_string[] = {
61 	"umc",
62 	"sdma",
63 	"gfx",
64 	"mmhub",
65 	"athub",
66 	"pcie_bif",
67 	"hdp",
68 	"xgmi_wafl",
69 	"df",
70 	"smn",
71 	"sem",
72 	"mp0",
73 	"mp1",
74 	"fuse",
75 	"mca",
76 	"vcn",
77 	"jpeg",
78 	"ih",
79 	"mpio",
80 };
81 
82 const char *ras_mca_block_string[] = {
83 	"mca_mp0",
84 	"mca_mp1",
85 	"mca_mpio",
86 	"mca_iohc",
87 };
88 
89 struct amdgpu_ras_block_list {
90 	/* ras block link */
91 	struct list_head node;
92 
93 	struct amdgpu_ras_block_object *ras_obj;
94 };
95 
get_ras_block_str(struct ras_common_if * ras_block)96 const char *get_ras_block_str(struct ras_common_if *ras_block)
97 {
98 	if (!ras_block)
99 		return "NULL";
100 
101 	if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
102 	    ras_block->block >= ARRAY_SIZE(ras_block_string))
103 		return "OUT OF RANGE";
104 
105 	if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
106 		return ras_mca_block_string[ras_block->sub_block_index];
107 
108 	return ras_block_string[ras_block->block];
109 }
110 
111 #define ras_block_str(_BLOCK_) \
112 	(((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
113 
114 #define ras_err_str(i) (ras_error_string[ffs(i)])
115 
116 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
117 
118 /* inject address is 52 bits */
119 #define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
120 
121 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
122 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
123 
124 #define MAX_UMC_POISON_POLLING_TIME_ASYNC  300  //ms
125 
126 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100  //ms
127 
128 #define MAX_FLUSH_RETIRE_DWORK_TIMES  100
129 
130 enum amdgpu_ras_retire_page_reservation {
131 	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
132 	AMDGPU_RAS_RETIRE_PAGE_PENDING,
133 	AMDGPU_RAS_RETIRE_PAGE_FAULT,
134 };
135 
136 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
137 
138 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
139 				uint64_t addr);
140 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
141 				uint64_t addr);
142 #ifdef CONFIG_X86_MCE_AMD
143 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
144 struct mce_notifier_adev_list {
145 	struct amdgpu_device *devs[MAX_GPU_INSTANCE];
146 	int num_gpu;
147 };
148 static struct mce_notifier_adev_list mce_adev_list;
149 #endif
150 
amdgpu_ras_set_error_query_ready(struct amdgpu_device * adev,bool ready)151 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
152 {
153 	if (adev && amdgpu_ras_get_context(adev))
154 		amdgpu_ras_get_context(adev)->error_query_ready = ready;
155 }
156 
amdgpu_ras_get_error_query_ready(struct amdgpu_device * adev)157 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
158 {
159 	if (adev && amdgpu_ras_get_context(adev))
160 		return amdgpu_ras_get_context(adev)->error_query_ready;
161 
162 	return false;
163 }
164 
amdgpu_reserve_page_direct(struct amdgpu_device * adev,uint64_t address)165 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
166 {
167 	struct ras_err_data err_data;
168 	struct eeprom_table_record err_rec;
169 	int ret;
170 
171 	if ((address >= adev->gmc.mc_vram_size) ||
172 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
173 		dev_warn(adev->dev,
174 		         "RAS WARN: input address 0x%llx is invalid.\n",
175 		         address);
176 		return -EINVAL;
177 	}
178 
179 	if (amdgpu_ras_check_bad_page(adev, address)) {
180 		dev_warn(adev->dev,
181 			 "RAS WARN: 0x%llx has already been marked as bad page!\n",
182 			 address);
183 		return 0;
184 	}
185 
186 	ret = amdgpu_ras_error_data_init(&err_data);
187 	if (ret)
188 		return ret;
189 
190 	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
191 	err_data.err_addr = &err_rec;
192 	amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
193 
194 	if (amdgpu_bad_page_threshold != 0) {
195 		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
196 					 err_data.err_addr_cnt, false);
197 		amdgpu_ras_save_bad_pages(adev, NULL);
198 	}
199 
200 	amdgpu_ras_error_data_fini(&err_data);
201 
202 	dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
203 	dev_warn(adev->dev, "Clear EEPROM:\n");
204 	dev_warn(adev->dev, "    echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
205 
206 	return 0;
207 }
208 
amdgpu_ras_debugfs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)209 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
210 					size_t size, loff_t *pos)
211 {
212 	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
213 	struct ras_query_if info = {
214 		.head = obj->head,
215 	};
216 	ssize_t s;
217 	char val[128];
218 
219 	if (amdgpu_ras_query_error_status(obj->adev, &info))
220 		return -EINVAL;
221 
222 	/* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
223 	if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
224 	    amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
225 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
226 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
227 	}
228 
229 	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
230 			"ue", info.ue_count,
231 			"ce", info.ce_count);
232 	if (*pos >= s)
233 		return 0;
234 
235 	s -= *pos;
236 	s = min_t(u64, s, size);
237 
238 
239 	if (copy_to_user(buf, &val[*pos], s))
240 		return -EINVAL;
241 
242 	*pos += s;
243 
244 	return s;
245 }
246 
247 static const struct file_operations amdgpu_ras_debugfs_ops = {
248 	.owner = THIS_MODULE,
249 	.read = amdgpu_ras_debugfs_read,
250 	.write = NULL,
251 	.llseek = default_llseek
252 };
253 
amdgpu_ras_find_block_id_by_name(const char * name,int * block_id)254 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
255 {
256 	int i;
257 
258 	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
259 		*block_id = i;
260 		if (strcmp(name, ras_block_string[i]) == 0)
261 			return 0;
262 	}
263 	return -EINVAL;
264 }
265 
amdgpu_ras_debugfs_ctrl_parse_data(struct file * f,const char __user * buf,size_t size,loff_t * pos,struct ras_debug_if * data)266 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
267 		const char __user *buf, size_t size,
268 		loff_t *pos, struct ras_debug_if *data)
269 {
270 	ssize_t s = min_t(u64, 64, size);
271 	char str[65];
272 	char block_name[33];
273 	char err[9] = "ue";
274 	int op = -1;
275 	int block_id;
276 	uint32_t sub_block;
277 	u64 address, value;
278 	/* default value is 0 if the mask is not set by user */
279 	u32 instance_mask = 0;
280 
281 	if (*pos)
282 		return -EINVAL;
283 	*pos = size;
284 
285 	memset(str, 0, sizeof(str));
286 	memset(data, 0, sizeof(*data));
287 
288 	if (copy_from_user(str, buf, s))
289 		return -EINVAL;
290 
291 	if (sscanf(str, "disable %32s", block_name) == 1)
292 		op = 0;
293 	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
294 		op = 1;
295 	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
296 		op = 2;
297 	else if (strstr(str, "retire_page") != NULL)
298 		op = 3;
299 	else if (str[0] && str[1] && str[2] && str[3])
300 		/* ascii string, but commands are not matched. */
301 		return -EINVAL;
302 
303 	if (op != -1) {
304 		if (op == 3) {
305 			if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
306 			    sscanf(str, "%*s %llu", &address) != 1)
307 				return -EINVAL;
308 
309 			data->op = op;
310 			data->inject.address = address;
311 
312 			return 0;
313 		}
314 
315 		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
316 			return -EINVAL;
317 
318 		data->head.block = block_id;
319 		/* only ue, ce and poison errors are supported */
320 		if (!memcmp("ue", err, 2))
321 			data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
322 		else if (!memcmp("ce", err, 2))
323 			data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
324 		else if (!memcmp("poison", err, 6))
325 			data->head.type = AMDGPU_RAS_ERROR__POISON;
326 		else
327 			return -EINVAL;
328 
329 		data->op = op;
330 
331 		if (op == 2) {
332 			if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
333 				   &sub_block, &address, &value, &instance_mask) != 4 &&
334 			    sscanf(str, "%*s %*s %*s %u %llu %llu %u",
335 				   &sub_block, &address, &value, &instance_mask) != 4 &&
336 				sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
337 				   &sub_block, &address, &value) != 3 &&
338 			    sscanf(str, "%*s %*s %*s %u %llu %llu",
339 				   &sub_block, &address, &value) != 3)
340 				return -EINVAL;
341 			data->head.sub_block_index = sub_block;
342 			data->inject.address = address;
343 			data->inject.value = value;
344 			data->inject.instance_mask = instance_mask;
345 		}
346 	} else {
347 		if (size < sizeof(*data))
348 			return -EINVAL;
349 
350 		if (copy_from_user(data, buf, sizeof(*data)))
351 			return -EINVAL;
352 	}
353 
354 	return 0;
355 }
356 
amdgpu_ras_instance_mask_check(struct amdgpu_device * adev,struct ras_debug_if * data)357 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
358 				struct ras_debug_if *data)
359 {
360 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
361 	uint32_t mask, inst_mask = data->inject.instance_mask;
362 
363 	/* no need to set instance mask if there is only one instance */
364 	if (num_xcc <= 1 && inst_mask) {
365 		data->inject.instance_mask = 0;
366 		dev_dbg(adev->dev,
367 			"RAS inject mask(0x%x) isn't supported and force it to 0.\n",
368 			inst_mask);
369 
370 		return;
371 	}
372 
373 	switch (data->head.block) {
374 	case AMDGPU_RAS_BLOCK__GFX:
375 		mask = GENMASK(num_xcc - 1, 0);
376 		break;
377 	case AMDGPU_RAS_BLOCK__SDMA:
378 		mask = GENMASK(adev->sdma.num_instances - 1, 0);
379 		break;
380 	case AMDGPU_RAS_BLOCK__VCN:
381 	case AMDGPU_RAS_BLOCK__JPEG:
382 		mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
383 		break;
384 	default:
385 		mask = inst_mask;
386 		break;
387 	}
388 
389 	/* remove invalid bits in instance mask */
390 	data->inject.instance_mask &= mask;
391 	if (inst_mask != data->inject.instance_mask)
392 		dev_dbg(adev->dev,
393 			"Adjust RAS inject mask 0x%x to 0x%x\n",
394 			inst_mask, data->inject.instance_mask);
395 }
396 
397 /**
398  * DOC: AMDGPU RAS debugfs control interface
399  *
400  * The control interface accepts struct ras_debug_if which has two members.
401  *
402  * First member: ras_debug_if::head or ras_debug_if::inject.
403  *
404  * head is used to indicate which IP block will be under control.
405  *
406  * head has four members, they are block, type, sub_block_index, name.
407  * block: which IP will be under control.
408  * type: what kind of error will be enabled/disabled/injected.
409  * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
410  * name: the name of IP.
411  *
412  * inject has three more members than head, they are address, value and mask.
413  * As their names indicate, inject operation will write the
414  * value to the address.
415  *
416  * The second member: struct ras_debug_if::op.
417  * It has three kinds of operations.
418  *
419  * - 0: disable RAS on the block. Take ::head as its data.
420  * - 1: enable RAS on the block. Take ::head as its data.
421  * - 2: inject errors on the block. Take ::inject as its data.
422  *
423  * How to use the interface?
424  *
425  * In a program
426  *
427  * Copy the struct ras_debug_if in your code and initialize it.
428  * Write the struct to the control interface.
429  *
430  * From shell
431  *
432  * .. code-block:: bash
433  *
434  *	echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
435  *	echo "enable  <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
436  *	echo "inject  <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
437  *
438  * Where N, is the card which you want to affect.
439  *
440  * "disable" requires only the block.
441  * "enable" requires the block and error type.
442  * "inject" requires the block, error type, address, and value.
443  *
444  * The block is one of: umc, sdma, gfx, etc.
445  *	see ras_block_string[] for details
446  *
447  * The error type is one of: ue, ce and poison where,
448  *	ue is multi-uncorrectable
449  *	ce is single-correctable
450  *	poison is poison
451  *
452  * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
453  * The address and value are hexadecimal numbers, leading 0x is optional.
454  * The mask means instance mask, is optional, default value is 0x1.
455  *
456  * For instance,
457  *
458  * .. code-block:: bash
459  *
460  *	echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
461  *	echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
462  *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
463  *
464  * How to check the result of the operation?
465  *
466  * To check disable/enable, see "ras" features at,
467  * /sys/class/drm/card[0/1/2...]/device/ras/features
468  *
469  * To check inject, see the corresponding error count at,
470  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
471  *
472  * .. note::
473  *	Operations are only allowed on blocks which are supported.
474  *	Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
475  *	to see which blocks support RAS on a particular asic.
476  *
477  */
amdgpu_ras_debugfs_ctrl_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)478 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
479 					     const char __user *buf,
480 					     size_t size, loff_t *pos)
481 {
482 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
483 	struct ras_debug_if data;
484 	int ret = 0;
485 
486 	if (!amdgpu_ras_get_error_query_ready(adev)) {
487 		dev_warn(adev->dev, "RAS WARN: error injection "
488 				"currently inaccessible\n");
489 		return size;
490 	}
491 
492 	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
493 	if (ret)
494 		return ret;
495 
496 	if (data.op == 3) {
497 		ret = amdgpu_reserve_page_direct(adev, data.inject.address);
498 		if (!ret)
499 			return size;
500 		else
501 			return ret;
502 	}
503 
504 	if (!amdgpu_ras_is_supported(adev, data.head.block))
505 		return -EINVAL;
506 
507 	switch (data.op) {
508 	case 0:
509 		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
510 		break;
511 	case 1:
512 		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
513 		break;
514 	case 2:
515 		if ((data.inject.address >= adev->gmc.mc_vram_size &&
516 		    adev->gmc.mc_vram_size) ||
517 		    (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
518 			dev_warn(adev->dev, "RAS WARN: input address "
519 					"0x%llx is invalid.",
520 					data.inject.address);
521 			ret = -EINVAL;
522 			break;
523 		}
524 
525 		/* umc ce/ue error injection for a bad page is not allowed */
526 		if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
527 		    amdgpu_ras_check_bad_page(adev, data.inject.address)) {
528 			dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
529 				 "already been marked as bad!\n",
530 				 data.inject.address);
531 			break;
532 		}
533 
534 		amdgpu_ras_instance_mask_check(adev, &data);
535 
536 		/* data.inject.address is offset instead of absolute gpu address */
537 		ret = amdgpu_ras_error_inject(adev, &data.inject);
538 		break;
539 	default:
540 		ret = -EINVAL;
541 		break;
542 	}
543 
544 	if (ret)
545 		return ret;
546 
547 	return size;
548 }
549 
550 /**
551  * DOC: AMDGPU RAS debugfs EEPROM table reset interface
552  *
553  * Some boards contain an EEPROM which is used to persistently store a list of
554  * bad pages which experiences ECC errors in vram.  This interface provides
555  * a way to reset the EEPROM, e.g., after testing error injection.
556  *
557  * Usage:
558  *
559  * .. code-block:: bash
560  *
561  *	echo 1 > ../ras/ras_eeprom_reset
562  *
563  * will reset EEPROM table to 0 entries.
564  *
565  */
amdgpu_ras_debugfs_eeprom_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)566 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
567 					       const char __user *buf,
568 					       size_t size, loff_t *pos)
569 {
570 	struct amdgpu_device *adev =
571 		(struct amdgpu_device *)file_inode(f)->i_private;
572 	int ret;
573 
574 	ret = amdgpu_ras_eeprom_reset_table(
575 		&(amdgpu_ras_get_context(adev)->eeprom_control));
576 
577 	if (!ret) {
578 		/* Something was written to EEPROM.
579 		 */
580 		amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
581 		return size;
582 	} else {
583 		return ret;
584 	}
585 }
586 
587 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
588 	.owner = THIS_MODULE,
589 	.read = NULL,
590 	.write = amdgpu_ras_debugfs_ctrl_write,
591 	.llseek = default_llseek
592 };
593 
594 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
595 	.owner = THIS_MODULE,
596 	.read = NULL,
597 	.write = amdgpu_ras_debugfs_eeprom_write,
598 	.llseek = default_llseek
599 };
600 
601 /**
602  * DOC: AMDGPU RAS sysfs Error Count Interface
603  *
604  * It allows the user to read the error count for each IP block on the gpu through
605  * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
606  *
607  * It outputs the multiple lines which report the uncorrected (ue) and corrected
608  * (ce) error counts.
609  *
610  * The format of one line is below,
611  *
612  * [ce|ue]: count
613  *
614  * Example:
615  *
616  * .. code-block:: bash
617  *
618  *	ue: 0
619  *	ce: 1
620  *
621  */
amdgpu_ras_sysfs_read(struct device * dev,struct device_attribute * attr,char * buf)622 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
623 		struct device_attribute *attr, char *buf)
624 {
625 	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
626 	struct ras_query_if info = {
627 		.head = obj->head,
628 	};
629 
630 	if (!amdgpu_ras_get_error_query_ready(obj->adev))
631 		return sysfs_emit(buf, "Query currently inaccessible\n");
632 
633 	if (amdgpu_ras_query_error_status(obj->adev, &info))
634 		return -EINVAL;
635 
636 	if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
637 	    amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
638 		if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
639 			dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
640 	}
641 
642 	if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
643 		return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
644 				"ce", info.ce_count, "de", info.de_count);
645 	else
646 		return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
647 				"ce", info.ce_count);
648 }
649 
650 /* obj begin */
651 
652 #define get_obj(obj) do { (obj)->use++; } while (0)
653 #define alive_obj(obj) ((obj)->use)
654 
put_obj(struct ras_manager * obj)655 static inline void put_obj(struct ras_manager *obj)
656 {
657 	if (obj && (--obj->use == 0)) {
658 		list_del(&obj->node);
659 		amdgpu_ras_error_data_fini(&obj->err_data);
660 	}
661 
662 	if (obj && (obj->use < 0))
663 		DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
664 }
665 
666 /* make one obj and return it. */
amdgpu_ras_create_obj(struct amdgpu_device * adev,struct ras_common_if * head)667 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
668 		struct ras_common_if *head)
669 {
670 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
671 	struct ras_manager *obj;
672 
673 	if (!adev->ras_enabled || !con)
674 		return NULL;
675 
676 	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
677 		return NULL;
678 
679 	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
680 		if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
681 			return NULL;
682 
683 		obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
684 	} else
685 		obj = &con->objs[head->block];
686 
687 	/* already exist. return obj? */
688 	if (alive_obj(obj))
689 		return NULL;
690 
691 	if (amdgpu_ras_error_data_init(&obj->err_data))
692 		return NULL;
693 
694 	obj->head = *head;
695 	obj->adev = adev;
696 	list_add(&obj->node, &con->head);
697 	get_obj(obj);
698 
699 	return obj;
700 }
701 
702 /* return an obj equal to head, or the first when head is NULL */
amdgpu_ras_find_obj(struct amdgpu_device * adev,struct ras_common_if * head)703 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
704 		struct ras_common_if *head)
705 {
706 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
707 	struct ras_manager *obj;
708 	int i;
709 
710 	if (!adev->ras_enabled || !con)
711 		return NULL;
712 
713 	if (head) {
714 		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
715 			return NULL;
716 
717 		if (head->block == AMDGPU_RAS_BLOCK__MCA) {
718 			if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
719 				return NULL;
720 
721 			obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
722 		} else
723 			obj = &con->objs[head->block];
724 
725 		if (alive_obj(obj))
726 			return obj;
727 	} else {
728 		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
729 			obj = &con->objs[i];
730 			if (alive_obj(obj))
731 				return obj;
732 		}
733 	}
734 
735 	return NULL;
736 }
737 /* obj end */
738 
739 /* feature ctl begin */
amdgpu_ras_is_feature_allowed(struct amdgpu_device * adev,struct ras_common_if * head)740 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
741 					 struct ras_common_if *head)
742 {
743 	return adev->ras_hw_enabled & BIT(head->block);
744 }
745 
amdgpu_ras_is_feature_enabled(struct amdgpu_device * adev,struct ras_common_if * head)746 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
747 		struct ras_common_if *head)
748 {
749 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
750 
751 	return con->features & BIT(head->block);
752 }
753 
754 /*
755  * if obj is not created, then create one.
756  * set feature enable flag.
757  */
__amdgpu_ras_feature_enable(struct amdgpu_device * adev,struct ras_common_if * head,int enable)758 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
759 		struct ras_common_if *head, int enable)
760 {
761 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
762 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
763 
764 	/* If hardware does not support ras, then do not create obj.
765 	 * But if hardware support ras, we can create the obj.
766 	 * Ras framework checks con->hw_supported to see if it need do
767 	 * corresponding initialization.
768 	 * IP checks con->support to see if it need disable ras.
769 	 */
770 	if (!amdgpu_ras_is_feature_allowed(adev, head))
771 		return 0;
772 
773 	if (enable) {
774 		if (!obj) {
775 			obj = amdgpu_ras_create_obj(adev, head);
776 			if (!obj)
777 				return -EINVAL;
778 		} else {
779 			/* In case we create obj somewhere else */
780 			get_obj(obj);
781 		}
782 		con->features |= BIT(head->block);
783 	} else {
784 		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
785 			con->features &= ~BIT(head->block);
786 			put_obj(obj);
787 		}
788 	}
789 
790 	return 0;
791 }
792 
793 /* wrapper of psp_ras_enable_features */
amdgpu_ras_feature_enable(struct amdgpu_device * adev,struct ras_common_if * head,bool enable)794 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
795 		struct ras_common_if *head, bool enable)
796 {
797 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
798 	union ta_ras_cmd_input *info;
799 	int ret;
800 
801 	if (!con)
802 		return -EINVAL;
803 
804 	/* For non-gfx ip, do not enable ras feature if it is not allowed */
805 	/* For gfx ip, regardless of feature support status, */
806 	/* Force issue enable or disable ras feature commands */
807 	if (head->block != AMDGPU_RAS_BLOCK__GFX &&
808 	    !amdgpu_ras_is_feature_allowed(adev, head))
809 		return 0;
810 
811 	/* Only enable gfx ras feature from host side */
812 	if (head->block == AMDGPU_RAS_BLOCK__GFX &&
813 	    !amdgpu_sriov_vf(adev) &&
814 	    !amdgpu_ras_intr_triggered()) {
815 		info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
816 		if (!info)
817 			return -ENOMEM;
818 
819 		if (!enable) {
820 			info->disable_features = (struct ta_ras_disable_features_input) {
821 				.block_id =  amdgpu_ras_block_to_ta(head->block),
822 				.error_type = amdgpu_ras_error_to_ta(head->type),
823 			};
824 		} else {
825 			info->enable_features = (struct ta_ras_enable_features_input) {
826 				.block_id =  amdgpu_ras_block_to_ta(head->block),
827 				.error_type = amdgpu_ras_error_to_ta(head->type),
828 			};
829 		}
830 
831 		ret = psp_ras_enable_features(&adev->psp, info, enable);
832 		if (ret) {
833 			dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
834 				enable ? "enable":"disable",
835 				get_ras_block_str(head),
836 				amdgpu_ras_is_poison_mode_supported(adev), ret);
837 			kfree(info);
838 			return ret;
839 		}
840 
841 		kfree(info);
842 	}
843 
844 	/* setup the obj */
845 	__amdgpu_ras_feature_enable(adev, head, enable);
846 
847 	return 0;
848 }
849 
850 /* Only used in device probe stage and called only once. */
amdgpu_ras_feature_enable_on_boot(struct amdgpu_device * adev,struct ras_common_if * head,bool enable)851 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
852 		struct ras_common_if *head, bool enable)
853 {
854 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
855 	int ret;
856 
857 	if (!con)
858 		return -EINVAL;
859 
860 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
861 		if (enable) {
862 			/* There is no harm to issue a ras TA cmd regardless of
863 			 * the currecnt ras state.
864 			 * If current state == target state, it will do nothing
865 			 * But sometimes it requests driver to reset and repost
866 			 * with error code -EAGAIN.
867 			 */
868 			ret = amdgpu_ras_feature_enable(adev, head, 1);
869 			/* With old ras TA, we might fail to enable ras.
870 			 * Log it and just setup the object.
871 			 * TODO need remove this WA in the future.
872 			 */
873 			if (ret == -EINVAL) {
874 				ret = __amdgpu_ras_feature_enable(adev, head, 1);
875 				if (!ret)
876 					dev_info(adev->dev,
877 						"RAS INFO: %s setup object\n",
878 						get_ras_block_str(head));
879 			}
880 		} else {
881 			/* setup the object then issue a ras TA disable cmd.*/
882 			ret = __amdgpu_ras_feature_enable(adev, head, 1);
883 			if (ret)
884 				return ret;
885 
886 			/* gfx block ras disable cmd must send to ras-ta */
887 			if (head->block == AMDGPU_RAS_BLOCK__GFX)
888 				con->features |= BIT(head->block);
889 
890 			ret = amdgpu_ras_feature_enable(adev, head, 0);
891 
892 			/* clean gfx block ras features flag */
893 			if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
894 				con->features &= ~BIT(head->block);
895 		}
896 	} else
897 		ret = amdgpu_ras_feature_enable(adev, head, enable);
898 
899 	return ret;
900 }
901 
amdgpu_ras_disable_all_features(struct amdgpu_device * adev,bool bypass)902 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
903 		bool bypass)
904 {
905 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
906 	struct ras_manager *obj, *tmp;
907 
908 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
909 		/* bypass psp.
910 		 * aka just release the obj and corresponding flags
911 		 */
912 		if (bypass) {
913 			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
914 				break;
915 		} else {
916 			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
917 				break;
918 		}
919 	}
920 
921 	return con->features;
922 }
923 
amdgpu_ras_enable_all_features(struct amdgpu_device * adev,bool bypass)924 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
925 		bool bypass)
926 {
927 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
928 	int i;
929 	const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
930 
931 	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
932 		struct ras_common_if head = {
933 			.block = i,
934 			.type = default_ras_type,
935 			.sub_block_index = 0,
936 		};
937 
938 		if (i == AMDGPU_RAS_BLOCK__MCA)
939 			continue;
940 
941 		if (bypass) {
942 			/*
943 			 * bypass psp. vbios enable ras for us.
944 			 * so just create the obj
945 			 */
946 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
947 				break;
948 		} else {
949 			if (amdgpu_ras_feature_enable(adev, &head, 1))
950 				break;
951 		}
952 	}
953 
954 	for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
955 		struct ras_common_if head = {
956 			.block = AMDGPU_RAS_BLOCK__MCA,
957 			.type = default_ras_type,
958 			.sub_block_index = i,
959 		};
960 
961 		if (bypass) {
962 			/*
963 			 * bypass psp. vbios enable ras for us.
964 			 * so just create the obj
965 			 */
966 			if (__amdgpu_ras_feature_enable(adev, &head, 1))
967 				break;
968 		} else {
969 			if (amdgpu_ras_feature_enable(adev, &head, 1))
970 				break;
971 		}
972 	}
973 
974 	return con->features;
975 }
976 /* feature ctl end */
977 
amdgpu_ras_block_match_default(struct amdgpu_ras_block_object * block_obj,enum amdgpu_ras_block block)978 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
979 		enum amdgpu_ras_block block)
980 {
981 	if (!block_obj)
982 		return -EINVAL;
983 
984 	if (block_obj->ras_comm.block == block)
985 		return 0;
986 
987 	return -EINVAL;
988 }
989 
amdgpu_ras_get_ras_block(struct amdgpu_device * adev,enum amdgpu_ras_block block,uint32_t sub_block_index)990 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
991 					enum amdgpu_ras_block block, uint32_t sub_block_index)
992 {
993 	struct amdgpu_ras_block_list *node, *tmp;
994 	struct amdgpu_ras_block_object *obj;
995 
996 	if (block >= AMDGPU_RAS_BLOCK__LAST)
997 		return NULL;
998 
999 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1000 		if (!node->ras_obj) {
1001 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1002 			continue;
1003 		}
1004 
1005 		obj = node->ras_obj;
1006 		if (obj->ras_block_match) {
1007 			if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1008 				return obj;
1009 		} else {
1010 			if (amdgpu_ras_block_match_default(obj, block) == 0)
1011 				return obj;
1012 		}
1013 	}
1014 
1015 	return NULL;
1016 }
1017 
amdgpu_ras_get_ecc_info(struct amdgpu_device * adev,struct ras_err_data * err_data)1018 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1019 {
1020 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1021 	int ret = 0;
1022 
1023 	/*
1024 	 * choosing right query method according to
1025 	 * whether smu support query error information
1026 	 */
1027 	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
1028 	if (ret == -EOPNOTSUPP) {
1029 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1030 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1031 			adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1032 
1033 		/* umc query_ras_error_address is also responsible for clearing
1034 		 * error status
1035 		 */
1036 		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1037 		    adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1038 			adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1039 	} else if (!ret) {
1040 		if (adev->umc.ras &&
1041 			adev->umc.ras->ecc_info_query_ras_error_count)
1042 			adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1043 
1044 		if (adev->umc.ras &&
1045 			adev->umc.ras->ecc_info_query_ras_error_address)
1046 			adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1047 	}
1048 }
1049 
amdgpu_ras_error_print_error_data(struct amdgpu_device * adev,struct ras_manager * ras_mgr,struct ras_err_data * err_data,struct ras_query_context * qctx,const char * blk_name,bool is_ue,bool is_de)1050 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1051 					      struct ras_manager *ras_mgr,
1052 					      struct ras_err_data *err_data,
1053 					      struct ras_query_context *qctx,
1054 					      const char *blk_name,
1055 					      bool is_ue,
1056 					      bool is_de)
1057 {
1058 	struct amdgpu_smuio_mcm_config_info *mcm_info;
1059 	struct ras_err_node *err_node;
1060 	struct ras_err_info *err_info;
1061 	u64 event_id = qctx->evid.event_id;
1062 
1063 	if (is_ue) {
1064 		for_each_ras_error(err_node, err_data) {
1065 			err_info = &err_node->err_info;
1066 			mcm_info = &err_info->mcm_info;
1067 			if (err_info->ue_count) {
1068 				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1069 					      "%lld new uncorrectable hardware errors detected in %s block\n",
1070 					      mcm_info->socket_id,
1071 					      mcm_info->die_id,
1072 					      err_info->ue_count,
1073 					      blk_name);
1074 			}
1075 		}
1076 
1077 		for_each_ras_error(err_node, &ras_mgr->err_data) {
1078 			err_info = &err_node->err_info;
1079 			mcm_info = &err_info->mcm_info;
1080 			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1081 				      "%lld uncorrectable hardware errors detected in total in %s block\n",
1082 				      mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1083 		}
1084 
1085 	} else {
1086 		if (is_de) {
1087 			for_each_ras_error(err_node, err_data) {
1088 				err_info = &err_node->err_info;
1089 				mcm_info = &err_info->mcm_info;
1090 				if (err_info->de_count) {
1091 					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1092 						      "%lld new deferred hardware errors detected in %s block\n",
1093 						      mcm_info->socket_id,
1094 						      mcm_info->die_id,
1095 						      err_info->de_count,
1096 						      blk_name);
1097 				}
1098 			}
1099 
1100 			for_each_ras_error(err_node, &ras_mgr->err_data) {
1101 				err_info = &err_node->err_info;
1102 				mcm_info = &err_info->mcm_info;
1103 				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1104 					      "%lld deferred hardware errors detected in total in %s block\n",
1105 					      mcm_info->socket_id, mcm_info->die_id,
1106 					      err_info->de_count, blk_name);
1107 			}
1108 		} else {
1109 			for_each_ras_error(err_node, err_data) {
1110 				err_info = &err_node->err_info;
1111 				mcm_info = &err_info->mcm_info;
1112 				if (err_info->ce_count) {
1113 					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1114 						      "%lld new correctable hardware errors detected in %s block\n",
1115 						      mcm_info->socket_id,
1116 						      mcm_info->die_id,
1117 						      err_info->ce_count,
1118 						      blk_name);
1119 				}
1120 			}
1121 
1122 			for_each_ras_error(err_node, &ras_mgr->err_data) {
1123 				err_info = &err_node->err_info;
1124 				mcm_info = &err_info->mcm_info;
1125 				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1126 					      "%lld correctable hardware errors detected in total in %s block\n",
1127 					      mcm_info->socket_id, mcm_info->die_id,
1128 					      err_info->ce_count, blk_name);
1129 			}
1130 		}
1131 	}
1132 }
1133 
err_data_has_source_info(struct ras_err_data * data)1134 static inline bool err_data_has_source_info(struct ras_err_data *data)
1135 {
1136 	return !list_empty(&data->err_node_list);
1137 }
1138 
amdgpu_ras_error_generate_report(struct amdgpu_device * adev,struct ras_query_if * query_if,struct ras_err_data * err_data,struct ras_query_context * qctx)1139 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1140 					     struct ras_query_if *query_if,
1141 					     struct ras_err_data *err_data,
1142 					     struct ras_query_context *qctx)
1143 {
1144 	struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1145 	const char *blk_name = get_ras_block_str(&query_if->head);
1146 	u64 event_id = qctx->evid.event_id;
1147 
1148 	if (err_data->ce_count) {
1149 		if (err_data_has_source_info(err_data)) {
1150 			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1151 							  blk_name, false, false);
1152 		} else if (!adev->aid_mask &&
1153 			   adev->smuio.funcs &&
1154 			   adev->smuio.funcs->get_socket_id &&
1155 			   adev->smuio.funcs->get_die_id) {
1156 			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1157 				      "%ld correctable hardware errors "
1158 				      "detected in %s block\n",
1159 				      adev->smuio.funcs->get_socket_id(adev),
1160 				      adev->smuio.funcs->get_die_id(adev),
1161 				      ras_mgr->err_data.ce_count,
1162 				      blk_name);
1163 		} else {
1164 			RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1165 				      "detected in %s block\n",
1166 				      ras_mgr->err_data.ce_count,
1167 				      blk_name);
1168 		}
1169 	}
1170 
1171 	if (err_data->ue_count) {
1172 		if (err_data_has_source_info(err_data)) {
1173 			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1174 							  blk_name, true, false);
1175 		} else if (!adev->aid_mask &&
1176 			   adev->smuio.funcs &&
1177 			   adev->smuio.funcs->get_socket_id &&
1178 			   adev->smuio.funcs->get_die_id) {
1179 			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1180 				      "%ld uncorrectable hardware errors "
1181 				      "detected in %s block\n",
1182 				      adev->smuio.funcs->get_socket_id(adev),
1183 				      adev->smuio.funcs->get_die_id(adev),
1184 				      ras_mgr->err_data.ue_count,
1185 				      blk_name);
1186 		} else {
1187 			RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1188 				      "detected in %s block\n",
1189 				      ras_mgr->err_data.ue_count,
1190 				      blk_name);
1191 		}
1192 	}
1193 
1194 	if (err_data->de_count) {
1195 		if (err_data_has_source_info(err_data)) {
1196 			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1197 							  blk_name, false, true);
1198 		} else if (!adev->aid_mask &&
1199 			   adev->smuio.funcs &&
1200 			   adev->smuio.funcs->get_socket_id &&
1201 			   adev->smuio.funcs->get_die_id) {
1202 			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1203 				      "%ld deferred hardware errors "
1204 				      "detected in %s block\n",
1205 				      adev->smuio.funcs->get_socket_id(adev),
1206 				      adev->smuio.funcs->get_die_id(adev),
1207 				      ras_mgr->err_data.de_count,
1208 				      blk_name);
1209 		} else {
1210 			RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1211 				      "detected in %s block\n",
1212 				      ras_mgr->err_data.de_count,
1213 				      blk_name);
1214 		}
1215 	}
1216 }
1217 
amdgpu_ras_virt_error_generate_report(struct amdgpu_device * adev,struct ras_query_if * query_if,struct ras_err_data * err_data,struct ras_query_context * qctx)1218 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1219 						  struct ras_query_if *query_if,
1220 						  struct ras_err_data *err_data,
1221 						  struct ras_query_context *qctx)
1222 {
1223 	unsigned long new_ue, new_ce, new_de;
1224 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1225 	const char *blk_name = get_ras_block_str(&query_if->head);
1226 	u64 event_id = qctx->evid.event_id;
1227 
1228 	new_ce = err_data->ce_count - obj->err_data.ce_count;
1229 	new_ue = err_data->ue_count - obj->err_data.ue_count;
1230 	new_de = err_data->de_count - obj->err_data.de_count;
1231 
1232 	if (new_ce) {
1233 		RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1234 			      "detected in %s block\n",
1235 			      new_ce,
1236 			      blk_name);
1237 	}
1238 
1239 	if (new_ue) {
1240 		RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1241 			      "detected in %s block\n",
1242 			      new_ue,
1243 			      blk_name);
1244 	}
1245 
1246 	if (new_de) {
1247 		RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1248 			      "detected in %s block\n",
1249 			      new_de,
1250 			      blk_name);
1251 	}
1252 }
1253 
amdgpu_rasmgr_error_data_statistic_update(struct ras_manager * obj,struct ras_err_data * err_data)1254 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1255 {
1256 	struct ras_err_node *err_node;
1257 	struct ras_err_info *err_info;
1258 
1259 	if (err_data_has_source_info(err_data)) {
1260 		for_each_ras_error(err_node, err_data) {
1261 			err_info = &err_node->err_info;
1262 			amdgpu_ras_error_statistic_de_count(&obj->err_data,
1263 					&err_info->mcm_info, err_info->de_count);
1264 			amdgpu_ras_error_statistic_ce_count(&obj->err_data,
1265 					&err_info->mcm_info, err_info->ce_count);
1266 			amdgpu_ras_error_statistic_ue_count(&obj->err_data,
1267 					&err_info->mcm_info, err_info->ue_count);
1268 		}
1269 	} else {
1270 		/* for legacy asic path which doesn't has error source info */
1271 		obj->err_data.ue_count += err_data->ue_count;
1272 		obj->err_data.ce_count += err_data->ce_count;
1273 		obj->err_data.de_count += err_data->de_count;
1274 	}
1275 }
1276 
amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager * obj,struct ras_err_data * err_data)1277 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1278 							     struct ras_err_data *err_data)
1279 {
1280 	/* Host reports absolute counts */
1281 	obj->err_data.ue_count = err_data->ue_count;
1282 	obj->err_data.ce_count = err_data->ce_count;
1283 	obj->err_data.de_count = err_data->de_count;
1284 }
1285 
get_ras_manager(struct amdgpu_device * adev,enum amdgpu_ras_block blk)1286 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1287 {
1288 	struct ras_common_if head;
1289 
1290 	memset(&head, 0, sizeof(head));
1291 	head.block = blk;
1292 
1293 	return amdgpu_ras_find_obj(adev, &head);
1294 }
1295 
amdgpu_ras_bind_aca(struct amdgpu_device * adev,enum amdgpu_ras_block blk,const struct aca_info * aca_info,void * data)1296 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1297 			const struct aca_info *aca_info, void *data)
1298 {
1299 	struct ras_manager *obj;
1300 
1301 	/* in resume phase, no need to create aca fs node */
1302 	if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1303 		return 0;
1304 
1305 	obj = get_ras_manager(adev, blk);
1306 	if (!obj)
1307 		return -EINVAL;
1308 
1309 	return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1310 }
1311 
amdgpu_ras_unbind_aca(struct amdgpu_device * adev,enum amdgpu_ras_block blk)1312 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1313 {
1314 	struct ras_manager *obj;
1315 
1316 	obj = get_ras_manager(adev, blk);
1317 	if (!obj)
1318 		return -EINVAL;
1319 
1320 	amdgpu_aca_remove_handle(&obj->aca_handle);
1321 
1322 	return 0;
1323 }
1324 
amdgpu_aca_log_ras_error_data(struct amdgpu_device * adev,enum amdgpu_ras_block blk,enum aca_error_type type,struct ras_err_data * err_data,struct ras_query_context * qctx)1325 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1326 					 enum aca_error_type type, struct ras_err_data *err_data,
1327 					 struct ras_query_context *qctx)
1328 {
1329 	struct ras_manager *obj;
1330 
1331 	obj = get_ras_manager(adev, blk);
1332 	if (!obj)
1333 		return -EINVAL;
1334 
1335 	return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
1336 }
1337 
amdgpu_ras_aca_sysfs_read(struct device * dev,struct device_attribute * attr,struct aca_handle * handle,char * buf,void * data)1338 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1339 				  struct aca_handle *handle, char *buf, void *data)
1340 {
1341 	struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1342 	struct ras_query_if info = {
1343 		.head = obj->head,
1344 	};
1345 
1346 	if (!amdgpu_ras_get_error_query_ready(obj->adev))
1347 		return sysfs_emit(buf, "Query currently inaccessible\n");
1348 
1349 	if (amdgpu_ras_query_error_status(obj->adev, &info))
1350 		return -EINVAL;
1351 
1352 	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1353 			  "ce", info.ce_count, "de", info.de_count);
1354 }
1355 
amdgpu_ras_query_error_status_helper(struct amdgpu_device * adev,struct ras_query_if * info,struct ras_err_data * err_data,struct ras_query_context * qctx,unsigned int error_query_mode)1356 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1357 						struct ras_query_if *info,
1358 						struct ras_err_data *err_data,
1359 						struct ras_query_context *qctx,
1360 						unsigned int error_query_mode)
1361 {
1362 	enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1363 	struct amdgpu_ras_block_object *block_obj = NULL;
1364 	int ret;
1365 
1366 	if (blk == AMDGPU_RAS_BLOCK_COUNT)
1367 		return -EINVAL;
1368 
1369 	if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1370 		return -EINVAL;
1371 
1372 	if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1373 		return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1374 	} else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1375 		if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1376 			amdgpu_ras_get_ecc_info(adev, err_data);
1377 		} else {
1378 			block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1379 			if (!block_obj || !block_obj->hw_ops) {
1380 				dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1381 					     get_ras_block_str(&info->head));
1382 				return -EINVAL;
1383 			}
1384 
1385 			if (block_obj->hw_ops->query_ras_error_count)
1386 				block_obj->hw_ops->query_ras_error_count(adev, err_data);
1387 
1388 			if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1389 			    (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1390 			    (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1391 				if (block_obj->hw_ops->query_ras_error_status)
1392 					block_obj->hw_ops->query_ras_error_status(adev);
1393 			}
1394 		}
1395 	} else {
1396 		if (amdgpu_aca_is_enabled(adev)) {
1397 			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
1398 			if (ret)
1399 				return ret;
1400 
1401 			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
1402 			if (ret)
1403 				return ret;
1404 
1405 			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1406 			if (ret)
1407 				return ret;
1408 		} else {
1409 			/* FIXME: add code to check return value later */
1410 			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1411 			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1412 		}
1413 	}
1414 
1415 	return 0;
1416 }
1417 
1418 /* query/inject/cure begin */
amdgpu_ras_query_error_status_with_event(struct amdgpu_device * adev,struct ras_query_if * info,enum ras_event_type type)1419 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1420 						    struct ras_query_if *info,
1421 						    enum ras_event_type type)
1422 {
1423 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1424 	struct ras_err_data err_data;
1425 	struct ras_query_context qctx;
1426 	unsigned int error_query_mode;
1427 	int ret;
1428 
1429 	if (!obj)
1430 		return -EINVAL;
1431 
1432 	ret = amdgpu_ras_error_data_init(&err_data);
1433 	if (ret)
1434 		return ret;
1435 
1436 	if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1437 		return -EINVAL;
1438 
1439 	memset(&qctx, 0, sizeof(qctx));
1440 	qctx.evid.type = type;
1441 	qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1442 
1443 	if (!down_read_trylock(&adev->reset_domain->sem)) {
1444 		ret = -EIO;
1445 		goto out_fini_err_data;
1446 	}
1447 
1448 	ret = amdgpu_ras_query_error_status_helper(adev, info,
1449 						   &err_data,
1450 						   &qctx,
1451 						   error_query_mode);
1452 	up_read(&adev->reset_domain->sem);
1453 	if (ret)
1454 		goto out_fini_err_data;
1455 
1456 	if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1457 		amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1458 		amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1459 	} else {
1460 		/* Host provides absolute error counts. First generate the report
1461 		 * using the previous VF internal count against new host count.
1462 		 * Then Update VF internal count.
1463 		 */
1464 		amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1465 		amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1466 	}
1467 
1468 	info->ue_count = obj->err_data.ue_count;
1469 	info->ce_count = obj->err_data.ce_count;
1470 	info->de_count = obj->err_data.de_count;
1471 
1472 out_fini_err_data:
1473 	amdgpu_ras_error_data_fini(&err_data);
1474 
1475 	return ret;
1476 }
1477 
amdgpu_ras_query_error_status(struct amdgpu_device * adev,struct ras_query_if * info)1478 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1479 {
1480 	return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1481 }
1482 
amdgpu_ras_reset_error_count(struct amdgpu_device * adev,enum amdgpu_ras_block block)1483 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1484 		enum amdgpu_ras_block block)
1485 {
1486 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1487 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1488 	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1489 
1490 	if (!block_obj || !block_obj->hw_ops) {
1491 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1492 				ras_block_str(block));
1493 		return -EOPNOTSUPP;
1494 	}
1495 
1496 	if (!amdgpu_ras_is_supported(adev, block) ||
1497 	    !amdgpu_ras_get_aca_debug_mode(adev))
1498 		return -EOPNOTSUPP;
1499 
1500 	/* skip ras error reset in gpu reset */
1501 	if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1502 	    ((smu_funcs && smu_funcs->set_debug_mode) ||
1503 	     (mca_funcs && mca_funcs->mca_set_debug_mode)))
1504 		return -EOPNOTSUPP;
1505 
1506 	if (block_obj->hw_ops->reset_ras_error_count)
1507 		block_obj->hw_ops->reset_ras_error_count(adev);
1508 
1509 	return 0;
1510 }
1511 
amdgpu_ras_reset_error_status(struct amdgpu_device * adev,enum amdgpu_ras_block block)1512 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1513 		enum amdgpu_ras_block block)
1514 {
1515 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1516 
1517 	if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1518 		return 0;
1519 
1520 	if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1521 	    (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1522 		if (block_obj->hw_ops->reset_ras_error_status)
1523 			block_obj->hw_ops->reset_ras_error_status(adev);
1524 	}
1525 
1526 	return 0;
1527 }
1528 
1529 /* wrapper of psp_ras_trigger_error */
amdgpu_ras_error_inject(struct amdgpu_device * adev,struct ras_inject_if * info)1530 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1531 		struct ras_inject_if *info)
1532 {
1533 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1534 	struct ta_ras_trigger_error_input block_info = {
1535 		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
1536 		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1537 		.sub_block_index = info->head.sub_block_index,
1538 		.address = info->address,
1539 		.value = info->value,
1540 	};
1541 	int ret = -EINVAL;
1542 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1543 							info->head.block,
1544 							info->head.sub_block_index);
1545 
1546 	/* inject on guest isn't allowed, return success directly */
1547 	if (amdgpu_sriov_vf(adev))
1548 		return 0;
1549 
1550 	if (!obj)
1551 		return -EINVAL;
1552 
1553 	if (!block_obj || !block_obj->hw_ops)	{
1554 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1555 			     get_ras_block_str(&info->head));
1556 		return -EINVAL;
1557 	}
1558 
1559 	/* Calculate XGMI relative offset */
1560 	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1561 	    info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1562 		block_info.address =
1563 			amdgpu_xgmi_get_relative_phy_addr(adev,
1564 							  block_info.address);
1565 	}
1566 
1567 	if (block_obj->hw_ops->ras_error_inject) {
1568 		if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1569 			ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1570 		else /* Special ras_error_inject is defined (e.g: xgmi) */
1571 			ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1572 						info->instance_mask);
1573 	} else {
1574 		/* default path */
1575 		ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1576 	}
1577 
1578 	if (ret)
1579 		dev_err(adev->dev, "ras inject %s failed %d\n",
1580 			get_ras_block_str(&info->head), ret);
1581 
1582 	return ret;
1583 }
1584 
1585 /**
1586  * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1587  * @adev: pointer to AMD GPU device
1588  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1589  * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1590  * @query_info: pointer to ras_query_if
1591  *
1592  * Return 0 for query success or do nothing, otherwise return an error
1593  * on failures
1594  */
amdgpu_ras_query_error_count_helper(struct amdgpu_device * adev,unsigned long * ce_count,unsigned long * ue_count,struct ras_query_if * query_info)1595 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1596 					       unsigned long *ce_count,
1597 					       unsigned long *ue_count,
1598 					       struct ras_query_if *query_info)
1599 {
1600 	int ret;
1601 
1602 	if (!query_info)
1603 		/* do nothing if query_info is not specified */
1604 		return 0;
1605 
1606 	ret = amdgpu_ras_query_error_status(adev, query_info);
1607 	if (ret)
1608 		return ret;
1609 
1610 	*ce_count += query_info->ce_count;
1611 	*ue_count += query_info->ue_count;
1612 
1613 	/* some hardware/IP supports read to clear
1614 	 * no need to explictly reset the err status after the query call */
1615 	if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1616 	    amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1617 		if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1618 			dev_warn(adev->dev,
1619 				 "Failed to reset error counter and error status\n");
1620 	}
1621 
1622 	return 0;
1623 }
1624 
1625 /**
1626  * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1627  * @adev: pointer to AMD GPU device
1628  * @ce_count: pointer to an integer to be set to the count of correctible errors.
1629  * @ue_count: pointer to an integer to be set to the count of uncorrectible
1630  * errors.
1631  * @query_info: pointer to ras_query_if if the query request is only for
1632  * specific ip block; if info is NULL, then the qurey request is for
1633  * all the ip blocks that support query ras error counters/status
1634  *
1635  * If set, @ce_count or @ue_count, count and return the corresponding
1636  * error counts in those integer pointers. Return 0 if the device
1637  * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1638  */
amdgpu_ras_query_error_count(struct amdgpu_device * adev,unsigned long * ce_count,unsigned long * ue_count,struct ras_query_if * query_info)1639 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1640 				 unsigned long *ce_count,
1641 				 unsigned long *ue_count,
1642 				 struct ras_query_if *query_info)
1643 {
1644 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1645 	struct ras_manager *obj;
1646 	unsigned long ce, ue;
1647 	int ret;
1648 
1649 	if (!adev->ras_enabled || !con)
1650 		return -EOPNOTSUPP;
1651 
1652 	/* Don't count since no reporting.
1653 	 */
1654 	if (!ce_count && !ue_count)
1655 		return 0;
1656 
1657 	ce = 0;
1658 	ue = 0;
1659 	if (!query_info) {
1660 		/* query all the ip blocks that support ras query interface */
1661 		list_for_each_entry(obj, &con->head, node) {
1662 			struct ras_query_if info = {
1663 				.head = obj->head,
1664 			};
1665 
1666 			ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1667 		}
1668 	} else {
1669 		/* query specific ip block */
1670 		ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1671 	}
1672 
1673 	if (ret)
1674 		return ret;
1675 
1676 	if (ce_count)
1677 		*ce_count = ce;
1678 
1679 	if (ue_count)
1680 		*ue_count = ue;
1681 
1682 	return 0;
1683 }
1684 /* query/inject/cure end */
1685 
1686 
1687 /* sysfs begin */
1688 
1689 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1690 		struct ras_badpage **bps, unsigned int *count);
1691 
amdgpu_ras_badpage_flags_str(unsigned int flags)1692 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1693 {
1694 	switch (flags) {
1695 	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1696 		return "R";
1697 	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1698 		return "P";
1699 	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1700 	default:
1701 		return "F";
1702 	}
1703 }
1704 
1705 /**
1706  * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1707  *
1708  * It allows user to read the bad pages of vram on the gpu through
1709  * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1710  *
1711  * It outputs multiple lines, and each line stands for one gpu page.
1712  *
1713  * The format of one line is below,
1714  * gpu pfn : gpu page size : flags
1715  *
1716  * gpu pfn and gpu page size are printed in hex format.
1717  * flags can be one of below character,
1718  *
1719  * R: reserved, this gpu page is reserved and not able to use.
1720  *
1721  * P: pending for reserve, this gpu page is marked as bad, will be reserved
1722  * in next window of page_reserve.
1723  *
1724  * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1725  *
1726  * Examples:
1727  *
1728  * .. code-block:: bash
1729  *
1730  *	0x00000001 : 0x00001000 : R
1731  *	0x00000002 : 0x00001000 : P
1732  *
1733  */
1734 
amdgpu_ras_sysfs_badpages_read(struct file * f,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t ppos,size_t count)1735 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1736 		struct kobject *kobj, struct bin_attribute *attr,
1737 		char *buf, loff_t ppos, size_t count)
1738 {
1739 	struct amdgpu_ras *con =
1740 		container_of(attr, struct amdgpu_ras, badpages_attr);
1741 	struct amdgpu_device *adev = con->adev;
1742 	const unsigned int element_size =
1743 		sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1744 	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1745 	unsigned int end = div64_ul(ppos + count - 1, element_size);
1746 	ssize_t s = 0;
1747 	struct ras_badpage *bps = NULL;
1748 	unsigned int bps_count = 0;
1749 
1750 	memset(buf, 0, count);
1751 
1752 	if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1753 		return 0;
1754 
1755 	for (; start < end && start < bps_count; start++)
1756 		s += scnprintf(&buf[s], element_size + 1,
1757 				"0x%08x : 0x%08x : %1s\n",
1758 				bps[start].bp,
1759 				bps[start].size,
1760 				amdgpu_ras_badpage_flags_str(bps[start].flags));
1761 
1762 	kfree(bps);
1763 
1764 	return s;
1765 }
1766 
amdgpu_ras_sysfs_features_read(struct device * dev,struct device_attribute * attr,char * buf)1767 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1768 		struct device_attribute *attr, char *buf)
1769 {
1770 	struct amdgpu_ras *con =
1771 		container_of(attr, struct amdgpu_ras, features_attr);
1772 
1773 	return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1774 }
1775 
amdgpu_ras_sysfs_version_show(struct device * dev,struct device_attribute * attr,char * buf)1776 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1777 		struct device_attribute *attr, char *buf)
1778 {
1779 	struct amdgpu_ras *con =
1780 		container_of(attr, struct amdgpu_ras, version_attr);
1781 	return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1782 }
1783 
amdgpu_ras_sysfs_schema_show(struct device * dev,struct device_attribute * attr,char * buf)1784 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1785 		struct device_attribute *attr, char *buf)
1786 {
1787 	struct amdgpu_ras *con =
1788 		container_of(attr, struct amdgpu_ras, schema_attr);
1789 	return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1790 }
1791 
1792 static struct {
1793 	enum ras_event_type type;
1794 	const char *name;
1795 } dump_event[] = {
1796 	{RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1797 	{RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1798 	{RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1799 };
1800 
amdgpu_ras_sysfs_event_state_show(struct device * dev,struct device_attribute * attr,char * buf)1801 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1802 						 struct device_attribute *attr, char *buf)
1803 {
1804 	struct amdgpu_ras *con =
1805 		container_of(attr, struct amdgpu_ras, event_state_attr);
1806 	struct ras_event_manager *event_mgr = con->event_mgr;
1807 	struct ras_event_state *event_state;
1808 	int i, size = 0;
1809 
1810 	if (!event_mgr)
1811 		return -EINVAL;
1812 
1813 	size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1814 	for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1815 		event_state = &event_mgr->event_state[dump_event[i].type];
1816 		size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1817 				      dump_event[i].name,
1818 				      atomic64_read(&event_state->count),
1819 				      event_state->last_seqno);
1820 	}
1821 
1822 	return (ssize_t)size;
1823 }
1824 
amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device * adev)1825 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1826 {
1827 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1828 
1829 	if (adev->dev->kobj.sd)
1830 		sysfs_remove_file_from_group(&adev->dev->kobj,
1831 				&con->badpages_attr.attr,
1832 				RAS_FS_NAME);
1833 }
1834 
amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device * adev)1835 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
1836 {
1837 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1838 	struct attribute *attrs[] = {
1839 		&con->features_attr.attr,
1840 		&con->version_attr.attr,
1841 		&con->schema_attr.attr,
1842 		&con->event_state_attr.attr,
1843 		NULL
1844 	};
1845 	struct attribute_group group = {
1846 		.name = RAS_FS_NAME,
1847 		.attrs = attrs,
1848 	};
1849 
1850 	if (adev->dev->kobj.sd)
1851 		sysfs_remove_group(&adev->dev->kobj, &group);
1852 
1853 	return 0;
1854 }
1855 
amdgpu_ras_sysfs_create(struct amdgpu_device * adev,struct ras_common_if * head)1856 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1857 		struct ras_common_if *head)
1858 {
1859 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1860 
1861 	if (amdgpu_aca_is_enabled(adev))
1862 		return 0;
1863 
1864 	if (!obj || obj->attr_inuse)
1865 		return -EINVAL;
1866 
1867 	get_obj(obj);
1868 
1869 	snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1870 		"%s_err_count", head->name);
1871 
1872 	obj->sysfs_attr = (struct device_attribute){
1873 		.attr = {
1874 			.name = obj->fs_data.sysfs_name,
1875 			.mode = S_IRUGO,
1876 		},
1877 			.show = amdgpu_ras_sysfs_read,
1878 	};
1879 	sysfs_attr_init(&obj->sysfs_attr.attr);
1880 
1881 	if (sysfs_add_file_to_group(&adev->dev->kobj,
1882 				&obj->sysfs_attr.attr,
1883 				RAS_FS_NAME)) {
1884 		put_obj(obj);
1885 		return -EINVAL;
1886 	}
1887 
1888 	obj->attr_inuse = 1;
1889 
1890 	return 0;
1891 }
1892 
amdgpu_ras_sysfs_remove(struct amdgpu_device * adev,struct ras_common_if * head)1893 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1894 		struct ras_common_if *head)
1895 {
1896 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1897 
1898 	if (amdgpu_aca_is_enabled(adev))
1899 		return 0;
1900 
1901 	if (!obj || !obj->attr_inuse)
1902 		return -EINVAL;
1903 
1904 	if (adev->dev->kobj.sd)
1905 		sysfs_remove_file_from_group(&adev->dev->kobj,
1906 				&obj->sysfs_attr.attr,
1907 				RAS_FS_NAME);
1908 	obj->attr_inuse = 0;
1909 	put_obj(obj);
1910 
1911 	return 0;
1912 }
1913 
amdgpu_ras_sysfs_remove_all(struct amdgpu_device * adev)1914 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1915 {
1916 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1917 	struct ras_manager *obj, *tmp;
1918 
1919 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
1920 		amdgpu_ras_sysfs_remove(adev, &obj->head);
1921 	}
1922 
1923 	if (amdgpu_bad_page_threshold != 0)
1924 		amdgpu_ras_sysfs_remove_bad_page_node(adev);
1925 
1926 	amdgpu_ras_sysfs_remove_dev_attr_node(adev);
1927 
1928 	return 0;
1929 }
1930 /* sysfs end */
1931 
1932 /**
1933  * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1934  *
1935  * Normally when there is an uncorrectable error, the driver will reset
1936  * the GPU to recover.  However, in the event of an unrecoverable error,
1937  * the driver provides an interface to reboot the system automatically
1938  * in that event.
1939  *
1940  * The following file in debugfs provides that interface:
1941  * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1942  *
1943  * Usage:
1944  *
1945  * .. code-block:: bash
1946  *
1947  *	echo true > .../ras/auto_reboot
1948  *
1949  */
1950 /* debugfs begin */
amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device * adev)1951 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1952 {
1953 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1954 	struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1955 	struct drm_minor  *minor = adev_to_drm(adev)->primary;
1956 	struct dentry     *dir;
1957 
1958 	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1959 	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1960 			    &amdgpu_ras_debugfs_ctrl_ops);
1961 	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1962 			    &amdgpu_ras_debugfs_eeprom_ops);
1963 	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1964 			   &con->bad_page_cnt_threshold);
1965 	debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1966 	debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1967 	debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1968 	debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1969 			    &amdgpu_ras_debugfs_eeprom_size_ops);
1970 	con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1971 						       S_IRUGO, dir, adev,
1972 						       &amdgpu_ras_debugfs_eeprom_table_ops);
1973 	amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1974 
1975 	/*
1976 	 * After one uncorrectable error happens, usually GPU recovery will
1977 	 * be scheduled. But due to the known problem in GPU recovery failing
1978 	 * to bring GPU back, below interface provides one direct way to
1979 	 * user to reboot system automatically in such case within
1980 	 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1981 	 * will never be called.
1982 	 */
1983 	debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1984 
1985 	/*
1986 	 * User could set this not to clean up hardware's error count register
1987 	 * of RAS IPs during ras recovery.
1988 	 */
1989 	debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1990 			    &con->disable_ras_err_cnt_harvest);
1991 	return dir;
1992 }
1993 
amdgpu_ras_debugfs_create(struct amdgpu_device * adev,struct ras_fs_if * head,struct dentry * dir)1994 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1995 				      struct ras_fs_if *head,
1996 				      struct dentry *dir)
1997 {
1998 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1999 
2000 	if (!obj || !dir)
2001 		return;
2002 
2003 	get_obj(obj);
2004 
2005 	memcpy(obj->fs_data.debugfs_name,
2006 			head->debugfs_name,
2007 			sizeof(obj->fs_data.debugfs_name));
2008 
2009 	debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2010 			    obj, &amdgpu_ras_debugfs_ops);
2011 }
2012 
amdgpu_ras_aca_is_supported(struct amdgpu_device * adev)2013 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2014 {
2015 	bool ret;
2016 
2017 	switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2018 	case IP_VERSION(13, 0, 6):
2019 	case IP_VERSION(13, 0, 12):
2020 	case IP_VERSION(13, 0, 14):
2021 		ret = true;
2022 		break;
2023 	default:
2024 		ret = false;
2025 		break;
2026 	}
2027 
2028 	return ret;
2029 }
2030 
amdgpu_ras_debugfs_create_all(struct amdgpu_device * adev)2031 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2032 {
2033 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2034 	struct dentry *dir;
2035 	struct ras_manager *obj;
2036 	struct ras_fs_if fs_info;
2037 
2038 	/*
2039 	 * it won't be called in resume path, no need to check
2040 	 * suspend and gpu reset status
2041 	 */
2042 	if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2043 		return;
2044 
2045 	dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2046 
2047 	list_for_each_entry(obj, &con->head, node) {
2048 		if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2049 			(obj->attr_inuse == 1)) {
2050 			sprintf(fs_info.debugfs_name, "%s_err_inject",
2051 					get_ras_block_str(&obj->head));
2052 			fs_info.head = obj->head;
2053 			amdgpu_ras_debugfs_create(adev, &fs_info, dir);
2054 		}
2055 	}
2056 
2057 	if (amdgpu_ras_aca_is_supported(adev)) {
2058 		if (amdgpu_aca_is_enabled(adev))
2059 			amdgpu_aca_smu_debugfs_init(adev, dir);
2060 		else
2061 			amdgpu_mca_smu_debugfs_init(adev, dir);
2062 	}
2063 }
2064 
2065 /* debugfs end */
2066 
2067 /* ras fs */
2068 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2069 		amdgpu_ras_sysfs_badpages_read, NULL, 0);
2070 static DEVICE_ATTR(features, S_IRUGO,
2071 		amdgpu_ras_sysfs_features_read, NULL);
2072 static DEVICE_ATTR(version, 0444,
2073 		amdgpu_ras_sysfs_version_show, NULL);
2074 static DEVICE_ATTR(schema, 0444,
2075 		amdgpu_ras_sysfs_schema_show, NULL);
2076 static DEVICE_ATTR(event_state, 0444,
2077 		   amdgpu_ras_sysfs_event_state_show, NULL);
amdgpu_ras_fs_init(struct amdgpu_device * adev)2078 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2079 {
2080 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2081 	struct attribute_group group = {
2082 		.name = RAS_FS_NAME,
2083 	};
2084 	struct attribute *attrs[] = {
2085 		&con->features_attr.attr,
2086 		&con->version_attr.attr,
2087 		&con->schema_attr.attr,
2088 		&con->event_state_attr.attr,
2089 		NULL
2090 	};
2091 	struct bin_attribute *bin_attrs[] = {
2092 		NULL,
2093 		NULL,
2094 	};
2095 	int r;
2096 
2097 	group.attrs = attrs;
2098 
2099 	/* add features entry */
2100 	con->features_attr = dev_attr_features;
2101 	sysfs_attr_init(attrs[0]);
2102 
2103 	/* add version entry */
2104 	con->version_attr = dev_attr_version;
2105 	sysfs_attr_init(attrs[1]);
2106 
2107 	/* add schema entry */
2108 	con->schema_attr = dev_attr_schema;
2109 	sysfs_attr_init(attrs[2]);
2110 
2111 	/* add event_state entry */
2112 	con->event_state_attr = dev_attr_event_state;
2113 	sysfs_attr_init(attrs[3]);
2114 
2115 	if (amdgpu_bad_page_threshold != 0) {
2116 		/* add bad_page_features entry */
2117 		bin_attr_gpu_vram_bad_pages.private = NULL;
2118 		con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2119 		bin_attrs[0] = &con->badpages_attr;
2120 		group.bin_attrs = bin_attrs;
2121 		sysfs_bin_attr_init(bin_attrs[0]);
2122 	}
2123 
2124 	r = sysfs_create_group(&adev->dev->kobj, &group);
2125 	if (r)
2126 		dev_err(adev->dev, "Failed to create RAS sysfs group!");
2127 
2128 	return 0;
2129 }
2130 
amdgpu_ras_fs_fini(struct amdgpu_device * adev)2131 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2132 {
2133 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2134 	struct ras_manager *con_obj, *ip_obj, *tmp;
2135 
2136 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2137 		list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2138 			ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2139 			if (ip_obj)
2140 				put_obj(ip_obj);
2141 		}
2142 	}
2143 
2144 	amdgpu_ras_sysfs_remove_all(adev);
2145 	return 0;
2146 }
2147 /* ras fs end */
2148 
2149 /* ih begin */
2150 
2151 /* For the hardware that cannot enable bif ring for both ras_controller_irq
2152  * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2153  * register to check whether the interrupt is triggered or not, and properly
2154  * ack the interrupt if it is there
2155  */
amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device * adev)2156 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2157 {
2158 	/* Fatal error events are handled on host side */
2159 	if (amdgpu_sriov_vf(adev))
2160 		return;
2161 	/**
2162 	 * If the current interrupt is caused by a non-fatal RAS error, skip
2163 	 * check for fatal error. For fatal errors, FED status of all devices
2164 	 * in XGMI hive gets set when the first device gets fatal error
2165 	 * interrupt. The error gets propagated to other devices as well, so
2166 	 * make sure to ack the interrupt regardless of FED status.
2167 	 */
2168 	if (!amdgpu_ras_get_fed_status(adev) &&
2169 	    amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2170 		return;
2171 
2172 	if (adev->nbio.ras &&
2173 	    adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2174 		adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2175 
2176 	if (adev->nbio.ras &&
2177 	    adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2178 		adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2179 }
2180 
amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2181 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2182 				struct amdgpu_iv_entry *entry)
2183 {
2184 	bool poison_stat = false;
2185 	struct amdgpu_device *adev = obj->adev;
2186 	struct amdgpu_ras_block_object *block_obj =
2187 		amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
2188 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2189 	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2190 	u64 event_id;
2191 	int ret;
2192 
2193 	if (!block_obj || !con)
2194 		return;
2195 
2196 	ret = amdgpu_ras_mark_ras_event(adev, type);
2197 	if (ret)
2198 		return;
2199 
2200 	amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
2201 	/* both query_poison_status and handle_poison_consumption are optional,
2202 	 * but at least one of them should be implemented if we need poison
2203 	 * consumption handler
2204 	 */
2205 	if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2206 		poison_stat = block_obj->hw_ops->query_poison_status(adev);
2207 		if (!poison_stat) {
2208 			/* Not poison consumption interrupt, no need to handle it */
2209 			dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2210 					block_obj->ras_comm.name);
2211 
2212 			return;
2213 		}
2214 	}
2215 
2216 	amdgpu_umc_poison_handler(adev, obj->head.block, 0);
2217 
2218 	if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2219 		poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2220 
2221 	/* gpu reset is fallback for failed and default cases.
2222 	 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2223 	 */
2224 	if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2225 		event_id = amdgpu_ras_acquire_event_id(adev, type);
2226 		RAS_EVENT_LOG(adev, event_id,
2227 			      "GPU reset for %s RAS poison consumption is issued!\n",
2228 			      block_obj->ras_comm.name);
2229 		amdgpu_ras_reset_gpu(adev);
2230 	}
2231 
2232 	if (!poison_stat)
2233 		amdgpu_gfx_poison_consumption_handler(adev, entry);
2234 }
2235 
amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2236 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2237 				struct amdgpu_iv_entry *entry)
2238 {
2239 	struct amdgpu_device *adev = obj->adev;
2240 	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2241 	u64 event_id;
2242 	int ret;
2243 
2244 	ret = amdgpu_ras_mark_ras_event(adev, type);
2245 	if (ret)
2246 		return;
2247 
2248 	event_id = amdgpu_ras_acquire_event_id(adev, type);
2249 	RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2250 
2251 	if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2252 		struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2253 
2254 		atomic_inc(&con->page_retirement_req_cnt);
2255 		atomic_inc(&con->poison_creation_count);
2256 
2257 		wake_up(&con->page_retirement_wq);
2258 	}
2259 }
2260 
amdgpu_ras_interrupt_umc_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2261 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2262 				struct amdgpu_iv_entry *entry)
2263 {
2264 	struct ras_ih_data *data = &obj->ih_data;
2265 	struct ras_err_data err_data;
2266 	int ret;
2267 
2268 	if (!data->cb)
2269 		return;
2270 
2271 	ret = amdgpu_ras_error_data_init(&err_data);
2272 	if (ret)
2273 		return;
2274 
2275 	/* Let IP handle its data, maybe we need get the output
2276 	 * from the callback to update the error type/count, etc
2277 	 */
2278 	amdgpu_ras_set_fed(obj->adev, true);
2279 	ret = data->cb(obj->adev, &err_data, entry);
2280 	/* ue will trigger an interrupt, and in that case
2281 	 * we need do a reset to recovery the whole system.
2282 	 * But leave IP do that recovery, here we just dispatch
2283 	 * the error.
2284 	 */
2285 	if (ret == AMDGPU_RAS_SUCCESS) {
2286 		/* these counts could be left as 0 if
2287 		 * some blocks do not count error number
2288 		 */
2289 		obj->err_data.ue_count += err_data.ue_count;
2290 		obj->err_data.ce_count += err_data.ce_count;
2291 		obj->err_data.de_count += err_data.de_count;
2292 	}
2293 
2294 	amdgpu_ras_error_data_fini(&err_data);
2295 }
2296 
amdgpu_ras_interrupt_handler(struct ras_manager * obj)2297 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2298 {
2299 	struct ras_ih_data *data = &obj->ih_data;
2300 	struct amdgpu_iv_entry entry;
2301 
2302 	while (data->rptr != data->wptr) {
2303 		rmb();
2304 		memcpy(&entry, &data->ring[data->rptr],
2305 				data->element_size);
2306 
2307 		wmb();
2308 		data->rptr = (data->aligned_element_size +
2309 				data->rptr) % data->ring_size;
2310 
2311 		if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2312 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2313 				amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
2314 			else
2315 				amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
2316 		} else {
2317 			if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2318 				amdgpu_ras_interrupt_umc_handler(obj, &entry);
2319 			else
2320 				dev_warn(obj->adev->dev,
2321 					"No RAS interrupt handler for non-UMC block with poison disabled.\n");
2322 		}
2323 	}
2324 }
2325 
amdgpu_ras_interrupt_process_handler(struct work_struct * work)2326 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2327 {
2328 	struct ras_ih_data *data =
2329 		container_of(work, struct ras_ih_data, ih_work);
2330 	struct ras_manager *obj =
2331 		container_of(data, struct ras_manager, ih_data);
2332 
2333 	amdgpu_ras_interrupt_handler(obj);
2334 }
2335 
amdgpu_ras_interrupt_dispatch(struct amdgpu_device * adev,struct ras_dispatch_if * info)2336 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2337 		struct ras_dispatch_if *info)
2338 {
2339 	struct ras_manager *obj;
2340 	struct ras_ih_data *data;
2341 
2342 	obj = amdgpu_ras_find_obj(adev, &info->head);
2343 	if (!obj)
2344 		return -EINVAL;
2345 
2346 	data = &obj->ih_data;
2347 
2348 	if (data->inuse == 0)
2349 		return 0;
2350 
2351 	/* Might be overflow... */
2352 	memcpy(&data->ring[data->wptr], info->entry,
2353 			data->element_size);
2354 
2355 	wmb();
2356 	data->wptr = (data->aligned_element_size +
2357 			data->wptr) % data->ring_size;
2358 
2359 	schedule_work(&data->ih_work);
2360 
2361 	return 0;
2362 }
2363 
amdgpu_ras_interrupt_remove_handler(struct amdgpu_device * adev,struct ras_common_if * head)2364 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2365 		struct ras_common_if *head)
2366 {
2367 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2368 	struct ras_ih_data *data;
2369 
2370 	if (!obj)
2371 		return -EINVAL;
2372 
2373 	data = &obj->ih_data;
2374 	if (data->inuse == 0)
2375 		return 0;
2376 
2377 	cancel_work_sync(&data->ih_work);
2378 
2379 	kfree(data->ring);
2380 	memset(data, 0, sizeof(*data));
2381 	put_obj(obj);
2382 
2383 	return 0;
2384 }
2385 
amdgpu_ras_interrupt_add_handler(struct amdgpu_device * adev,struct ras_common_if * head)2386 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2387 		struct ras_common_if *head)
2388 {
2389 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2390 	struct ras_ih_data *data;
2391 	struct amdgpu_ras_block_object *ras_obj;
2392 
2393 	if (!obj) {
2394 		/* in case we registe the IH before enable ras feature */
2395 		obj = amdgpu_ras_create_obj(adev, head);
2396 		if (!obj)
2397 			return -EINVAL;
2398 	} else
2399 		get_obj(obj);
2400 
2401 	ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2402 
2403 	data = &obj->ih_data;
2404 	/* add the callback.etc */
2405 	*data = (struct ras_ih_data) {
2406 		.inuse = 0,
2407 		.cb = ras_obj->ras_cb,
2408 		.element_size = sizeof(struct amdgpu_iv_entry),
2409 		.rptr = 0,
2410 		.wptr = 0,
2411 	};
2412 
2413 	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2414 
2415 	data->aligned_element_size = ALIGN(data->element_size, 8);
2416 	/* the ring can store 64 iv entries. */
2417 	data->ring_size = 64 * data->aligned_element_size;
2418 	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2419 	if (!data->ring) {
2420 		put_obj(obj);
2421 		return -ENOMEM;
2422 	}
2423 
2424 	/* IH is ready */
2425 	data->inuse = 1;
2426 
2427 	return 0;
2428 }
2429 
amdgpu_ras_interrupt_remove_all(struct amdgpu_device * adev)2430 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2431 {
2432 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2433 	struct ras_manager *obj, *tmp;
2434 
2435 	list_for_each_entry_safe(obj, tmp, &con->head, node) {
2436 		amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
2437 	}
2438 
2439 	return 0;
2440 }
2441 /* ih end */
2442 
2443 /* traversal all IPs except NBIO to query error counter */
amdgpu_ras_log_on_err_counter(struct amdgpu_device * adev,enum ras_event_type type)2444 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2445 {
2446 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2447 	struct ras_manager *obj;
2448 
2449 	if (!adev->ras_enabled || !con)
2450 		return;
2451 
2452 	list_for_each_entry(obj, &con->head, node) {
2453 		struct ras_query_if info = {
2454 			.head = obj->head,
2455 		};
2456 
2457 		/*
2458 		 * PCIE_BIF IP has one different isr by ras controller
2459 		 * interrupt, the specific ras counter query will be
2460 		 * done in that isr. So skip such block from common
2461 		 * sync flood interrupt isr calling.
2462 		 */
2463 		if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2464 			continue;
2465 
2466 		/*
2467 		 * this is a workaround for aldebaran, skip send msg to
2468 		 * smu to get ecc_info table due to smu handle get ecc
2469 		 * info table failed temporarily.
2470 		 * should be removed until smu fix handle ecc_info table.
2471 		 */
2472 		if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2473 		    (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2474 		     IP_VERSION(13, 0, 2)))
2475 			continue;
2476 
2477 		amdgpu_ras_query_error_status_with_event(adev, &info, type);
2478 
2479 		if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2480 			    IP_VERSION(11, 0, 2) &&
2481 		    amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2482 			    IP_VERSION(11, 0, 4) &&
2483 		    amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2484 			    IP_VERSION(13, 0, 0)) {
2485 			if (amdgpu_ras_reset_error_status(adev, info.head.block))
2486 				dev_warn(adev->dev, "Failed to reset error counter and error status");
2487 		}
2488 	}
2489 }
2490 
2491 /* Parse RdRspStatus and WrRspStatus */
amdgpu_ras_error_status_query(struct amdgpu_device * adev,struct ras_query_if * info)2492 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2493 					  struct ras_query_if *info)
2494 {
2495 	struct amdgpu_ras_block_object *block_obj;
2496 	/*
2497 	 * Only two block need to query read/write
2498 	 * RspStatus at current state
2499 	 */
2500 	if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2501 		(info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2502 		return;
2503 
2504 	block_obj = amdgpu_ras_get_ras_block(adev,
2505 					info->head.block,
2506 					info->head.sub_block_index);
2507 
2508 	if (!block_obj || !block_obj->hw_ops) {
2509 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2510 			     get_ras_block_str(&info->head));
2511 		return;
2512 	}
2513 
2514 	if (block_obj->hw_ops->query_ras_error_status)
2515 		block_obj->hw_ops->query_ras_error_status(adev);
2516 
2517 }
2518 
amdgpu_ras_query_err_status(struct amdgpu_device * adev)2519 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2520 {
2521 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2522 	struct ras_manager *obj;
2523 
2524 	if (!adev->ras_enabled || !con)
2525 		return;
2526 
2527 	list_for_each_entry(obj, &con->head, node) {
2528 		struct ras_query_if info = {
2529 			.head = obj->head,
2530 		};
2531 
2532 		amdgpu_ras_error_status_query(adev, &info);
2533 	}
2534 }
2535 
2536 /* recovery begin */
2537 
2538 /* return 0 on success.
2539  * caller need free bps.
2540  */
amdgpu_ras_badpages_read(struct amdgpu_device * adev,struct ras_badpage ** bps,unsigned int * count)2541 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2542 		struct ras_badpage **bps, unsigned int *count)
2543 {
2544 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2545 	struct ras_err_handler_data *data;
2546 	int i = 0;
2547 	int ret = 0, status;
2548 
2549 	if (!con || !con->eh_data || !bps || !count)
2550 		return -EINVAL;
2551 
2552 	mutex_lock(&con->recovery_lock);
2553 	data = con->eh_data;
2554 	if (!data || data->count == 0) {
2555 		*bps = NULL;
2556 		ret = -EINVAL;
2557 		goto out;
2558 	}
2559 
2560 	*bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2561 	if (!*bps) {
2562 		ret = -ENOMEM;
2563 		goto out;
2564 	}
2565 
2566 	for (; i < data->count; i++) {
2567 		(*bps)[i] = (struct ras_badpage){
2568 			.bp = data->bps[i].retired_page,
2569 			.size = AMDGPU_GPU_PAGE_SIZE,
2570 			.flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2571 		};
2572 		status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2573 				data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
2574 		if (status == -EBUSY)
2575 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2576 		else if (status == -ENOENT)
2577 			(*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2578 	}
2579 
2580 	*count = data->count;
2581 out:
2582 	mutex_unlock(&con->recovery_lock);
2583 	return ret;
2584 }
2585 
amdgpu_ras_set_fed_all(struct amdgpu_device * adev,struct amdgpu_hive_info * hive,bool status)2586 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2587 				   struct amdgpu_hive_info *hive, bool status)
2588 {
2589 	struct amdgpu_device *tmp_adev;
2590 
2591 	if (hive) {
2592 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2593 			amdgpu_ras_set_fed(tmp_adev, status);
2594 	} else {
2595 		amdgpu_ras_set_fed(adev, status);
2596 	}
2597 }
2598 
amdgpu_ras_in_recovery(struct amdgpu_device * adev)2599 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2600 {
2601 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2602 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2603 	int hive_ras_recovery = 0;
2604 
2605 	if (hive) {
2606 		hive_ras_recovery = atomic_read(&hive->ras_recovery);
2607 		amdgpu_put_xgmi_hive(hive);
2608 	}
2609 
2610 	if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2611 		return true;
2612 
2613 	return false;
2614 }
2615 
amdgpu_ras_get_fatal_error_event(struct amdgpu_device * adev)2616 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2617 {
2618 	if (amdgpu_ras_intr_triggered())
2619 		return RAS_EVENT_TYPE_FATAL;
2620 	else
2621 		return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2622 }
2623 
amdgpu_ras_do_recovery(struct work_struct * work)2624 static void amdgpu_ras_do_recovery(struct work_struct *work)
2625 {
2626 	struct amdgpu_ras *ras =
2627 		container_of(work, struct amdgpu_ras, recovery_work);
2628 	struct amdgpu_device *remote_adev = NULL;
2629 	struct amdgpu_device *adev = ras->adev;
2630 	struct list_head device_list, *device_list_handle =  NULL;
2631 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2632 	enum ras_event_type type;
2633 
2634 	if (hive) {
2635 		atomic_set(&hive->ras_recovery, 1);
2636 
2637 		/* If any device which is part of the hive received RAS fatal
2638 		 * error interrupt, set fatal error status on all. This
2639 		 * condition will need a recovery, and flag will be cleared
2640 		 * as part of recovery.
2641 		 */
2642 		list_for_each_entry(remote_adev, &hive->device_list,
2643 				    gmc.xgmi.head)
2644 			if (amdgpu_ras_get_fed_status(remote_adev)) {
2645 				amdgpu_ras_set_fed_all(adev, hive, true);
2646 				break;
2647 			}
2648 	}
2649 	if (!ras->disable_ras_err_cnt_harvest) {
2650 
2651 		/* Build list of devices to query RAS related errors */
2652 		if  (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2653 			device_list_handle = &hive->device_list;
2654 		} else {
2655 			INIT_LIST_HEAD(&device_list);
2656 			list_add_tail(&adev->gmc.xgmi.head, &device_list);
2657 			device_list_handle = &device_list;
2658 		}
2659 
2660 		type = amdgpu_ras_get_fatal_error_event(adev);
2661 		list_for_each_entry(remote_adev,
2662 				device_list_handle, gmc.xgmi.head) {
2663 			amdgpu_ras_query_err_status(remote_adev);
2664 			amdgpu_ras_log_on_err_counter(remote_adev, type);
2665 		}
2666 
2667 	}
2668 
2669 	if (amdgpu_device_should_recover_gpu(ras->adev)) {
2670 		struct amdgpu_reset_context reset_context;
2671 		memset(&reset_context, 0, sizeof(reset_context));
2672 
2673 		reset_context.method = AMD_RESET_METHOD_NONE;
2674 		reset_context.reset_req_dev = adev;
2675 		reset_context.src = AMDGPU_RESET_SRC_RAS;
2676 		set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
2677 
2678 		/* Perform full reset in fatal error mode */
2679 		if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2680 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2681 		else {
2682 			clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2683 
2684 			if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2685 				ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2686 				reset_context.method = AMD_RESET_METHOD_MODE2;
2687 			}
2688 
2689 			/* Fatal error occurs in poison mode, mode1 reset is used to
2690 			 * recover gpu.
2691 			 */
2692 			if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2693 				ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2694 				set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2695 
2696 				psp_fatal_error_recovery_quirk(&adev->psp);
2697 			}
2698 		}
2699 
2700 		amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2701 	}
2702 	atomic_set(&ras->in_recovery, 0);
2703 	if (hive) {
2704 		atomic_set(&hive->ras_recovery, 0);
2705 		amdgpu_put_xgmi_hive(hive);
2706 	}
2707 }
2708 
2709 /* alloc/realloc bps array */
amdgpu_ras_realloc_eh_data_space(struct amdgpu_device * adev,struct ras_err_handler_data * data,int pages)2710 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2711 		struct ras_err_handler_data *data, int pages)
2712 {
2713 	unsigned int old_space = data->count + data->space_left;
2714 	unsigned int new_space = old_space + pages;
2715 	unsigned int align_space = ALIGN(new_space, 512);
2716 	void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2717 
2718 	if (!bps) {
2719 		return -ENOMEM;
2720 	}
2721 
2722 	if (data->bps) {
2723 		memcpy(bps, data->bps,
2724 				data->count * sizeof(*data->bps));
2725 		kfree(data->bps);
2726 	}
2727 
2728 	data->bps = bps;
2729 	data->space_left += align_space - old_space;
2730 	return 0;
2731 }
2732 
amdgpu_ras_mca2pa_by_idx(struct amdgpu_device * adev,struct eeprom_table_record * bps,struct ras_err_data * err_data)2733 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
2734 			struct eeprom_table_record *bps,
2735 			struct ras_err_data *err_data)
2736 {
2737 	struct ta_ras_query_address_input addr_in;
2738 	uint32_t socket = 0;
2739 	int ret = 0;
2740 
2741 	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2742 		socket = adev->smuio.funcs->get_socket_id(adev);
2743 
2744 	/* reinit err_data */
2745 	err_data->err_addr_cnt = 0;
2746 	err_data->err_addr_len = adev->umc.retire_unit;
2747 
2748 	memset(&addr_in, 0, sizeof(addr_in));
2749 	addr_in.ma.err_addr = bps->address;
2750 	addr_in.ma.socket_id = socket;
2751 	addr_in.ma.ch_inst = bps->mem_channel;
2752 	/* tell RAS TA the node instance is not used */
2753 	addr_in.ma.node_inst = TA_RAS_INV_NODE;
2754 
2755 	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2756 		ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2757 				&addr_in, NULL, false);
2758 
2759 	return ret;
2760 }
2761 
amdgpu_ras_mca2pa(struct amdgpu_device * adev,struct eeprom_table_record * bps,struct ras_err_data * err_data)2762 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2763 			struct eeprom_table_record *bps,
2764 			struct ras_err_data *err_data)
2765 {
2766 	struct ta_ras_query_address_input addr_in;
2767 	uint32_t die_id, socket = 0;
2768 
2769 	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2770 		socket = adev->smuio.funcs->get_socket_id(adev);
2771 
2772 	/* although die id is gotten from PA in nps1 mode, the id is
2773 	 * fitable for any nps mode
2774 	 */
2775 	if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2776 		die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2777 					bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2778 	else
2779 		return -EINVAL;
2780 
2781 	/* reinit err_data */
2782 	err_data->err_addr_cnt = 0;
2783 	err_data->err_addr_len = adev->umc.retire_unit;
2784 
2785 	memset(&addr_in, 0, sizeof(addr_in));
2786 	addr_in.ma.err_addr = bps->address;
2787 	addr_in.ma.ch_inst = bps->mem_channel;
2788 	addr_in.ma.umc_inst = bps->mcumc_id;
2789 	addr_in.ma.node_inst = die_id;
2790 	addr_in.ma.socket_id = socket;
2791 
2792 	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2793 		return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2794 					&addr_in, NULL, false);
2795 	else
2796 		return  -EINVAL;
2797 }
2798 
2799 /* it deal with vram only. */
amdgpu_ras_add_bad_pages(struct amdgpu_device * adev,struct eeprom_table_record * bps,int pages,bool from_rom)2800 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2801 		struct eeprom_table_record *bps, int pages, bool from_rom)
2802 {
2803 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2804 	struct ras_err_handler_data *data;
2805 	struct ras_err_data err_data;
2806 	struct eeprom_table_record *err_rec;
2807 	struct amdgpu_ras_eeprom_control *control =
2808 			&adev->psp.ras_context.ras->eeprom_control;
2809 	enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
2810 	int ret = 0;
2811 	uint32_t i, j, loop_cnt = 1;
2812 	bool find_pages_per_pa = false;
2813 
2814 	if (!con || !con->eh_data || !bps || pages <= 0)
2815 		return 0;
2816 
2817 	if (from_rom) {
2818 		err_data.err_addr =
2819 			kcalloc(adev->umc.retire_unit,
2820 				sizeof(struct eeprom_table_record), GFP_KERNEL);
2821 		if (!err_data.err_addr) {
2822 			dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
2823 			ret = -ENOMEM;
2824 			goto out;
2825 		}
2826 
2827 		err_rec = err_data.err_addr;
2828 		loop_cnt = adev->umc.retire_unit;
2829 		if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2830 			nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
2831 	}
2832 
2833 	mutex_lock(&con->recovery_lock);
2834 	data = con->eh_data;
2835 	if (!data) {
2836 		/* Returning 0 as the absence of eh_data is acceptable */
2837 		goto free;
2838 	}
2839 
2840 	for (i = 0; i < pages; i++) {
2841 		if (from_rom &&
2842 		    control->rec_type == AMDGPU_RAS_EEPROM_REC_MCA) {
2843 			if (!find_pages_per_pa) {
2844 				if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2845 					if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2846 						/* may use old RAS TA, use PA to find pages in
2847 						 * one row
2848 						 */
2849 						if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2850 									      bps[i].retired_page <<
2851 									      AMDGPU_GPU_PAGE_SHIFT)) {
2852 							ret = -EINVAL;
2853 							goto free;
2854 						} else {
2855 							find_pages_per_pa = true;
2856 						}
2857 					} else {
2858 						/* unsupported cases */
2859 						ret = -EOPNOTSUPP;
2860 						goto free;
2861 					}
2862 				}
2863 			} else {
2864 				if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2865 						bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
2866 					ret = -EINVAL;
2867 					goto free;
2868 				}
2869 			}
2870 		} else {
2871 			if (from_rom && !find_pages_per_pa) {
2872 				if (bps[i].retired_page & UMC_CHANNEL_IDX_V2) {
2873 					/* bad page in any NPS mode in eeprom */
2874 					if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2875 						ret = -EINVAL;
2876 						goto free;
2877 					}
2878 				} else {
2879 					/* legacy bad page in eeprom, generated only in
2880 					 * NPS1 mode
2881 					 */
2882 					if (amdgpu_ras_mca2pa(adev, &bps[i], &err_data)) {
2883 						/* old RAS TA or ASICs which don't support to
2884 						 * convert addrss via mca address
2885 						 */
2886 						if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2887 							find_pages_per_pa = true;
2888 							err_rec = &bps[i];
2889 							loop_cnt = 1;
2890 						} else {
2891 							/* non-nps1 mode, old RAS TA
2892 							 * can't support it
2893 							 */
2894 							ret = -EOPNOTSUPP;
2895 							goto free;
2896 						}
2897 					}
2898 				}
2899 
2900 				if (!find_pages_per_pa)
2901 					i += (adev->umc.retire_unit - 1);
2902 			} else {
2903 				err_rec = &bps[i];
2904 			}
2905 		}
2906 
2907 		for (j = 0; j < loop_cnt; j++) {
2908 			if (amdgpu_ras_check_bad_page_unlock(con,
2909 				err_rec[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2910 				continue;
2911 
2912 			if (!data->space_left &&
2913 			    amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2914 				ret = -ENOMEM;
2915 				goto free;
2916 			}
2917 
2918 			amdgpu_ras_reserve_page(adev, err_rec[j].retired_page);
2919 
2920 			memcpy(&data->bps[data->count], &(err_rec[j]),
2921 					sizeof(struct eeprom_table_record));
2922 			data->count++;
2923 			data->space_left--;
2924 		}
2925 	}
2926 
2927 free:
2928 	if (from_rom)
2929 		kfree(err_data.err_addr);
2930 out:
2931 	mutex_unlock(&con->recovery_lock);
2932 
2933 	return ret;
2934 }
2935 
2936 /*
2937  * write error record array to eeprom, the function should be
2938  * protected by recovery_lock
2939  * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
2940  */
amdgpu_ras_save_bad_pages(struct amdgpu_device * adev,unsigned long * new_cnt)2941 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2942 		unsigned long *new_cnt)
2943 {
2944 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2945 	struct ras_err_handler_data *data;
2946 	struct amdgpu_ras_eeprom_control *control;
2947 	int save_count, unit_num, bad_page_num, i;
2948 
2949 	if (!con || !con->eh_data) {
2950 		if (new_cnt)
2951 			*new_cnt = 0;
2952 
2953 		return 0;
2954 	}
2955 
2956 	mutex_lock(&con->recovery_lock);
2957 	control = &con->eeprom_control;
2958 	data = con->eh_data;
2959 	bad_page_num = control->ras_num_bad_pages;
2960 	save_count = data->count - bad_page_num;
2961 	mutex_unlock(&con->recovery_lock);
2962 
2963 	unit_num = save_count / adev->umc.retire_unit;
2964 	if (new_cnt)
2965 		*new_cnt = unit_num;
2966 
2967 	/* only new entries are saved */
2968 	if (save_count > 0) {
2969 		if (control->rec_type == AMDGPU_RAS_EEPROM_REC_PA) {
2970 			if (amdgpu_ras_eeprom_append(control,
2971 						     &data->bps[control->ras_num_recs],
2972 						     save_count)) {
2973 				dev_err(adev->dev, "Failed to save EEPROM table data!");
2974 				return -EIO;
2975 			}
2976 		} else {
2977 			for (i = 0; i < unit_num; i++) {
2978 				if (amdgpu_ras_eeprom_append(control,
2979 						&data->bps[bad_page_num + i * adev->umc.retire_unit],
2980 						1)) {
2981 					dev_err(adev->dev, "Failed to save EEPROM table data!");
2982 					return -EIO;
2983 				}
2984 			}
2985 		}
2986 
2987 		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2988 	}
2989 
2990 	return 0;
2991 }
2992 
2993 /*
2994  * read error record array in eeprom and reserve enough space for
2995  * storing new bad pages
2996  */
amdgpu_ras_load_bad_pages(struct amdgpu_device * adev)2997 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2998 {
2999 	struct amdgpu_ras_eeprom_control *control =
3000 		&adev->psp.ras_context.ras->eeprom_control;
3001 	struct eeprom_table_record *bps;
3002 	int ret;
3003 
3004 	/* no bad page record, skip eeprom access */
3005 	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3006 		return 0;
3007 
3008 	bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3009 	if (!bps)
3010 		return -ENOMEM;
3011 
3012 	ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
3013 	if (ret) {
3014 		dev_err(adev->dev, "Failed to load EEPROM table records!");
3015 	} else {
3016 		if (control->ras_num_recs > 1 &&
3017 		    adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3018 			if ((bps[0].address == bps[1].address) &&
3019 			    (bps[0].mem_channel == bps[1].mem_channel))
3020 				control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3021 			else
3022 				control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3023 		}
3024 
3025 		ret = amdgpu_ras_eeprom_check(control);
3026 		if (ret)
3027 			goto out;
3028 
3029 		/* HW not usable */
3030 		if (amdgpu_ras_is_rma(adev)) {
3031 			ret = -EHWPOISON;
3032 			goto out;
3033 		}
3034 
3035 		ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
3036 	}
3037 
3038 out:
3039 	kfree(bps);
3040 	return ret;
3041 }
3042 
amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras * con,uint64_t addr)3043 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3044 				uint64_t addr)
3045 {
3046 	struct ras_err_handler_data *data = con->eh_data;
3047 	int i;
3048 
3049 	addr >>= AMDGPU_GPU_PAGE_SHIFT;
3050 	for (i = 0; i < data->count; i++)
3051 		if (addr == data->bps[i].retired_page)
3052 			return true;
3053 
3054 	return false;
3055 }
3056 
3057 /*
3058  * check if an address belongs to bad page
3059  *
3060  * Note: this check is only for umc block
3061  */
amdgpu_ras_check_bad_page(struct amdgpu_device * adev,uint64_t addr)3062 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3063 				uint64_t addr)
3064 {
3065 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3066 	bool ret = false;
3067 
3068 	if (!con || !con->eh_data)
3069 		return ret;
3070 
3071 	mutex_lock(&con->recovery_lock);
3072 	ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3073 	mutex_unlock(&con->recovery_lock);
3074 	return ret;
3075 }
3076 
amdgpu_ras_validate_threshold(struct amdgpu_device * adev,uint32_t max_count)3077 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3078 					  uint32_t max_count)
3079 {
3080 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3081 
3082 	/*
3083 	 * Justification of value bad_page_cnt_threshold in ras structure
3084 	 *
3085 	 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
3086 	 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
3087 	 * scenarios accordingly.
3088 	 *
3089 	 * Bad page retirement enablement:
3090 	 *    - If amdgpu_bad_page_threshold = -2,
3091 	 *      bad_page_cnt_threshold = typical value by formula.
3092 	 *
3093 	 *    - When the value from user is 0 < amdgpu_bad_page_threshold <
3094 	 *      max record length in eeprom, use it directly.
3095 	 *
3096 	 * Bad page retirement disablement:
3097 	 *    - If amdgpu_bad_page_threshold = 0, bad page retirement
3098 	 *      functionality is disabled, and bad_page_cnt_threshold will
3099 	 *      take no effect.
3100 	 */
3101 
3102 	if (amdgpu_bad_page_threshold < 0) {
3103 		u64 val = adev->gmc.mc_vram_size;
3104 
3105 		do_div(val, RAS_BAD_PAGE_COVER);
3106 		con->bad_page_cnt_threshold = min(lower_32_bits(val),
3107 						  max_count);
3108 	} else {
3109 		con->bad_page_cnt_threshold = min_t(int, max_count,
3110 						    amdgpu_bad_page_threshold);
3111 	}
3112 }
3113 
amdgpu_ras_put_poison_req(struct amdgpu_device * adev,enum amdgpu_ras_block block,uint16_t pasid,pasid_notify pasid_fn,void * data,uint32_t reset)3114 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3115 		enum amdgpu_ras_block block, uint16_t pasid,
3116 		pasid_notify pasid_fn, void *data, uint32_t reset)
3117 {
3118 	int ret = 0;
3119 	struct ras_poison_msg poison_msg;
3120 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3121 
3122 	memset(&poison_msg, 0, sizeof(poison_msg));
3123 	poison_msg.block = block;
3124 	poison_msg.pasid = pasid;
3125 	poison_msg.reset = reset;
3126 	poison_msg.pasid_fn = pasid_fn;
3127 	poison_msg.data = data;
3128 
3129 	ret = kfifo_put(&con->poison_fifo, poison_msg);
3130 	if (!ret) {
3131 		dev_err(adev->dev, "Poison message fifo is full!\n");
3132 		return -ENOSPC;
3133 	}
3134 
3135 	return 0;
3136 }
3137 
amdgpu_ras_get_poison_req(struct amdgpu_device * adev,struct ras_poison_msg * poison_msg)3138 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3139 		struct ras_poison_msg *poison_msg)
3140 {
3141 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3142 
3143 	return kfifo_get(&con->poison_fifo, poison_msg);
3144 }
3145 
amdgpu_ras_ecc_log_init(struct ras_ecc_log_info * ecc_log)3146 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3147 {
3148 	mutex_init(&ecc_log->lock);
3149 
3150 	INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3151 	ecc_log->de_queried_count = 0;
3152 	ecc_log->prev_de_queried_count = 0;
3153 }
3154 
amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info * ecc_log)3155 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3156 {
3157 	struct radix_tree_iter iter;
3158 	void __rcu **slot;
3159 	struct ras_ecc_err *ecc_err;
3160 
3161 	mutex_lock(&ecc_log->lock);
3162 	radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3163 		ecc_err = radix_tree_deref_slot(slot);
3164 		kfree(ecc_err->err_pages.pfn);
3165 		kfree(ecc_err);
3166 		radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3167 	}
3168 	mutex_unlock(&ecc_log->lock);
3169 
3170 	mutex_destroy(&ecc_log->lock);
3171 	ecc_log->de_queried_count = 0;
3172 	ecc_log->prev_de_queried_count = 0;
3173 }
3174 
amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras * con,uint32_t delayed_ms)3175 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3176 				uint32_t delayed_ms)
3177 {
3178 	int ret;
3179 
3180 	mutex_lock(&con->umc_ecc_log.lock);
3181 	ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3182 			UMC_ECC_NEW_DETECTED_TAG);
3183 	mutex_unlock(&con->umc_ecc_log.lock);
3184 
3185 	if (ret)
3186 		schedule_delayed_work(&con->page_retirement_dwork,
3187 			msecs_to_jiffies(delayed_ms));
3188 
3189 	return ret ? true : false;
3190 }
3191 
amdgpu_ras_do_page_retirement(struct work_struct * work)3192 static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3193 {
3194 	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3195 					      page_retirement_dwork.work);
3196 	struct amdgpu_device *adev = con->adev;
3197 	struct ras_err_data err_data;
3198 	unsigned long err_cnt;
3199 
3200 	/* If gpu reset is ongoing, delay retiring the bad pages */
3201 	if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3202 		amdgpu_ras_schedule_retirement_dwork(con,
3203 				AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3204 		return;
3205 	}
3206 
3207 	amdgpu_ras_error_data_init(&err_data);
3208 
3209 	amdgpu_umc_handle_bad_pages(adev, &err_data);
3210 	err_cnt = err_data.err_addr_cnt;
3211 
3212 	amdgpu_ras_error_data_fini(&err_data);
3213 
3214 	if (err_cnt && amdgpu_ras_is_rma(adev))
3215 		amdgpu_ras_reset_gpu(adev);
3216 
3217 	amdgpu_ras_schedule_retirement_dwork(con,
3218 			AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3219 }
3220 
amdgpu_ras_poison_creation_handler(struct amdgpu_device * adev,uint32_t poison_creation_count)3221 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3222 				uint32_t poison_creation_count)
3223 {
3224 	int ret = 0;
3225 	struct ras_ecc_log_info *ecc_log;
3226 	struct ras_query_if info;
3227 	uint32_t timeout = 0;
3228 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3229 	uint64_t de_queried_count;
3230 	uint32_t new_detect_count, total_detect_count;
3231 	uint32_t need_query_count = poison_creation_count;
3232 	bool query_data_timeout = false;
3233 	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3234 
3235 	memset(&info, 0, sizeof(info));
3236 	info.head.block = AMDGPU_RAS_BLOCK__UMC;
3237 
3238 	ecc_log = &ras->umc_ecc_log;
3239 	total_detect_count = 0;
3240 	do {
3241 		ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
3242 		if (ret)
3243 			return ret;
3244 
3245 		de_queried_count = ecc_log->de_queried_count;
3246 		if (de_queried_count > ecc_log->prev_de_queried_count) {
3247 			new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3248 			ecc_log->prev_de_queried_count = de_queried_count;
3249 			timeout = 0;
3250 		} else {
3251 			new_detect_count = 0;
3252 		}
3253 
3254 		if (new_detect_count) {
3255 			total_detect_count += new_detect_count;
3256 		} else {
3257 			if (!timeout && need_query_count)
3258 				timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3259 
3260 			if (timeout) {
3261 				if (!--timeout) {
3262 					query_data_timeout = true;
3263 					break;
3264 				}
3265 				msleep(1);
3266 			}
3267 		}
3268 	} while (total_detect_count < need_query_count);
3269 
3270 	if (query_data_timeout) {
3271 		dev_warn(adev->dev, "Can't find deferred error! count: %u\n",
3272 			(need_query_count - total_detect_count));
3273 		return -ENOENT;
3274 	}
3275 
3276 	if (total_detect_count)
3277 		schedule_delayed_work(&ras->page_retirement_dwork, 0);
3278 
3279 	return 0;
3280 }
3281 
amdgpu_ras_clear_poison_fifo(struct amdgpu_device * adev)3282 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3283 {
3284 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3285 	struct ras_poison_msg msg;
3286 	int ret;
3287 
3288 	do {
3289 		ret = kfifo_get(&con->poison_fifo, &msg);
3290 	} while (ret);
3291 }
3292 
amdgpu_ras_poison_consumption_handler(struct amdgpu_device * adev,uint32_t msg_count,uint32_t * gpu_reset)3293 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3294 			uint32_t msg_count, uint32_t *gpu_reset)
3295 {
3296 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3297 	uint32_t reset_flags = 0, reset = 0;
3298 	struct ras_poison_msg msg;
3299 	int ret, i;
3300 
3301 	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3302 
3303 	for (i = 0; i < msg_count; i++) {
3304 		ret = amdgpu_ras_get_poison_req(adev, &msg);
3305 		if (!ret)
3306 			continue;
3307 
3308 		if (msg.pasid_fn)
3309 			msg.pasid_fn(adev, msg.pasid, msg.data);
3310 
3311 		reset_flags |= msg.reset;
3312 	}
3313 
3314 	/* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3315 	if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3316 		if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3317 			reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3318 		else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3319 			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3320 		else
3321 			reset = reset_flags;
3322 
3323 		flush_delayed_work(&con->page_retirement_dwork);
3324 
3325 		con->gpu_reset_flags |= reset;
3326 		amdgpu_ras_reset_gpu(adev);
3327 
3328 		*gpu_reset = reset;
3329 
3330 		/* Wait for gpu recovery to complete */
3331 		flush_work(&con->recovery_work);
3332 	}
3333 
3334 	return 0;
3335 }
3336 
amdgpu_ras_page_retirement_thread(void * param)3337 static int amdgpu_ras_page_retirement_thread(void *param)
3338 {
3339 	struct amdgpu_device *adev = (struct amdgpu_device *)param;
3340 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3341 	uint32_t poison_creation_count, msg_count;
3342 	uint32_t gpu_reset;
3343 	int ret;
3344 
3345 	while (!kthread_should_stop()) {
3346 
3347 		wait_event_interruptible(con->page_retirement_wq,
3348 				kthread_should_stop() ||
3349 				atomic_read(&con->page_retirement_req_cnt));
3350 
3351 		if (kthread_should_stop())
3352 			break;
3353 
3354 		gpu_reset = 0;
3355 
3356 		do {
3357 			poison_creation_count = atomic_read(&con->poison_creation_count);
3358 			ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3359 			if (ret == -EIO)
3360 				break;
3361 
3362 			if (poison_creation_count) {
3363 				atomic_sub(poison_creation_count, &con->poison_creation_count);
3364 				atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3365 			}
3366 		} while (atomic_read(&con->poison_creation_count));
3367 
3368 		if (ret != -EIO) {
3369 			msg_count = kfifo_len(&con->poison_fifo);
3370 			if (msg_count) {
3371 				ret = amdgpu_ras_poison_consumption_handler(adev,
3372 						msg_count, &gpu_reset);
3373 				if ((ret != -EIO) &&
3374 				    (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3375 					atomic_sub(msg_count, &con->page_retirement_req_cnt);
3376 			}
3377 		}
3378 
3379 		if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3380 			/* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3381 			/* Clear poison creation request */
3382 			atomic_set(&con->poison_creation_count, 0);
3383 
3384 			/* Clear poison fifo */
3385 			amdgpu_ras_clear_poison_fifo(adev);
3386 
3387 			/* Clear all poison requests */
3388 			atomic_set(&con->page_retirement_req_cnt, 0);
3389 
3390 			if (ret == -EIO) {
3391 				/* Wait for mode-1 reset to complete */
3392 				down_read(&adev->reset_domain->sem);
3393 				up_read(&adev->reset_domain->sem);
3394 			}
3395 
3396 			/* Wake up work to save bad pages to eeprom */
3397 			schedule_delayed_work(&con->page_retirement_dwork, 0);
3398 		} else if (gpu_reset) {
3399 			/* gpu just completed mode-2 reset or other reset */
3400 			/* Clear poison consumption messages cached in fifo */
3401 			msg_count = kfifo_len(&con->poison_fifo);
3402 			if (msg_count) {
3403 				amdgpu_ras_clear_poison_fifo(adev);
3404 				atomic_sub(msg_count, &con->page_retirement_req_cnt);
3405 			}
3406 
3407 			/* Wake up work to save bad pages to eeprom */
3408 			schedule_delayed_work(&con->page_retirement_dwork, 0);
3409 		}
3410 	}
3411 
3412 	return 0;
3413 }
3414 
amdgpu_ras_init_badpage_info(struct amdgpu_device * adev)3415 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3416 {
3417 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3418 	struct amdgpu_ras_eeprom_control *control;
3419 	int ret;
3420 
3421 	if (!con || amdgpu_sriov_vf(adev))
3422 		return 0;
3423 
3424 	control = &con->eeprom_control;
3425 	ret = amdgpu_ras_eeprom_init(control);
3426 	if (ret)
3427 		return ret;
3428 
3429 	if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3430 		control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3431 
3432 	/* default status is MCA storage */
3433 	if (control->ras_num_recs <= 1 &&
3434 	    adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3435 		control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3436 
3437 	if (control->ras_num_recs) {
3438 		ret = amdgpu_ras_load_bad_pages(adev);
3439 		if (ret)
3440 			return ret;
3441 
3442 		amdgpu_dpm_send_hbm_bad_pages_num(
3443 			adev, control->ras_num_bad_pages);
3444 
3445 		if (con->update_channel_flag == true) {
3446 			amdgpu_dpm_send_hbm_bad_channel_flag(
3447 				adev, control->bad_channel_bitmap);
3448 			con->update_channel_flag = false;
3449 		}
3450 	}
3451 
3452 	return ret;
3453 }
3454 
amdgpu_ras_recovery_init(struct amdgpu_device * adev,bool init_bp_info)3455 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3456 {
3457 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3458 	struct ras_err_handler_data **data;
3459 	u32  max_eeprom_records_count = 0;
3460 	int ret;
3461 
3462 	if (!con || amdgpu_sriov_vf(adev))
3463 		return 0;
3464 
3465 	/* Allow access to RAS EEPROM via debugfs, when the ASIC
3466 	 * supports RAS and debugfs is enabled, but when
3467 	 * adev->ras_enabled is unset, i.e. when "ras_enable"
3468 	 * module parameter is set to 0.
3469 	 */
3470 	con->adev = adev;
3471 
3472 	if (!adev->ras_enabled)
3473 		return 0;
3474 
3475 	data = &con->eh_data;
3476 	*data = kzalloc(sizeof(**data), GFP_KERNEL);
3477 	if (!*data) {
3478 		ret = -ENOMEM;
3479 		goto out;
3480 	}
3481 
3482 	mutex_init(&con->recovery_lock);
3483 	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3484 	atomic_set(&con->in_recovery, 0);
3485 	con->eeprom_control.bad_channel_bitmap = 0;
3486 
3487 	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
3488 	amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
3489 
3490 	if (init_bp_info) {
3491 		ret = amdgpu_ras_init_badpage_info(adev);
3492 		if (ret)
3493 			goto free;
3494 	}
3495 
3496 	mutex_init(&con->page_rsv_lock);
3497 	INIT_KFIFO(con->poison_fifo);
3498 	mutex_init(&con->page_retirement_lock);
3499 	init_waitqueue_head(&con->page_retirement_wq);
3500 	atomic_set(&con->page_retirement_req_cnt, 0);
3501 	atomic_set(&con->poison_creation_count, 0);
3502 	con->page_retirement_thread =
3503 		kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3504 	if (IS_ERR(con->page_retirement_thread)) {
3505 		con->page_retirement_thread = NULL;
3506 		dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3507 	}
3508 
3509 	INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3510 	amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
3511 #ifdef CONFIG_X86_MCE_AMD
3512 	if ((adev->asic_type == CHIP_ALDEBARAN) &&
3513 	    (adev->gmc.xgmi.connected_to_cpu))
3514 		amdgpu_register_bad_pages_mca_notifier(adev);
3515 #endif
3516 	return 0;
3517 
3518 free:
3519 	kfree((*data)->bps);
3520 	kfree(*data);
3521 	con->eh_data = NULL;
3522 out:
3523 	dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3524 
3525 	/*
3526 	 * Except error threshold exceeding case, other failure cases in this
3527 	 * function would not fail amdgpu driver init.
3528 	 */
3529 	if (!amdgpu_ras_is_rma(adev))
3530 		ret = 0;
3531 	else
3532 		ret = -EINVAL;
3533 
3534 	return ret;
3535 }
3536 
amdgpu_ras_recovery_fini(struct amdgpu_device * adev)3537 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3538 {
3539 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3540 	struct ras_err_handler_data *data = con->eh_data;
3541 	int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3542 	bool ret;
3543 
3544 	/* recovery_init failed to init it, fini is useless */
3545 	if (!data)
3546 		return 0;
3547 
3548 	/* Save all cached bad pages to eeprom */
3549 	do {
3550 		flush_delayed_work(&con->page_retirement_dwork);
3551 		ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3552 	} while (ret && max_flush_timeout--);
3553 
3554 	if (con->page_retirement_thread)
3555 		kthread_stop(con->page_retirement_thread);
3556 
3557 	atomic_set(&con->page_retirement_req_cnt, 0);
3558 	atomic_set(&con->poison_creation_count, 0);
3559 
3560 	mutex_destroy(&con->page_rsv_lock);
3561 
3562 	cancel_work_sync(&con->recovery_work);
3563 
3564 	cancel_delayed_work_sync(&con->page_retirement_dwork);
3565 
3566 	amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3567 
3568 	mutex_lock(&con->recovery_lock);
3569 	con->eh_data = NULL;
3570 	kfree(data->bps);
3571 	kfree(data);
3572 	mutex_unlock(&con->recovery_lock);
3573 
3574 	return 0;
3575 }
3576 /* recovery end */
3577 
amdgpu_ras_asic_supported(struct amdgpu_device * adev)3578 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3579 {
3580 	if (amdgpu_sriov_vf(adev)) {
3581 		switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3582 		case IP_VERSION(13, 0, 2):
3583 		case IP_VERSION(13, 0, 6):
3584 		case IP_VERSION(13, 0, 12):
3585 		case IP_VERSION(13, 0, 14):
3586 			return true;
3587 		default:
3588 			return false;
3589 		}
3590 	}
3591 
3592 	if (adev->asic_type == CHIP_IP_DISCOVERY) {
3593 		switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3594 		case IP_VERSION(13, 0, 0):
3595 		case IP_VERSION(13, 0, 6):
3596 		case IP_VERSION(13, 0, 10):
3597 		case IP_VERSION(13, 0, 12):
3598 		case IP_VERSION(13, 0, 14):
3599 		case IP_VERSION(14, 0, 3):
3600 			return true;
3601 		default:
3602 			return false;
3603 		}
3604 	}
3605 
3606 	return adev->asic_type == CHIP_VEGA10 ||
3607 		adev->asic_type == CHIP_VEGA20 ||
3608 		adev->asic_type == CHIP_ARCTURUS ||
3609 		adev->asic_type == CHIP_ALDEBARAN ||
3610 		adev->asic_type == CHIP_SIENNA_CICHLID;
3611 }
3612 
3613 /*
3614  * this is workaround for vega20 workstation sku,
3615  * force enable gfx ras, ignore vbios gfx ras flag
3616  * due to GC EDC can not write
3617  */
amdgpu_ras_get_quirks(struct amdgpu_device * adev)3618 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
3619 {
3620 	struct atom_context *ctx = adev->mode_info.atom_context;
3621 
3622 	if (!ctx)
3623 		return;
3624 
3625 	if (strnstr(ctx->vbios_pn, "D16406",
3626 		    sizeof(ctx->vbios_pn)) ||
3627 		strnstr(ctx->vbios_pn, "D36002",
3628 			sizeof(ctx->vbios_pn)))
3629 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
3630 }
3631 
3632 /* Query ras capablity via atomfirmware interface */
amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device * adev)3633 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3634 {
3635 	/* mem_ecc cap */
3636 	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3637 		dev_info(adev->dev, "MEM ECC is active.\n");
3638 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3639 					 1 << AMDGPU_RAS_BLOCK__DF);
3640 	} else {
3641 		dev_info(adev->dev, "MEM ECC is not presented.\n");
3642 	}
3643 
3644 	/* sram_ecc cap */
3645 	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3646 		dev_info(adev->dev, "SRAM ECC is active.\n");
3647 		if (!amdgpu_sriov_vf(adev))
3648 			adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3649 						  1 << AMDGPU_RAS_BLOCK__DF);
3650 		else
3651 			adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3652 						 1 << AMDGPU_RAS_BLOCK__SDMA |
3653 						 1 << AMDGPU_RAS_BLOCK__GFX);
3654 
3655 		/*
3656 		 * VCN/JPEG RAS can be supported on both bare metal and
3657 		 * SRIOV environment
3658 		 */
3659 		if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3660 		    amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
3661 		    amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3))
3662 			adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3663 						 1 << AMDGPU_RAS_BLOCK__JPEG);
3664 		else
3665 			adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3666 						  1 << AMDGPU_RAS_BLOCK__JPEG);
3667 
3668 		/*
3669 		 * XGMI RAS is not supported if xgmi num physical nodes
3670 		 * is zero
3671 		 */
3672 		if (!adev->gmc.xgmi.num_physical_nodes)
3673 			adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3674 	} else {
3675 		dev_info(adev->dev, "SRAM ECC is not presented.\n");
3676 	}
3677 }
3678 
3679 /* Query poison mode from umc/df IP callbacks */
amdgpu_ras_query_poison_mode(struct amdgpu_device * adev)3680 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3681 {
3682 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3683 	bool df_poison, umc_poison;
3684 
3685 	/* poison setting is useless on SRIOV guest */
3686 	if (amdgpu_sriov_vf(adev) || !con)
3687 		return;
3688 
3689 	/* Init poison supported flag, the default value is false */
3690 	if (adev->gmc.xgmi.connected_to_cpu ||
3691 	    adev->gmc.is_app_apu) {
3692 		/* enabled by default when GPU is connected to CPU */
3693 		con->poison_supported = true;
3694 	} else if (adev->df.funcs &&
3695 	    adev->df.funcs->query_ras_poison_mode &&
3696 	    adev->umc.ras &&
3697 	    adev->umc.ras->query_ras_poison_mode) {
3698 		df_poison =
3699 			adev->df.funcs->query_ras_poison_mode(adev);
3700 		umc_poison =
3701 			adev->umc.ras->query_ras_poison_mode(adev);
3702 
3703 		/* Only poison is set in both DF and UMC, we can support it */
3704 		if (df_poison && umc_poison)
3705 			con->poison_supported = true;
3706 		else if (df_poison != umc_poison)
3707 			dev_warn(adev->dev,
3708 				"Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3709 				df_poison, umc_poison);
3710 	}
3711 }
3712 
3713 /*
3714  * check hardware's ras ability which will be saved in hw_supported.
3715  * if hardware does not support ras, we can skip some ras initializtion and
3716  * forbid some ras operations from IP.
3717  * if software itself, say boot parameter, limit the ras ability. We still
3718  * need allow IP do some limited operations, like disable. In such case,
3719  * we have to initialize ras as normal. but need check if operation is
3720  * allowed or not in each function.
3721  */
amdgpu_ras_check_supported(struct amdgpu_device * adev)3722 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
3723 {
3724 	adev->ras_hw_enabled = adev->ras_enabled = 0;
3725 
3726 	if (!amdgpu_ras_asic_supported(adev))
3727 		return;
3728 
3729 	if (amdgpu_sriov_vf(adev)) {
3730 		if (amdgpu_virt_get_ras_capability(adev))
3731 			goto init_ras_enabled_flag;
3732 	}
3733 
3734 	/* query ras capability from psp */
3735 	if (amdgpu_psp_get_ras_capability(&adev->psp))
3736 		goto init_ras_enabled_flag;
3737 
3738 	/* query ras capablity from bios */
3739 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3740 		amdgpu_ras_query_ras_capablity_from_vbios(adev);
3741 	} else {
3742 		/* driver only manages a few IP blocks RAS feature
3743 		 * when GPU is connected cpu through XGMI */
3744 		adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
3745 					   1 << AMDGPU_RAS_BLOCK__SDMA |
3746 					   1 << AMDGPU_RAS_BLOCK__MMHUB);
3747 	}
3748 
3749 	/* apply asic specific settings (vega20 only for now) */
3750 	amdgpu_ras_get_quirks(adev);
3751 
3752 	/* query poison mode from umc/df ip callback */
3753 	amdgpu_ras_query_poison_mode(adev);
3754 
3755 init_ras_enabled_flag:
3756 	/* hw_supported needs to be aligned with RAS block mask. */
3757 	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
3758 
3759 	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3760 		adev->ras_hw_enabled & amdgpu_ras_mask;
3761 
3762 	/* aca is disabled by default */
3763 	adev->aca.is_enabled = false;
3764 
3765 	/* bad page feature is not applicable to specific app platform */
3766 	if (adev->gmc.is_app_apu &&
3767 	    amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3768 		amdgpu_bad_page_threshold = 0;
3769 }
3770 
amdgpu_ras_counte_dw(struct work_struct * work)3771 static void amdgpu_ras_counte_dw(struct work_struct *work)
3772 {
3773 	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3774 					      ras_counte_delay_work.work);
3775 	struct amdgpu_device *adev = con->adev;
3776 	struct drm_device *dev = adev_to_drm(adev);
3777 	unsigned long ce_count, ue_count;
3778 	int res;
3779 
3780 	res = pm_runtime_get_sync(dev->dev);
3781 	if (res < 0)
3782 		goto Out;
3783 
3784 	/* Cache new values.
3785 	 */
3786 	if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
3787 		atomic_set(&con->ras_ce_count, ce_count);
3788 		atomic_set(&con->ras_ue_count, ue_count);
3789 	}
3790 
3791 	pm_runtime_mark_last_busy(dev->dev);
3792 Out:
3793 	pm_runtime_put_autosuspend(dev->dev);
3794 }
3795 
amdgpu_get_ras_schema(struct amdgpu_device * adev)3796 static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3797 {
3798 	return  amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3799 			AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3800 			AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3801 			AMDGPU_RAS_ERROR__PARITY;
3802 }
3803 
ras_event_mgr_init(struct ras_event_manager * mgr)3804 static void ras_event_mgr_init(struct ras_event_manager *mgr)
3805 {
3806 	struct ras_event_state *event_state;
3807 	int i;
3808 
3809 	memset(mgr, 0, sizeof(*mgr));
3810 	atomic64_set(&mgr->seqno, 0);
3811 
3812 	for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3813 		event_state = &mgr->event_state[i];
3814 		event_state->last_seqno = RAS_EVENT_INVALID_ID;
3815 		atomic64_set(&event_state->count, 0);
3816 	}
3817 }
3818 
amdgpu_ras_event_mgr_init(struct amdgpu_device * adev)3819 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3820 {
3821 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3822 	struct amdgpu_hive_info *hive;
3823 
3824 	if (!ras)
3825 		return;
3826 
3827 	hive = amdgpu_get_xgmi_hive(adev);
3828 	ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3829 
3830 	/* init event manager with node 0 on xgmi system */
3831 	if (!amdgpu_reset_in_recovery(adev)) {
3832 		if (!hive || adev->gmc.xgmi.node_id == 0)
3833 			ras_event_mgr_init(ras->event_mgr);
3834 	}
3835 
3836 	if (hive)
3837 		amdgpu_put_xgmi_hive(hive);
3838 }
3839 
amdgpu_ras_init_reserved_vram_size(struct amdgpu_device * adev)3840 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3841 {
3842 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3843 
3844 	if (!con || (adev->flags & AMD_IS_APU))
3845 		return;
3846 
3847 	switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3848 	case IP_VERSION(13, 0, 2):
3849 	case IP_VERSION(13, 0, 6):
3850 	case IP_VERSION(13, 0, 12):
3851 	case IP_VERSION(13, 0, 14):
3852 		con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE;
3853 		break;
3854 	default:
3855 		break;
3856 	}
3857 }
3858 
amdgpu_ras_init(struct amdgpu_device * adev)3859 int amdgpu_ras_init(struct amdgpu_device *adev)
3860 {
3861 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3862 	int r;
3863 
3864 	if (con)
3865 		return 0;
3866 
3867 	con = kzalloc(sizeof(*con) +
3868 			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3869 			sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
3870 			GFP_KERNEL);
3871 	if (!con)
3872 		return -ENOMEM;
3873 
3874 	con->adev = adev;
3875 	INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3876 	atomic_set(&con->ras_ce_count, 0);
3877 	atomic_set(&con->ras_ue_count, 0);
3878 
3879 	con->objs = (struct ras_manager *)(con + 1);
3880 
3881 	amdgpu_ras_set_context(adev, con);
3882 
3883 	amdgpu_ras_check_supported(adev);
3884 
3885 	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
3886 		/* set gfx block ras context feature for VEGA20 Gaming
3887 		 * send ras disable cmd to ras ta during ras late init.
3888 		 */
3889 		if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
3890 			con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3891 
3892 			return 0;
3893 		}
3894 
3895 		r = 0;
3896 		goto release_con;
3897 	}
3898 
3899 	con->update_channel_flag = false;
3900 	con->features = 0;
3901 	con->schema = 0;
3902 	INIT_LIST_HEAD(&con->head);
3903 	/* Might need get this flag from vbios. */
3904 	con->flags = RAS_DEFAULT_FLAGS;
3905 
3906 	/* initialize nbio ras function ahead of any other
3907 	 * ras functions so hardware fatal error interrupt
3908 	 * can be enabled as early as possible */
3909 	switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
3910 	case IP_VERSION(7, 4, 0):
3911 	case IP_VERSION(7, 4, 1):
3912 	case IP_VERSION(7, 4, 4):
3913 		if (!adev->gmc.xgmi.connected_to_cpu)
3914 			adev->nbio.ras = &nbio_v7_4_ras;
3915 		break;
3916 	case IP_VERSION(4, 3, 0):
3917 		if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3918 			/* unlike other generation of nbio ras,
3919 			 * nbio v4_3 only support fatal error interrupt
3920 			 * to inform software that DF is freezed due to
3921 			 * system fatal error event. driver should not
3922 			 * enable nbio ras in such case. Instead,
3923 			 * check DF RAS */
3924 			adev->nbio.ras = &nbio_v4_3_ras;
3925 		break;
3926 	case IP_VERSION(6, 3, 1):
3927 		if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3928 			/* unlike other generation of nbio ras,
3929 			 * nbif v6_3_1 only support fatal error interrupt
3930 			 * to inform software that DF is freezed due to
3931 			 * system fatal error event. driver should not
3932 			 * enable nbio ras in such case. Instead,
3933 			 * check DF RAS
3934 			 */
3935 			adev->nbio.ras = &nbif_v6_3_1_ras;
3936 		break;
3937 	case IP_VERSION(7, 9, 0):
3938 	case IP_VERSION(7, 9, 1):
3939 		if (!adev->gmc.is_app_apu)
3940 			adev->nbio.ras = &nbio_v7_9_ras;
3941 		break;
3942 	default:
3943 		/* nbio ras is not available */
3944 		break;
3945 	}
3946 
3947 	/* nbio ras block needs to be enabled ahead of other ras blocks
3948 	 * to handle fatal error */
3949 	r = amdgpu_nbio_ras_sw_init(adev);
3950 	if (r)
3951 		return r;
3952 
3953 	if (adev->nbio.ras &&
3954 	    adev->nbio.ras->init_ras_controller_interrupt) {
3955 		r = adev->nbio.ras->init_ras_controller_interrupt(adev);
3956 		if (r)
3957 			goto release_con;
3958 	}
3959 
3960 	if (adev->nbio.ras &&
3961 	    adev->nbio.ras->init_ras_err_event_athub_interrupt) {
3962 		r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
3963 		if (r)
3964 			goto release_con;
3965 	}
3966 
3967 	/* Packed socket_id to ras feature mask bits[31:29] */
3968 	if (adev->smuio.funcs &&
3969 	    adev->smuio.funcs->get_socket_id)
3970 		con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
3971 					AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
3972 
3973 	/* Get RAS schema for particular SOC */
3974 	con->schema = amdgpu_get_ras_schema(adev);
3975 
3976 	amdgpu_ras_init_reserved_vram_size(adev);
3977 
3978 	if (amdgpu_ras_fs_init(adev)) {
3979 		r = -EINVAL;
3980 		goto release_con;
3981 	}
3982 
3983 	if (amdgpu_ras_aca_is_supported(adev)) {
3984 		if (amdgpu_aca_is_enabled(adev))
3985 			r = amdgpu_aca_init(adev);
3986 		else
3987 			r = amdgpu_mca_init(adev);
3988 		if (r)
3989 			goto release_con;
3990 	}
3991 
3992 	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
3993 		 "hardware ability[%x] ras_mask[%x]\n",
3994 		 adev->ras_hw_enabled, adev->ras_enabled);
3995 
3996 	return 0;
3997 release_con:
3998 	amdgpu_ras_set_context(adev, NULL);
3999 	kfree(con);
4000 
4001 	return r;
4002 }
4003 
amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device * adev)4004 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4005 {
4006 	if (adev->gmc.xgmi.connected_to_cpu ||
4007 	    adev->gmc.is_app_apu)
4008 		return 1;
4009 	return 0;
4010 }
4011 
amdgpu_persistent_edc_harvesting(struct amdgpu_device * adev,struct ras_common_if * ras_block)4012 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4013 					struct ras_common_if *ras_block)
4014 {
4015 	struct ras_query_if info = {
4016 		.head = *ras_block,
4017 	};
4018 
4019 	if (!amdgpu_persistent_edc_harvesting_supported(adev))
4020 		return 0;
4021 
4022 	if (amdgpu_ras_query_error_status(adev, &info) != 0)
4023 		DRM_WARN("RAS init harvest failure");
4024 
4025 	if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4026 		DRM_WARN("RAS init harvest reset failure");
4027 
4028 	return 0;
4029 }
4030 
amdgpu_ras_is_poison_mode_supported(struct amdgpu_device * adev)4031 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4032 {
4033        struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4034 
4035        if (!con)
4036                return false;
4037 
4038        return con->poison_supported;
4039 }
4040 
4041 /* helper function to handle common stuff in ip late init phase */
amdgpu_ras_block_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)4042 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4043 			 struct ras_common_if *ras_block)
4044 {
4045 	struct amdgpu_ras_block_object *ras_obj = NULL;
4046 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4047 	struct ras_query_if *query_info;
4048 	unsigned long ue_count, ce_count;
4049 	int r;
4050 
4051 	/* disable RAS feature per IP block if it is not supported */
4052 	if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4053 		amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4054 		return 0;
4055 	}
4056 
4057 	r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4058 	if (r) {
4059 		if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4060 			/* in resume phase, if fail to enable ras,
4061 			 * clean up all ras fs nodes, and disable ras */
4062 			goto cleanup;
4063 		} else
4064 			return r;
4065 	}
4066 
4067 	/* check for errors on warm reset edc persisant supported ASIC */
4068 	amdgpu_persistent_edc_harvesting(adev, ras_block);
4069 
4070 	/* in resume phase, no need to create ras fs node */
4071 	if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4072 		return 0;
4073 
4074 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4075 	if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4076 	    (ras_obj->hw_ops->query_poison_status ||
4077 	    ras_obj->hw_ops->handle_poison_consumption))) {
4078 		r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
4079 		if (r)
4080 			goto cleanup;
4081 	}
4082 
4083 	if (ras_obj->hw_ops &&
4084 	    (ras_obj->hw_ops->query_ras_error_count ||
4085 	     ras_obj->hw_ops->query_ras_error_status)) {
4086 		r = amdgpu_ras_sysfs_create(adev, ras_block);
4087 		if (r)
4088 			goto interrupt;
4089 
4090 		/* Those are the cached values at init.
4091 		 */
4092 		query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4093 		if (!query_info)
4094 			return -ENOMEM;
4095 		memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4096 
4097 		if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4098 			atomic_set(&con->ras_ce_count, ce_count);
4099 			atomic_set(&con->ras_ue_count, ue_count);
4100 		}
4101 
4102 		kfree(query_info);
4103 	}
4104 
4105 	return 0;
4106 
4107 interrupt:
4108 	if (ras_obj->ras_cb)
4109 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4110 cleanup:
4111 	amdgpu_ras_feature_enable(adev, ras_block, 0);
4112 	return r;
4113 }
4114 
amdgpu_ras_block_late_init_default(struct amdgpu_device * adev,struct ras_common_if * ras_block)4115 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4116 			 struct ras_common_if *ras_block)
4117 {
4118 	return amdgpu_ras_block_late_init(adev, ras_block);
4119 }
4120 
4121 /* helper function to remove ras fs node and interrupt handler */
amdgpu_ras_block_late_fini(struct amdgpu_device * adev,struct ras_common_if * ras_block)4122 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4123 			  struct ras_common_if *ras_block)
4124 {
4125 	struct amdgpu_ras_block_object *ras_obj;
4126 	if (!ras_block)
4127 		return;
4128 
4129 	amdgpu_ras_sysfs_remove(adev, ras_block);
4130 
4131 	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4132 	if (ras_obj->ras_cb)
4133 		amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4134 }
4135 
amdgpu_ras_block_late_fini_default(struct amdgpu_device * adev,struct ras_common_if * ras_block)4136 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4137 			  struct ras_common_if *ras_block)
4138 {
4139 	return amdgpu_ras_block_late_fini(adev, ras_block);
4140 }
4141 
4142 /* do some init work after IP late init as dependence.
4143  * and it runs in resume/gpu reset/booting up cases.
4144  */
amdgpu_ras_resume(struct amdgpu_device * adev)4145 void amdgpu_ras_resume(struct amdgpu_device *adev)
4146 {
4147 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4148 	struct ras_manager *obj, *tmp;
4149 
4150 	if (!adev->ras_enabled || !con) {
4151 		/* clean ras context for VEGA20 Gaming after send ras disable cmd */
4152 		amdgpu_release_ras_context(adev);
4153 
4154 		return;
4155 	}
4156 
4157 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4158 		/* Set up all other IPs which are not implemented. There is a
4159 		 * tricky thing that IP's actual ras error type should be
4160 		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4161 		 * ERROR_NONE make sense anyway.
4162 		 */
4163 		amdgpu_ras_enable_all_features(adev, 1);
4164 
4165 		/* We enable ras on all hw_supported block, but as boot
4166 		 * parameter might disable some of them and one or more IP has
4167 		 * not implemented yet. So we disable them on behalf.
4168 		 */
4169 		list_for_each_entry_safe(obj, tmp, &con->head, node) {
4170 			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4171 				amdgpu_ras_feature_enable(adev, &obj->head, 0);
4172 				/* there should be no any reference. */
4173 				WARN_ON(alive_obj(obj));
4174 			}
4175 		}
4176 	}
4177 }
4178 
amdgpu_ras_suspend(struct amdgpu_device * adev)4179 void amdgpu_ras_suspend(struct amdgpu_device *adev)
4180 {
4181 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4182 
4183 	if (!adev->ras_enabled || !con)
4184 		return;
4185 
4186 	amdgpu_ras_disable_all_features(adev, 0);
4187 	/* Make sure all ras objects are disabled. */
4188 	if (AMDGPU_RAS_GET_FEATURES(con->features))
4189 		amdgpu_ras_disable_all_features(adev, 1);
4190 }
4191 
amdgpu_ras_late_init(struct amdgpu_device * adev)4192 int amdgpu_ras_late_init(struct amdgpu_device *adev)
4193 {
4194 	struct amdgpu_ras_block_list *node, *tmp;
4195 	struct amdgpu_ras_block_object *obj;
4196 	int r;
4197 
4198 	amdgpu_ras_event_mgr_init(adev);
4199 
4200 	if (amdgpu_ras_aca_is_supported(adev)) {
4201 		if (amdgpu_reset_in_recovery(adev)) {
4202 			if (amdgpu_aca_is_enabled(adev))
4203 				r = amdgpu_aca_reset(adev);
4204 			else
4205 				r = amdgpu_mca_reset(adev);
4206 			if (r)
4207 				return r;
4208 		}
4209 
4210 		if (!amdgpu_sriov_vf(adev)) {
4211 			if (amdgpu_aca_is_enabled(adev))
4212 				amdgpu_ras_set_aca_debug_mode(adev, false);
4213 			else
4214 				amdgpu_ras_set_mca_debug_mode(adev, false);
4215 		}
4216 	}
4217 
4218 	/* Guest side doesn't need init ras feature */
4219 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4220 		return 0;
4221 
4222 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4223 		obj = node->ras_obj;
4224 		if (!obj) {
4225 			dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4226 			continue;
4227 		}
4228 
4229 		if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4230 			continue;
4231 
4232 		if (obj->ras_late_init) {
4233 			r = obj->ras_late_init(adev, &obj->ras_comm);
4234 			if (r) {
4235 				dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4236 					obj->ras_comm.name, r);
4237 				return r;
4238 			}
4239 		} else
4240 			amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
4241 	}
4242 
4243 	return 0;
4244 }
4245 
4246 /* do some fini work before IP fini as dependence */
amdgpu_ras_pre_fini(struct amdgpu_device * adev)4247 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4248 {
4249 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4250 
4251 	if (!adev->ras_enabled || !con)
4252 		return 0;
4253 
4254 
4255 	/* Need disable ras on all IPs here before ip [hw/sw]fini */
4256 	if (AMDGPU_RAS_GET_FEATURES(con->features))
4257 		amdgpu_ras_disable_all_features(adev, 0);
4258 	amdgpu_ras_recovery_fini(adev);
4259 	return 0;
4260 }
4261 
amdgpu_ras_fini(struct amdgpu_device * adev)4262 int amdgpu_ras_fini(struct amdgpu_device *adev)
4263 {
4264 	struct amdgpu_ras_block_list *ras_node, *tmp;
4265 	struct amdgpu_ras_block_object *obj = NULL;
4266 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4267 
4268 	if (!adev->ras_enabled || !con)
4269 		return 0;
4270 
4271 	list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4272 		if (ras_node->ras_obj) {
4273 			obj = ras_node->ras_obj;
4274 			if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4275 			    obj->ras_fini)
4276 				obj->ras_fini(adev, &obj->ras_comm);
4277 			else
4278 				amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
4279 		}
4280 
4281 		/* Clear ras blocks from ras_list and free ras block list node */
4282 		list_del(&ras_node->node);
4283 		kfree(ras_node);
4284 	}
4285 
4286 	amdgpu_ras_fs_fini(adev);
4287 	amdgpu_ras_interrupt_remove_all(adev);
4288 
4289 	if (amdgpu_ras_aca_is_supported(adev)) {
4290 		if (amdgpu_aca_is_enabled(adev))
4291 			amdgpu_aca_fini(adev);
4292 		else
4293 			amdgpu_mca_fini(adev);
4294 	}
4295 
4296 	WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4297 
4298 	if (AMDGPU_RAS_GET_FEATURES(con->features))
4299 		amdgpu_ras_disable_all_features(adev, 0);
4300 
4301 	cancel_delayed_work_sync(&con->ras_counte_delay_work);
4302 
4303 	amdgpu_ras_set_context(adev, NULL);
4304 	kfree(con);
4305 
4306 	return 0;
4307 }
4308 
amdgpu_ras_get_fed_status(struct amdgpu_device * adev)4309 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4310 {
4311 	struct amdgpu_ras *ras;
4312 
4313 	ras = amdgpu_ras_get_context(adev);
4314 	if (!ras)
4315 		return false;
4316 
4317 	return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4318 }
4319 
amdgpu_ras_set_fed(struct amdgpu_device * adev,bool status)4320 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4321 {
4322 	struct amdgpu_ras *ras;
4323 
4324 	ras = amdgpu_ras_get_context(adev);
4325 	if (ras) {
4326 		if (status)
4327 			set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4328 		else
4329 			clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4330 	}
4331 }
4332 
amdgpu_ras_clear_err_state(struct amdgpu_device * adev)4333 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4334 {
4335 	struct amdgpu_ras *ras;
4336 
4337 	ras = amdgpu_ras_get_context(adev);
4338 	if (ras)
4339 		ras->ras_err_state = 0;
4340 }
4341 
amdgpu_ras_set_err_poison(struct amdgpu_device * adev,enum amdgpu_ras_block block)4342 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4343 			       enum amdgpu_ras_block block)
4344 {
4345 	struct amdgpu_ras *ras;
4346 
4347 	ras = amdgpu_ras_get_context(adev);
4348 	if (ras)
4349 		set_bit(block, &ras->ras_err_state);
4350 }
4351 
amdgpu_ras_is_err_state(struct amdgpu_device * adev,int block)4352 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4353 {
4354 	struct amdgpu_ras *ras;
4355 
4356 	ras = amdgpu_ras_get_context(adev);
4357 	if (ras) {
4358 		if (block == AMDGPU_RAS_BLOCK__ANY)
4359 			return (ras->ras_err_state != 0);
4360 		else
4361 			return test_bit(block, &ras->ras_err_state) ||
4362 			       test_bit(AMDGPU_RAS_BLOCK__LAST,
4363 					&ras->ras_err_state);
4364 	}
4365 
4366 	return false;
4367 }
4368 
__get_ras_event_mgr(struct amdgpu_device * adev)4369 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4370 {
4371 	struct amdgpu_ras *ras;
4372 
4373 	ras = amdgpu_ras_get_context(adev);
4374 	if (!ras)
4375 		return NULL;
4376 
4377 	return ras->event_mgr;
4378 }
4379 
amdgpu_ras_mark_ras_event_caller(struct amdgpu_device * adev,enum ras_event_type type,const void * caller)4380 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4381 				     const void *caller)
4382 {
4383 	struct ras_event_manager *event_mgr;
4384 	struct ras_event_state *event_state;
4385 	int ret = 0;
4386 
4387 	if (type >= RAS_EVENT_TYPE_COUNT) {
4388 		ret = -EINVAL;
4389 		goto out;
4390 	}
4391 
4392 	event_mgr = __get_ras_event_mgr(adev);
4393 	if (!event_mgr) {
4394 		ret = -EINVAL;
4395 		goto out;
4396 	}
4397 
4398 	event_state = &event_mgr->event_state[type];
4399 	event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4400 	atomic64_inc(&event_state->count);
4401 
4402 out:
4403 	if (ret && caller)
4404 		dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4405 			 (int)type, caller, ret);
4406 
4407 	return ret;
4408 }
4409 
amdgpu_ras_acquire_event_id(struct amdgpu_device * adev,enum ras_event_type type)4410 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4411 {
4412 	struct ras_event_manager *event_mgr;
4413 	u64 id;
4414 
4415 	if (type >= RAS_EVENT_TYPE_COUNT)
4416 		return RAS_EVENT_INVALID_ID;
4417 
4418 	switch (type) {
4419 	case RAS_EVENT_TYPE_FATAL:
4420 	case RAS_EVENT_TYPE_POISON_CREATION:
4421 	case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4422 		event_mgr = __get_ras_event_mgr(adev);
4423 		if (!event_mgr)
4424 			return RAS_EVENT_INVALID_ID;
4425 
4426 		id = event_mgr->event_state[type].last_seqno;
4427 		break;
4428 	case RAS_EVENT_TYPE_INVALID:
4429 	default:
4430 		id = RAS_EVENT_INVALID_ID;
4431 		break;
4432 	}
4433 
4434 	return id;
4435 }
4436 
amdgpu_ras_global_ras_isr(struct amdgpu_device * adev)4437 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4438 {
4439 	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
4440 		struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4441 		enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4442 		u64 event_id;
4443 
4444 		if (amdgpu_ras_mark_ras_event(adev, type))
4445 			return;
4446 
4447 		event_id = amdgpu_ras_acquire_event_id(adev, type);
4448 
4449 		RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4450 			      "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4451 
4452 		amdgpu_ras_set_fed(adev, true);
4453 		ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4454 		amdgpu_ras_reset_gpu(adev);
4455 	}
4456 }
4457 
amdgpu_ras_need_emergency_restart(struct amdgpu_device * adev)4458 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4459 {
4460 	if (adev->asic_type == CHIP_VEGA20 &&
4461 	    adev->pm.fw_version <= 0x283400) {
4462 		return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4463 				amdgpu_ras_intr_triggered();
4464 	}
4465 
4466 	return false;
4467 }
4468 
amdgpu_release_ras_context(struct amdgpu_device * adev)4469 void amdgpu_release_ras_context(struct amdgpu_device *adev)
4470 {
4471 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4472 
4473 	if (!con)
4474 		return;
4475 
4476 	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4477 		con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4478 		amdgpu_ras_set_context(adev, NULL);
4479 		kfree(con);
4480 	}
4481 }
4482 
4483 #ifdef CONFIG_X86_MCE_AMD
find_adev(uint32_t node_id)4484 static struct amdgpu_device *find_adev(uint32_t node_id)
4485 {
4486 	int i;
4487 	struct amdgpu_device *adev = NULL;
4488 
4489 	for (i = 0; i < mce_adev_list.num_gpu; i++) {
4490 		adev = mce_adev_list.devs[i];
4491 
4492 		if (adev && adev->gmc.xgmi.connected_to_cpu &&
4493 		    adev->gmc.xgmi.physical_node_id == node_id)
4494 			break;
4495 		adev = NULL;
4496 	}
4497 
4498 	return adev;
4499 }
4500 
4501 #define GET_MCA_IPID_GPUID(m)	(((m) >> 44) & 0xF)
4502 #define GET_UMC_INST(m)		(((m) >> 21) & 0x7)
4503 #define GET_CHAN_INDEX(m)	((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4504 #define GPU_ID_OFFSET		8
4505 
amdgpu_bad_page_notifier(struct notifier_block * nb,unsigned long val,void * data)4506 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4507 				    unsigned long val, void *data)
4508 {
4509 	struct mce *m = (struct mce *)data;
4510 	struct amdgpu_device *adev = NULL;
4511 	uint32_t gpu_id = 0;
4512 	uint32_t umc_inst = 0, ch_inst = 0;
4513 
4514 	/*
4515 	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4516 	 * and error occurred in DramECC (Extended error code = 0) then only
4517 	 * process the error, else bail out.
4518 	 */
4519 	if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
4520 		    (XEC(m->status, 0x3f) == 0x0)))
4521 		return NOTIFY_DONE;
4522 
4523 	/*
4524 	 * If it is correctable error, return.
4525 	 */
4526 	if (mce_is_correctable(m))
4527 		return NOTIFY_OK;
4528 
4529 	/*
4530 	 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4531 	 */
4532 	gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4533 
4534 	adev = find_adev(gpu_id);
4535 	if (!adev) {
4536 		DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4537 								gpu_id);
4538 		return NOTIFY_DONE;
4539 	}
4540 
4541 	/*
4542 	 * If it is uncorrectable error, then find out UMC instance and
4543 	 * channel index.
4544 	 */
4545 	umc_inst = GET_UMC_INST(m->ipid);
4546 	ch_inst = GET_CHAN_INDEX(m->ipid);
4547 
4548 	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4549 			     umc_inst, ch_inst);
4550 
4551 	if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4552 		return NOTIFY_OK;
4553 	else
4554 		return NOTIFY_DONE;
4555 }
4556 
4557 static struct notifier_block amdgpu_bad_page_nb = {
4558 	.notifier_call  = amdgpu_bad_page_notifier,
4559 	.priority       = MCE_PRIO_UC,
4560 };
4561 
amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device * adev)4562 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4563 {
4564 	/*
4565 	 * Add the adev to the mce_adev_list.
4566 	 * During mode2 reset, amdgpu device is temporarily
4567 	 * removed from the mgpu_info list which can cause
4568 	 * page retirement to fail.
4569 	 * Use this list instead of mgpu_info to find the amdgpu
4570 	 * device on which the UMC error was reported.
4571 	 */
4572 	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4573 
4574 	/*
4575 	 * Register the x86 notifier only once
4576 	 * with MCE subsystem.
4577 	 */
4578 	if (notifier_registered == false) {
4579 		mce_register_decode_chain(&amdgpu_bad_page_nb);
4580 		notifier_registered = true;
4581 	}
4582 }
4583 #endif
4584 
amdgpu_ras_get_context(struct amdgpu_device * adev)4585 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
4586 {
4587 	if (!adev)
4588 		return NULL;
4589 
4590 	return adev->psp.ras_context.ras;
4591 }
4592 
amdgpu_ras_set_context(struct amdgpu_device * adev,struct amdgpu_ras * ras_con)4593 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
4594 {
4595 	if (!adev)
4596 		return -EINVAL;
4597 
4598 	adev->psp.ras_context.ras = ras_con;
4599 	return 0;
4600 }
4601 
4602 /* check if ras is supported on block, say, sdma, gfx */
amdgpu_ras_is_supported(struct amdgpu_device * adev,unsigned int block)4603 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4604 		unsigned int block)
4605 {
4606 	int ret = 0;
4607 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4608 
4609 	if (block >= AMDGPU_RAS_BLOCK_COUNT)
4610 		return 0;
4611 
4612 	ret = ras && (adev->ras_enabled & (1 << block));
4613 
4614 	/* For the special asic with mem ecc enabled but sram ecc
4615 	 * not enabled, even if the ras block is not supported on
4616 	 * .ras_enabled, if the asic supports poison mode and the
4617 	 * ras block has ras configuration, it can be considered
4618 	 * that the ras block supports ras function.
4619 	 */
4620 	if (!ret &&
4621 	    (block == AMDGPU_RAS_BLOCK__GFX ||
4622 	     block == AMDGPU_RAS_BLOCK__SDMA ||
4623 	     block == AMDGPU_RAS_BLOCK__VCN ||
4624 	     block == AMDGPU_RAS_BLOCK__JPEG) &&
4625 		(amdgpu_ras_mask & (1 << block)) &&
4626 	    amdgpu_ras_is_poison_mode_supported(adev) &&
4627 	    amdgpu_ras_get_ras_block(adev, block, 0))
4628 		ret = 1;
4629 
4630 	return ret;
4631 }
4632 
amdgpu_ras_reset_gpu(struct amdgpu_device * adev)4633 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4634 {
4635 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4636 
4637 	/* mode1 is the only selection for RMA status */
4638 	if (amdgpu_ras_is_rma(adev)) {
4639 		ras->gpu_reset_flags = 0;
4640 		ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4641 	}
4642 
4643 	if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4644 		struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4645 		int hive_ras_recovery = 0;
4646 
4647 		if (hive) {
4648 			hive_ras_recovery = atomic_read(&hive->ras_recovery);
4649 			amdgpu_put_xgmi_hive(hive);
4650 		}
4651 		/* In the case of multiple GPUs, after a GPU has started
4652 		 * resetting all GPUs on hive, other GPUs do not need to
4653 		 * trigger GPU reset again.
4654 		 */
4655 		if (!hive_ras_recovery)
4656 			amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4657 		else
4658 			atomic_set(&ras->in_recovery, 0);
4659 	} else {
4660 		flush_work(&ras->recovery_work);
4661 		amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4662 	}
4663 
4664 	return 0;
4665 }
4666 
amdgpu_ras_set_mca_debug_mode(struct amdgpu_device * adev,bool enable)4667 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
4668 {
4669 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4670 	int ret = 0;
4671 
4672 	if (con) {
4673 		ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4674 		if (!ret)
4675 			con->is_aca_debug_mode = enable;
4676 	}
4677 
4678 	return ret;
4679 }
4680 
amdgpu_ras_set_aca_debug_mode(struct amdgpu_device * adev,bool enable)4681 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4682 {
4683 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4684 	int ret = 0;
4685 
4686 	if (con) {
4687 		if (amdgpu_aca_is_enabled(adev))
4688 			ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4689 		else
4690 			ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4691 		if (!ret)
4692 			con->is_aca_debug_mode = enable;
4693 	}
4694 
4695 	return ret;
4696 }
4697 
amdgpu_ras_get_aca_debug_mode(struct amdgpu_device * adev)4698 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
4699 {
4700 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4701 	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4702 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4703 
4704 	if (!con)
4705 		return false;
4706 
4707 	if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4708 	    (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4709 		return con->is_aca_debug_mode;
4710 	else
4711 		return true;
4712 }
4713 
amdgpu_ras_get_error_query_mode(struct amdgpu_device * adev,unsigned int * error_query_mode)4714 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4715 				     unsigned int *error_query_mode)
4716 {
4717 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4718 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4719 	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4720 
4721 	if (!con) {
4722 		*error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4723 		return false;
4724 	}
4725 
4726 	if (amdgpu_sriov_vf(adev)) {
4727 		*error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4728 	} else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
4729 		*error_query_mode =
4730 			(con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
4731 	} else {
4732 		*error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
4733 	}
4734 
4735 	return true;
4736 }
4737 
4738 /* Register each ip ras block into amdgpu ras */
amdgpu_ras_register_ras_block(struct amdgpu_device * adev,struct amdgpu_ras_block_object * ras_block_obj)4739 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
4740 		struct amdgpu_ras_block_object *ras_block_obj)
4741 {
4742 	struct amdgpu_ras_block_list *ras_node;
4743 	if (!adev || !ras_block_obj)
4744 		return -EINVAL;
4745 
4746 	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4747 	if (!ras_node)
4748 		return -ENOMEM;
4749 
4750 	INIT_LIST_HEAD(&ras_node->node);
4751 	ras_node->ras_obj = ras_block_obj;
4752 	list_add_tail(&ras_node->node, &adev->ras_list);
4753 
4754 	return 0;
4755 }
4756 
amdgpu_ras_get_error_type_name(uint32_t err_type,char * err_type_name)4757 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4758 {
4759 	if (!err_type_name)
4760 		return;
4761 
4762 	switch (err_type) {
4763 	case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4764 		sprintf(err_type_name, "correctable");
4765 		break;
4766 	case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4767 		sprintf(err_type_name, "uncorrectable");
4768 		break;
4769 	default:
4770 		sprintf(err_type_name, "unknown");
4771 		break;
4772 	}
4773 }
4774 
amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_entry,uint32_t instance,uint32_t * memory_id)4775 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4776 					 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4777 					 uint32_t instance,
4778 					 uint32_t *memory_id)
4779 {
4780 	uint32_t err_status_lo_data, err_status_lo_offset;
4781 
4782 	if (!reg_entry)
4783 		return false;
4784 
4785 	err_status_lo_offset =
4786 		AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4787 					    reg_entry->seg_lo, reg_entry->reg_lo);
4788 	err_status_lo_data = RREG32(err_status_lo_offset);
4789 
4790 	if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4791 	    !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4792 		return false;
4793 
4794 	*memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4795 
4796 	return true;
4797 }
4798 
amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_entry,uint32_t instance,unsigned long * err_cnt)4799 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4800 				       const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4801 				       uint32_t instance,
4802 				       unsigned long *err_cnt)
4803 {
4804 	uint32_t err_status_hi_data, err_status_hi_offset;
4805 
4806 	if (!reg_entry)
4807 		return false;
4808 
4809 	err_status_hi_offset =
4810 		AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4811 					    reg_entry->seg_hi, reg_entry->reg_hi);
4812 	err_status_hi_data = RREG32(err_status_hi_offset);
4813 
4814 	if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4815 	    !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
4816 		/* keep the check here in case we need to refer to the result later */
4817 		dev_dbg(adev->dev, "Invalid err_info field\n");
4818 
4819 	/* read err count */
4820 	*err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4821 
4822 	return true;
4823 }
4824 
amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_list,uint32_t reg_list_size,const struct amdgpu_ras_memory_id_entry * mem_list,uint32_t mem_list_size,uint32_t instance,uint32_t err_type,unsigned long * err_count)4825 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4826 					   const struct amdgpu_ras_err_status_reg_entry *reg_list,
4827 					   uint32_t reg_list_size,
4828 					   const struct amdgpu_ras_memory_id_entry *mem_list,
4829 					   uint32_t mem_list_size,
4830 					   uint32_t instance,
4831 					   uint32_t err_type,
4832 					   unsigned long *err_count)
4833 {
4834 	uint32_t memory_id;
4835 	unsigned long err_cnt;
4836 	char err_type_name[16];
4837 	uint32_t i, j;
4838 
4839 	for (i = 0; i < reg_list_size; i++) {
4840 		/* query memory_id from err_status_lo */
4841 		if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
4842 							 instance, &memory_id))
4843 			continue;
4844 
4845 		/* query err_cnt from err_status_hi */
4846 		if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
4847 						       instance, &err_cnt) ||
4848 		    !err_cnt)
4849 			continue;
4850 
4851 		*err_count += err_cnt;
4852 
4853 		/* log the errors */
4854 		amdgpu_ras_get_error_type_name(err_type, err_type_name);
4855 		if (!mem_list) {
4856 			/* memory_list is not supported */
4857 			dev_info(adev->dev,
4858 				 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4859 				 err_cnt, err_type_name,
4860 				 reg_list[i].block_name,
4861 				 instance, memory_id);
4862 		} else {
4863 			for (j = 0; j < mem_list_size; j++) {
4864 				if (memory_id == mem_list[j].memory_id) {
4865 					dev_info(adev->dev,
4866 						 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4867 						 err_cnt, err_type_name,
4868 						 reg_list[i].block_name,
4869 						 instance, mem_list[j].name);
4870 					break;
4871 				}
4872 			}
4873 		}
4874 	}
4875 }
4876 
amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_list,uint32_t reg_list_size,uint32_t instance)4877 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4878 					   const struct amdgpu_ras_err_status_reg_entry *reg_list,
4879 					   uint32_t reg_list_size,
4880 					   uint32_t instance)
4881 {
4882 	uint32_t err_status_lo_offset, err_status_hi_offset;
4883 	uint32_t i;
4884 
4885 	for (i = 0; i < reg_list_size; i++) {
4886 		err_status_lo_offset =
4887 			AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4888 						    reg_list[i].seg_lo, reg_list[i].reg_lo);
4889 		err_status_hi_offset =
4890 			AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4891 						    reg_list[i].seg_hi, reg_list[i].reg_hi);
4892 		WREG32(err_status_lo_offset, 0);
4893 		WREG32(err_status_hi_offset, 0);
4894 	}
4895 }
4896 
amdgpu_ras_error_data_init(struct ras_err_data * err_data)4897 int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
4898 {
4899 	memset(err_data, 0, sizeof(*err_data));
4900 
4901 	INIT_LIST_HEAD(&err_data->err_node_list);
4902 
4903 	return 0;
4904 }
4905 
amdgpu_ras_error_node_release(struct ras_err_node * err_node)4906 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
4907 {
4908 	if (!err_node)
4909 		return;
4910 
4911 	list_del(&err_node->node);
4912 	kvfree(err_node);
4913 }
4914 
amdgpu_ras_error_data_fini(struct ras_err_data * err_data)4915 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
4916 {
4917 	struct ras_err_node *err_node, *tmp;
4918 
4919 	list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
4920 		amdgpu_ras_error_node_release(err_node);
4921 }
4922 
amdgpu_ras_error_find_node_by_id(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info)4923 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
4924 							     struct amdgpu_smuio_mcm_config_info *mcm_info)
4925 {
4926 	struct ras_err_node *err_node;
4927 	struct amdgpu_smuio_mcm_config_info *ref_id;
4928 
4929 	if (!err_data || !mcm_info)
4930 		return NULL;
4931 
4932 	for_each_ras_error(err_node, err_data) {
4933 		ref_id = &err_node->err_info.mcm_info;
4934 
4935 		if (mcm_info->socket_id == ref_id->socket_id &&
4936 		    mcm_info->die_id == ref_id->die_id)
4937 			return err_node;
4938 	}
4939 
4940 	return NULL;
4941 }
4942 
amdgpu_ras_error_node_new(void)4943 static struct ras_err_node *amdgpu_ras_error_node_new(void)
4944 {
4945 	struct ras_err_node *err_node;
4946 
4947 	err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
4948 	if (!err_node)
4949 		return NULL;
4950 
4951 	INIT_LIST_HEAD(&err_node->node);
4952 
4953 	return err_node;
4954 }
4955 
ras_err_info_cmp(void * priv,const struct list_head * a,const struct list_head * b)4956 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
4957 {
4958 	struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
4959 	struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
4960 	struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
4961 	struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
4962 
4963 	if (unlikely(infoa->socket_id != infob->socket_id))
4964 		return infoa->socket_id - infob->socket_id;
4965 	else
4966 		return infoa->die_id - infob->die_id;
4967 
4968 	return 0;
4969 }
4970 
amdgpu_ras_error_get_info(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info)4971 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
4972 				struct amdgpu_smuio_mcm_config_info *mcm_info)
4973 {
4974 	struct ras_err_node *err_node;
4975 
4976 	err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
4977 	if (err_node)
4978 		return &err_node->err_info;
4979 
4980 	err_node = amdgpu_ras_error_node_new();
4981 	if (!err_node)
4982 		return NULL;
4983 
4984 	memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
4985 
4986 	err_data->err_list_count++;
4987 	list_add_tail(&err_node->node, &err_data->err_node_list);
4988 	list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
4989 
4990 	return &err_node->err_info;
4991 }
4992 
amdgpu_ras_error_statistic_ue_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)4993 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
4994 					struct amdgpu_smuio_mcm_config_info *mcm_info,
4995 					u64 count)
4996 {
4997 	struct ras_err_info *err_info;
4998 
4999 	if (!err_data || !mcm_info)
5000 		return -EINVAL;
5001 
5002 	if (!count)
5003 		return 0;
5004 
5005 	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5006 	if (!err_info)
5007 		return -EINVAL;
5008 
5009 	err_info->ue_count += count;
5010 	err_data->ue_count += count;
5011 
5012 	return 0;
5013 }
5014 
amdgpu_ras_error_statistic_ce_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)5015 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5016 					struct amdgpu_smuio_mcm_config_info *mcm_info,
5017 					u64 count)
5018 {
5019 	struct ras_err_info *err_info;
5020 
5021 	if (!err_data || !mcm_info)
5022 		return -EINVAL;
5023 
5024 	if (!count)
5025 		return 0;
5026 
5027 	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5028 	if (!err_info)
5029 		return -EINVAL;
5030 
5031 	err_info->ce_count += count;
5032 	err_data->ce_count += count;
5033 
5034 	return 0;
5035 }
5036 
amdgpu_ras_error_statistic_de_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)5037 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5038 					struct amdgpu_smuio_mcm_config_info *mcm_info,
5039 					u64 count)
5040 {
5041 	struct ras_err_info *err_info;
5042 
5043 	if (!err_data || !mcm_info)
5044 		return -EINVAL;
5045 
5046 	if (!count)
5047 		return 0;
5048 
5049 	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5050 	if (!err_info)
5051 		return -EINVAL;
5052 
5053 	err_info->de_count += count;
5054 	err_data->de_count += count;
5055 
5056 	return 0;
5057 }
5058 
5059 #define mmMP0_SMN_C2PMSG_92	0x1609C
5060 #define mmMP0_SMN_C2PMSG_126	0x160BE
amdgpu_ras_boot_time_error_reporting(struct amdgpu_device * adev,u32 instance)5061 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5062 						 u32 instance)
5063 {
5064 	u32 socket_id, aid_id, hbm_id;
5065 	u32 fw_status;
5066 	u32 boot_error;
5067 	u64 reg_addr;
5068 
5069 	/* The pattern for smn addressing in other SOC could be different from
5070 	 * the one for aqua_vanjaram. We should revisit the code if the pattern
5071 	 * is changed. In such case, replace the aqua_vanjaram implementation
5072 	 * with more common helper */
5073 	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5074 		   aqua_vanjaram_encode_ext_smn_addressing(instance);
5075 	fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5076 
5077 	reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5078 		   aqua_vanjaram_encode_ext_smn_addressing(instance);
5079 	boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5080 
5081 	socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5082 	aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5083 	hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5084 
5085 	if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5086 		dev_info(adev->dev,
5087 			 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5088 			 socket_id, aid_id, hbm_id, fw_status);
5089 
5090 	if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5091 		dev_info(adev->dev,
5092 			 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5093 			 socket_id, aid_id, fw_status);
5094 
5095 	if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5096 		dev_info(adev->dev,
5097 			 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5098 			 socket_id, aid_id, fw_status);
5099 
5100 	if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5101 		dev_info(adev->dev,
5102 			 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5103 			 socket_id, aid_id, fw_status);
5104 
5105 	if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5106 		dev_info(adev->dev,
5107 			 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5108 			 socket_id, aid_id, fw_status);
5109 
5110 	if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5111 		dev_info(adev->dev,
5112 			 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5113 			 socket_id, aid_id, fw_status);
5114 
5115 	if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5116 		dev_info(adev->dev,
5117 			 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5118 			 socket_id, aid_id, hbm_id, fw_status);
5119 
5120 	if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5121 		dev_info(adev->dev,
5122 			 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5123 			 socket_id, aid_id, hbm_id, fw_status);
5124 
5125 	if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5126 		dev_info(adev->dev,
5127 			 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5128 			 socket_id, aid_id, fw_status);
5129 
5130 	if (AMDGPU_RAS_GPU_ERR_UNKNOWN(boot_error))
5131 		dev_info(adev->dev,
5132 			 "socket: %d, aid: %d, fw_status: 0x%x, unknown boot time errors\n",
5133 			 socket_id, aid_id, fw_status);
5134 }
5135 
amdgpu_ras_boot_error_detected(struct amdgpu_device * adev,u32 instance)5136 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5137 					   u32 instance)
5138 {
5139 	u64 reg_addr;
5140 	u32 reg_data;
5141 	int retry_loop;
5142 
5143 	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5144 		   aqua_vanjaram_encode_ext_smn_addressing(instance);
5145 
5146 	for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5147 		reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5148 		if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5149 			return false;
5150 		else
5151 			msleep(1);
5152 	}
5153 
5154 	return true;
5155 }
5156 
amdgpu_ras_query_boot_status(struct amdgpu_device * adev,u32 num_instances)5157 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5158 {
5159 	u32 i;
5160 
5161 	for (i = 0; i < num_instances; i++) {
5162 		if (amdgpu_ras_boot_error_detected(adev, i))
5163 			amdgpu_ras_boot_time_error_reporting(adev, i);
5164 	}
5165 }
5166 
amdgpu_ras_reserve_page(struct amdgpu_device * adev,uint64_t pfn)5167 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5168 {
5169 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5170 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5171 	uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5172 	int ret = 0;
5173 
5174 	mutex_lock(&con->page_rsv_lock);
5175 	ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5176 	if (ret == -ENOENT)
5177 		ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5178 	mutex_unlock(&con->page_rsv_lock);
5179 
5180 	return ret;
5181 }
5182 
amdgpu_ras_event_log_print(struct amdgpu_device * adev,u64 event_id,const char * fmt,...)5183 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5184 				const char *fmt, ...)
5185 {
5186 	struct va_format vaf;
5187 	va_list args;
5188 
5189 	va_start(args, fmt);
5190 	vaf.fmt = fmt;
5191 	vaf.va = &args;
5192 
5193 	if (RAS_EVENT_ID_IS_VALID(event_id))
5194 		dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5195 	else
5196 		dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5197 
5198 	va_end(args);
5199 }
5200 
amdgpu_ras_is_rma(struct amdgpu_device * adev)5201 bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5202 {
5203 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5204 
5205 	if (!con)
5206 		return false;
5207 
5208 	return con->is_rma;
5209 }
5210