1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/debugfs.h>
25 #include <linux/list.h>
26 #include <linux/module.h>
27 #include <linux/uaccess.h>
28 #include <linux/reboot.h>
29 #include <linux/syscalls.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/list_sort.h>
32
33 #include "amdgpu.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_atomfirmware.h"
36 #include "amdgpu_xgmi.h"
37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
38 #include "nbio_v4_3.h"
39 #include "nbif_v6_3_1.h"
40 #include "nbio_v7_9.h"
41 #include "atom.h"
42 #include "amdgpu_reset.h"
43 #include "amdgpu_psp.h"
44
45 #ifdef CONFIG_X86_MCE_AMD
46 #include <asm/mce.h>
47
48 static bool notifier_registered;
49 #endif
50 static const char *RAS_FS_NAME = "ras";
51
52 const char *ras_error_string[] = {
53 "none",
54 "parity",
55 "single_correctable",
56 "multi_uncorrectable",
57 "poison",
58 };
59
60 const char *ras_block_string[] = {
61 "umc",
62 "sdma",
63 "gfx",
64 "mmhub",
65 "athub",
66 "pcie_bif",
67 "hdp",
68 "xgmi_wafl",
69 "df",
70 "smn",
71 "sem",
72 "mp0",
73 "mp1",
74 "fuse",
75 "mca",
76 "vcn",
77 "jpeg",
78 "ih",
79 "mpio",
80 };
81
82 const char *ras_mca_block_string[] = {
83 "mca_mp0",
84 "mca_mp1",
85 "mca_mpio",
86 "mca_iohc",
87 };
88
89 struct amdgpu_ras_block_list {
90 /* ras block link */
91 struct list_head node;
92
93 struct amdgpu_ras_block_object *ras_obj;
94 };
95
get_ras_block_str(struct ras_common_if * ras_block)96 const char *get_ras_block_str(struct ras_common_if *ras_block)
97 {
98 if (!ras_block)
99 return "NULL";
100
101 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
102 ras_block->block >= ARRAY_SIZE(ras_block_string))
103 return "OUT OF RANGE";
104
105 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
106 return ras_mca_block_string[ras_block->sub_block_index];
107
108 return ras_block_string[ras_block->block];
109 }
110
111 #define ras_block_str(_BLOCK_) \
112 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
113
114 #define ras_err_str(i) (ras_error_string[ffs(i)])
115
116 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
117
118 /* inject address is 52 bits */
119 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
120
121 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
122 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
123
124 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms
125
126 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms
127
128 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100
129
130 enum amdgpu_ras_retire_page_reservation {
131 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
132 AMDGPU_RAS_RETIRE_PAGE_PENDING,
133 AMDGPU_RAS_RETIRE_PAGE_FAULT,
134 };
135
136 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
137
138 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
139 uint64_t addr);
140 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
141 uint64_t addr);
142 #ifdef CONFIG_X86_MCE_AMD
143 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
144 struct mce_notifier_adev_list {
145 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
146 int num_gpu;
147 };
148 static struct mce_notifier_adev_list mce_adev_list;
149 #endif
150
amdgpu_ras_set_error_query_ready(struct amdgpu_device * adev,bool ready)151 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
152 {
153 if (adev && amdgpu_ras_get_context(adev))
154 amdgpu_ras_get_context(adev)->error_query_ready = ready;
155 }
156
amdgpu_ras_get_error_query_ready(struct amdgpu_device * adev)157 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
158 {
159 if (adev && amdgpu_ras_get_context(adev))
160 return amdgpu_ras_get_context(adev)->error_query_ready;
161
162 return false;
163 }
164
amdgpu_reserve_page_direct(struct amdgpu_device * adev,uint64_t address)165 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
166 {
167 struct ras_err_data err_data;
168 struct eeprom_table_record err_rec;
169 int ret;
170
171 if ((address >= adev->gmc.mc_vram_size) ||
172 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
173 dev_warn(adev->dev,
174 "RAS WARN: input address 0x%llx is invalid.\n",
175 address);
176 return -EINVAL;
177 }
178
179 if (amdgpu_ras_check_bad_page(adev, address)) {
180 dev_warn(adev->dev,
181 "RAS WARN: 0x%llx has already been marked as bad page!\n",
182 address);
183 return 0;
184 }
185
186 ret = amdgpu_ras_error_data_init(&err_data);
187 if (ret)
188 return ret;
189
190 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
191 err_data.err_addr = &err_rec;
192 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
193
194 if (amdgpu_bad_page_threshold != 0) {
195 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
196 err_data.err_addr_cnt, false);
197 amdgpu_ras_save_bad_pages(adev, NULL);
198 }
199
200 amdgpu_ras_error_data_fini(&err_data);
201
202 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
203 dev_warn(adev->dev, "Clear EEPROM:\n");
204 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
205
206 return 0;
207 }
208
amdgpu_ras_debugfs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)209 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
210 size_t size, loff_t *pos)
211 {
212 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
213 struct ras_query_if info = {
214 .head = obj->head,
215 };
216 ssize_t s;
217 char val[128];
218
219 if (amdgpu_ras_query_error_status(obj->adev, &info))
220 return -EINVAL;
221
222 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
223 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
224 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
225 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
226 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
227 }
228
229 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
230 "ue", info.ue_count,
231 "ce", info.ce_count);
232 if (*pos >= s)
233 return 0;
234
235 s -= *pos;
236 s = min_t(u64, s, size);
237
238
239 if (copy_to_user(buf, &val[*pos], s))
240 return -EINVAL;
241
242 *pos += s;
243
244 return s;
245 }
246
247 static const struct file_operations amdgpu_ras_debugfs_ops = {
248 .owner = THIS_MODULE,
249 .read = amdgpu_ras_debugfs_read,
250 .write = NULL,
251 .llseek = default_llseek
252 };
253
amdgpu_ras_find_block_id_by_name(const char * name,int * block_id)254 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
255 {
256 int i;
257
258 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
259 *block_id = i;
260 if (strcmp(name, ras_block_string[i]) == 0)
261 return 0;
262 }
263 return -EINVAL;
264 }
265
amdgpu_ras_debugfs_ctrl_parse_data(struct file * f,const char __user * buf,size_t size,loff_t * pos,struct ras_debug_if * data)266 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
267 const char __user *buf, size_t size,
268 loff_t *pos, struct ras_debug_if *data)
269 {
270 ssize_t s = min_t(u64, 64, size);
271 char str[65];
272 char block_name[33];
273 char err[9] = "ue";
274 int op = -1;
275 int block_id;
276 uint32_t sub_block;
277 u64 address, value;
278 /* default value is 0 if the mask is not set by user */
279 u32 instance_mask = 0;
280
281 if (*pos)
282 return -EINVAL;
283 *pos = size;
284
285 memset(str, 0, sizeof(str));
286 memset(data, 0, sizeof(*data));
287
288 if (copy_from_user(str, buf, s))
289 return -EINVAL;
290
291 if (sscanf(str, "disable %32s", block_name) == 1)
292 op = 0;
293 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
294 op = 1;
295 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
296 op = 2;
297 else if (strstr(str, "retire_page") != NULL)
298 op = 3;
299 else if (str[0] && str[1] && str[2] && str[3])
300 /* ascii string, but commands are not matched. */
301 return -EINVAL;
302
303 if (op != -1) {
304 if (op == 3) {
305 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
306 sscanf(str, "%*s %llu", &address) != 1)
307 return -EINVAL;
308
309 data->op = op;
310 data->inject.address = address;
311
312 return 0;
313 }
314
315 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
316 return -EINVAL;
317
318 data->head.block = block_id;
319 /* only ue, ce and poison errors are supported */
320 if (!memcmp("ue", err, 2))
321 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
322 else if (!memcmp("ce", err, 2))
323 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
324 else if (!memcmp("poison", err, 6))
325 data->head.type = AMDGPU_RAS_ERROR__POISON;
326 else
327 return -EINVAL;
328
329 data->op = op;
330
331 if (op == 2) {
332 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
333 &sub_block, &address, &value, &instance_mask) != 4 &&
334 sscanf(str, "%*s %*s %*s %u %llu %llu %u",
335 &sub_block, &address, &value, &instance_mask) != 4 &&
336 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
337 &sub_block, &address, &value) != 3 &&
338 sscanf(str, "%*s %*s %*s %u %llu %llu",
339 &sub_block, &address, &value) != 3)
340 return -EINVAL;
341 data->head.sub_block_index = sub_block;
342 data->inject.address = address;
343 data->inject.value = value;
344 data->inject.instance_mask = instance_mask;
345 }
346 } else {
347 if (size < sizeof(*data))
348 return -EINVAL;
349
350 if (copy_from_user(data, buf, sizeof(*data)))
351 return -EINVAL;
352 }
353
354 return 0;
355 }
356
amdgpu_ras_instance_mask_check(struct amdgpu_device * adev,struct ras_debug_if * data)357 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
358 struct ras_debug_if *data)
359 {
360 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
361 uint32_t mask, inst_mask = data->inject.instance_mask;
362
363 /* no need to set instance mask if there is only one instance */
364 if (num_xcc <= 1 && inst_mask) {
365 data->inject.instance_mask = 0;
366 dev_dbg(adev->dev,
367 "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
368 inst_mask);
369
370 return;
371 }
372
373 switch (data->head.block) {
374 case AMDGPU_RAS_BLOCK__GFX:
375 mask = GENMASK(num_xcc - 1, 0);
376 break;
377 case AMDGPU_RAS_BLOCK__SDMA:
378 mask = GENMASK(adev->sdma.num_instances - 1, 0);
379 break;
380 case AMDGPU_RAS_BLOCK__VCN:
381 case AMDGPU_RAS_BLOCK__JPEG:
382 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
383 break;
384 default:
385 mask = inst_mask;
386 break;
387 }
388
389 /* remove invalid bits in instance mask */
390 data->inject.instance_mask &= mask;
391 if (inst_mask != data->inject.instance_mask)
392 dev_dbg(adev->dev,
393 "Adjust RAS inject mask 0x%x to 0x%x\n",
394 inst_mask, data->inject.instance_mask);
395 }
396
397 /**
398 * DOC: AMDGPU RAS debugfs control interface
399 *
400 * The control interface accepts struct ras_debug_if which has two members.
401 *
402 * First member: ras_debug_if::head or ras_debug_if::inject.
403 *
404 * head is used to indicate which IP block will be under control.
405 *
406 * head has four members, they are block, type, sub_block_index, name.
407 * block: which IP will be under control.
408 * type: what kind of error will be enabled/disabled/injected.
409 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
410 * name: the name of IP.
411 *
412 * inject has three more members than head, they are address, value and mask.
413 * As their names indicate, inject operation will write the
414 * value to the address.
415 *
416 * The second member: struct ras_debug_if::op.
417 * It has three kinds of operations.
418 *
419 * - 0: disable RAS on the block. Take ::head as its data.
420 * - 1: enable RAS on the block. Take ::head as its data.
421 * - 2: inject errors on the block. Take ::inject as its data.
422 *
423 * How to use the interface?
424 *
425 * In a program
426 *
427 * Copy the struct ras_debug_if in your code and initialize it.
428 * Write the struct to the control interface.
429 *
430 * From shell
431 *
432 * .. code-block:: bash
433 *
434 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
435 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
436 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
437 *
438 * Where N, is the card which you want to affect.
439 *
440 * "disable" requires only the block.
441 * "enable" requires the block and error type.
442 * "inject" requires the block, error type, address, and value.
443 *
444 * The block is one of: umc, sdma, gfx, etc.
445 * see ras_block_string[] for details
446 *
447 * The error type is one of: ue, ce and poison where,
448 * ue is multi-uncorrectable
449 * ce is single-correctable
450 * poison is poison
451 *
452 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
453 * The address and value are hexadecimal numbers, leading 0x is optional.
454 * The mask means instance mask, is optional, default value is 0x1.
455 *
456 * For instance,
457 *
458 * .. code-block:: bash
459 *
460 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
461 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
462 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
463 *
464 * How to check the result of the operation?
465 *
466 * To check disable/enable, see "ras" features at,
467 * /sys/class/drm/card[0/1/2...]/device/ras/features
468 *
469 * To check inject, see the corresponding error count at,
470 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
471 *
472 * .. note::
473 * Operations are only allowed on blocks which are supported.
474 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
475 * to see which blocks support RAS on a particular asic.
476 *
477 */
amdgpu_ras_debugfs_ctrl_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)478 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
479 const char __user *buf,
480 size_t size, loff_t *pos)
481 {
482 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
483 struct ras_debug_if data;
484 int ret = 0;
485
486 if (!amdgpu_ras_get_error_query_ready(adev)) {
487 dev_warn(adev->dev, "RAS WARN: error injection "
488 "currently inaccessible\n");
489 return size;
490 }
491
492 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
493 if (ret)
494 return ret;
495
496 if (data.op == 3) {
497 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
498 if (!ret)
499 return size;
500 else
501 return ret;
502 }
503
504 if (!amdgpu_ras_is_supported(adev, data.head.block))
505 return -EINVAL;
506
507 switch (data.op) {
508 case 0:
509 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
510 break;
511 case 1:
512 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
513 break;
514 case 2:
515 if ((data.inject.address >= adev->gmc.mc_vram_size &&
516 adev->gmc.mc_vram_size) ||
517 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
518 dev_warn(adev->dev, "RAS WARN: input address "
519 "0x%llx is invalid.",
520 data.inject.address);
521 ret = -EINVAL;
522 break;
523 }
524
525 /* umc ce/ue error injection for a bad page is not allowed */
526 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
527 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
528 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
529 "already been marked as bad!\n",
530 data.inject.address);
531 break;
532 }
533
534 amdgpu_ras_instance_mask_check(adev, &data);
535
536 /* data.inject.address is offset instead of absolute gpu address */
537 ret = amdgpu_ras_error_inject(adev, &data.inject);
538 break;
539 default:
540 ret = -EINVAL;
541 break;
542 }
543
544 if (ret)
545 return ret;
546
547 return size;
548 }
549
550 /**
551 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
552 *
553 * Some boards contain an EEPROM which is used to persistently store a list of
554 * bad pages which experiences ECC errors in vram. This interface provides
555 * a way to reset the EEPROM, e.g., after testing error injection.
556 *
557 * Usage:
558 *
559 * .. code-block:: bash
560 *
561 * echo 1 > ../ras/ras_eeprom_reset
562 *
563 * will reset EEPROM table to 0 entries.
564 *
565 */
amdgpu_ras_debugfs_eeprom_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)566 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
567 const char __user *buf,
568 size_t size, loff_t *pos)
569 {
570 struct amdgpu_device *adev =
571 (struct amdgpu_device *)file_inode(f)->i_private;
572 int ret;
573
574 ret = amdgpu_ras_eeprom_reset_table(
575 &(amdgpu_ras_get_context(adev)->eeprom_control));
576
577 if (!ret) {
578 /* Something was written to EEPROM.
579 */
580 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
581 return size;
582 } else {
583 return ret;
584 }
585 }
586
587 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
588 .owner = THIS_MODULE,
589 .read = NULL,
590 .write = amdgpu_ras_debugfs_ctrl_write,
591 .llseek = default_llseek
592 };
593
594 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
595 .owner = THIS_MODULE,
596 .read = NULL,
597 .write = amdgpu_ras_debugfs_eeprom_write,
598 .llseek = default_llseek
599 };
600
601 /**
602 * DOC: AMDGPU RAS sysfs Error Count Interface
603 *
604 * It allows the user to read the error count for each IP block on the gpu through
605 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
606 *
607 * It outputs the multiple lines which report the uncorrected (ue) and corrected
608 * (ce) error counts.
609 *
610 * The format of one line is below,
611 *
612 * [ce|ue]: count
613 *
614 * Example:
615 *
616 * .. code-block:: bash
617 *
618 * ue: 0
619 * ce: 1
620 *
621 */
amdgpu_ras_sysfs_read(struct device * dev,struct device_attribute * attr,char * buf)622 static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
623 struct device_attribute *attr, char *buf)
624 {
625 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
626 struct ras_query_if info = {
627 .head = obj->head,
628 };
629
630 if (!amdgpu_ras_get_error_query_ready(obj->adev))
631 return sysfs_emit(buf, "Query currently inaccessible\n");
632
633 if (amdgpu_ras_query_error_status(obj->adev, &info))
634 return -EINVAL;
635
636 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
637 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
638 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
639 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
640 }
641
642 if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
643 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
644 "ce", info.ce_count, "de", info.de_count);
645 else
646 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
647 "ce", info.ce_count);
648 }
649
650 /* obj begin */
651
652 #define get_obj(obj) do { (obj)->use++; } while (0)
653 #define alive_obj(obj) ((obj)->use)
654
put_obj(struct ras_manager * obj)655 static inline void put_obj(struct ras_manager *obj)
656 {
657 if (obj && (--obj->use == 0)) {
658 list_del(&obj->node);
659 amdgpu_ras_error_data_fini(&obj->err_data);
660 }
661
662 if (obj && (obj->use < 0))
663 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
664 }
665
666 /* make one obj and return it. */
amdgpu_ras_create_obj(struct amdgpu_device * adev,struct ras_common_if * head)667 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
668 struct ras_common_if *head)
669 {
670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
671 struct ras_manager *obj;
672
673 if (!adev->ras_enabled || !con)
674 return NULL;
675
676 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
677 return NULL;
678
679 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
680 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
681 return NULL;
682
683 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
684 } else
685 obj = &con->objs[head->block];
686
687 /* already exist. return obj? */
688 if (alive_obj(obj))
689 return NULL;
690
691 if (amdgpu_ras_error_data_init(&obj->err_data))
692 return NULL;
693
694 obj->head = *head;
695 obj->adev = adev;
696 list_add(&obj->node, &con->head);
697 get_obj(obj);
698
699 return obj;
700 }
701
702 /* return an obj equal to head, or the first when head is NULL */
amdgpu_ras_find_obj(struct amdgpu_device * adev,struct ras_common_if * head)703 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
704 struct ras_common_if *head)
705 {
706 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
707 struct ras_manager *obj;
708 int i;
709
710 if (!adev->ras_enabled || !con)
711 return NULL;
712
713 if (head) {
714 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
715 return NULL;
716
717 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
718 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
719 return NULL;
720
721 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
722 } else
723 obj = &con->objs[head->block];
724
725 if (alive_obj(obj))
726 return obj;
727 } else {
728 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
729 obj = &con->objs[i];
730 if (alive_obj(obj))
731 return obj;
732 }
733 }
734
735 return NULL;
736 }
737 /* obj end */
738
739 /* feature ctl begin */
amdgpu_ras_is_feature_allowed(struct amdgpu_device * adev,struct ras_common_if * head)740 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
741 struct ras_common_if *head)
742 {
743 return adev->ras_hw_enabled & BIT(head->block);
744 }
745
amdgpu_ras_is_feature_enabled(struct amdgpu_device * adev,struct ras_common_if * head)746 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
747 struct ras_common_if *head)
748 {
749 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
750
751 return con->features & BIT(head->block);
752 }
753
754 /*
755 * if obj is not created, then create one.
756 * set feature enable flag.
757 */
__amdgpu_ras_feature_enable(struct amdgpu_device * adev,struct ras_common_if * head,int enable)758 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
759 struct ras_common_if *head, int enable)
760 {
761 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
762 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
763
764 /* If hardware does not support ras, then do not create obj.
765 * But if hardware support ras, we can create the obj.
766 * Ras framework checks con->hw_supported to see if it need do
767 * corresponding initialization.
768 * IP checks con->support to see if it need disable ras.
769 */
770 if (!amdgpu_ras_is_feature_allowed(adev, head))
771 return 0;
772
773 if (enable) {
774 if (!obj) {
775 obj = amdgpu_ras_create_obj(adev, head);
776 if (!obj)
777 return -EINVAL;
778 } else {
779 /* In case we create obj somewhere else */
780 get_obj(obj);
781 }
782 con->features |= BIT(head->block);
783 } else {
784 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
785 con->features &= ~BIT(head->block);
786 put_obj(obj);
787 }
788 }
789
790 return 0;
791 }
792
793 /* wrapper of psp_ras_enable_features */
amdgpu_ras_feature_enable(struct amdgpu_device * adev,struct ras_common_if * head,bool enable)794 int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
795 struct ras_common_if *head, bool enable)
796 {
797 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
798 union ta_ras_cmd_input *info;
799 int ret;
800
801 if (!con)
802 return -EINVAL;
803
804 /* For non-gfx ip, do not enable ras feature if it is not allowed */
805 /* For gfx ip, regardless of feature support status, */
806 /* Force issue enable or disable ras feature commands */
807 if (head->block != AMDGPU_RAS_BLOCK__GFX &&
808 !amdgpu_ras_is_feature_allowed(adev, head))
809 return 0;
810
811 /* Only enable gfx ras feature from host side */
812 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
813 !amdgpu_sriov_vf(adev) &&
814 !amdgpu_ras_intr_triggered()) {
815 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
816 if (!info)
817 return -ENOMEM;
818
819 if (!enable) {
820 info->disable_features = (struct ta_ras_disable_features_input) {
821 .block_id = amdgpu_ras_block_to_ta(head->block),
822 .error_type = amdgpu_ras_error_to_ta(head->type),
823 };
824 } else {
825 info->enable_features = (struct ta_ras_enable_features_input) {
826 .block_id = amdgpu_ras_block_to_ta(head->block),
827 .error_type = amdgpu_ras_error_to_ta(head->type),
828 };
829 }
830
831 ret = psp_ras_enable_features(&adev->psp, info, enable);
832 if (ret) {
833 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
834 enable ? "enable":"disable",
835 get_ras_block_str(head),
836 amdgpu_ras_is_poison_mode_supported(adev), ret);
837 kfree(info);
838 return ret;
839 }
840
841 kfree(info);
842 }
843
844 /* setup the obj */
845 __amdgpu_ras_feature_enable(adev, head, enable);
846
847 return 0;
848 }
849
850 /* Only used in device probe stage and called only once. */
amdgpu_ras_feature_enable_on_boot(struct amdgpu_device * adev,struct ras_common_if * head,bool enable)851 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
852 struct ras_common_if *head, bool enable)
853 {
854 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
855 int ret;
856
857 if (!con)
858 return -EINVAL;
859
860 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
861 if (enable) {
862 /* There is no harm to issue a ras TA cmd regardless of
863 * the currecnt ras state.
864 * If current state == target state, it will do nothing
865 * But sometimes it requests driver to reset and repost
866 * with error code -EAGAIN.
867 */
868 ret = amdgpu_ras_feature_enable(adev, head, 1);
869 /* With old ras TA, we might fail to enable ras.
870 * Log it and just setup the object.
871 * TODO need remove this WA in the future.
872 */
873 if (ret == -EINVAL) {
874 ret = __amdgpu_ras_feature_enable(adev, head, 1);
875 if (!ret)
876 dev_info(adev->dev,
877 "RAS INFO: %s setup object\n",
878 get_ras_block_str(head));
879 }
880 } else {
881 /* setup the object then issue a ras TA disable cmd.*/
882 ret = __amdgpu_ras_feature_enable(adev, head, 1);
883 if (ret)
884 return ret;
885
886 /* gfx block ras disable cmd must send to ras-ta */
887 if (head->block == AMDGPU_RAS_BLOCK__GFX)
888 con->features |= BIT(head->block);
889
890 ret = amdgpu_ras_feature_enable(adev, head, 0);
891
892 /* clean gfx block ras features flag */
893 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
894 con->features &= ~BIT(head->block);
895 }
896 } else
897 ret = amdgpu_ras_feature_enable(adev, head, enable);
898
899 return ret;
900 }
901
amdgpu_ras_disable_all_features(struct amdgpu_device * adev,bool bypass)902 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
903 bool bypass)
904 {
905 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
906 struct ras_manager *obj, *tmp;
907
908 list_for_each_entry_safe(obj, tmp, &con->head, node) {
909 /* bypass psp.
910 * aka just release the obj and corresponding flags
911 */
912 if (bypass) {
913 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
914 break;
915 } else {
916 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
917 break;
918 }
919 }
920
921 return con->features;
922 }
923
amdgpu_ras_enable_all_features(struct amdgpu_device * adev,bool bypass)924 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
925 bool bypass)
926 {
927 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
928 int i;
929 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
930
931 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
932 struct ras_common_if head = {
933 .block = i,
934 .type = default_ras_type,
935 .sub_block_index = 0,
936 };
937
938 if (i == AMDGPU_RAS_BLOCK__MCA)
939 continue;
940
941 if (bypass) {
942 /*
943 * bypass psp. vbios enable ras for us.
944 * so just create the obj
945 */
946 if (__amdgpu_ras_feature_enable(adev, &head, 1))
947 break;
948 } else {
949 if (amdgpu_ras_feature_enable(adev, &head, 1))
950 break;
951 }
952 }
953
954 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
955 struct ras_common_if head = {
956 .block = AMDGPU_RAS_BLOCK__MCA,
957 .type = default_ras_type,
958 .sub_block_index = i,
959 };
960
961 if (bypass) {
962 /*
963 * bypass psp. vbios enable ras for us.
964 * so just create the obj
965 */
966 if (__amdgpu_ras_feature_enable(adev, &head, 1))
967 break;
968 } else {
969 if (amdgpu_ras_feature_enable(adev, &head, 1))
970 break;
971 }
972 }
973
974 return con->features;
975 }
976 /* feature ctl end */
977
amdgpu_ras_block_match_default(struct amdgpu_ras_block_object * block_obj,enum amdgpu_ras_block block)978 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
979 enum amdgpu_ras_block block)
980 {
981 if (!block_obj)
982 return -EINVAL;
983
984 if (block_obj->ras_comm.block == block)
985 return 0;
986
987 return -EINVAL;
988 }
989
amdgpu_ras_get_ras_block(struct amdgpu_device * adev,enum amdgpu_ras_block block,uint32_t sub_block_index)990 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
991 enum amdgpu_ras_block block, uint32_t sub_block_index)
992 {
993 struct amdgpu_ras_block_list *node, *tmp;
994 struct amdgpu_ras_block_object *obj;
995
996 if (block >= AMDGPU_RAS_BLOCK__LAST)
997 return NULL;
998
999 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1000 if (!node->ras_obj) {
1001 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1002 continue;
1003 }
1004
1005 obj = node->ras_obj;
1006 if (obj->ras_block_match) {
1007 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1008 return obj;
1009 } else {
1010 if (amdgpu_ras_block_match_default(obj, block) == 0)
1011 return obj;
1012 }
1013 }
1014
1015 return NULL;
1016 }
1017
amdgpu_ras_get_ecc_info(struct amdgpu_device * adev,struct ras_err_data * err_data)1018 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1019 {
1020 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1021 int ret = 0;
1022
1023 /*
1024 * choosing right query method according to
1025 * whether smu support query error information
1026 */
1027 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
1028 if (ret == -EOPNOTSUPP) {
1029 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1030 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1032
1033 /* umc query_ras_error_address is also responsible for clearing
1034 * error status
1035 */
1036 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1037 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1039 } else if (!ret) {
1040 if (adev->umc.ras &&
1041 adev->umc.ras->ecc_info_query_ras_error_count)
1042 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1043
1044 if (adev->umc.ras &&
1045 adev->umc.ras->ecc_info_query_ras_error_address)
1046 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1047 }
1048 }
1049
amdgpu_ras_error_print_error_data(struct amdgpu_device * adev,struct ras_manager * ras_mgr,struct ras_err_data * err_data,struct ras_query_context * qctx,const char * blk_name,bool is_ue,bool is_de)1050 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1051 struct ras_manager *ras_mgr,
1052 struct ras_err_data *err_data,
1053 struct ras_query_context *qctx,
1054 const char *blk_name,
1055 bool is_ue,
1056 bool is_de)
1057 {
1058 struct amdgpu_smuio_mcm_config_info *mcm_info;
1059 struct ras_err_node *err_node;
1060 struct ras_err_info *err_info;
1061 u64 event_id = qctx->evid.event_id;
1062
1063 if (is_ue) {
1064 for_each_ras_error(err_node, err_data) {
1065 err_info = &err_node->err_info;
1066 mcm_info = &err_info->mcm_info;
1067 if (err_info->ue_count) {
1068 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1069 "%lld new uncorrectable hardware errors detected in %s block\n",
1070 mcm_info->socket_id,
1071 mcm_info->die_id,
1072 err_info->ue_count,
1073 blk_name);
1074 }
1075 }
1076
1077 for_each_ras_error(err_node, &ras_mgr->err_data) {
1078 err_info = &err_node->err_info;
1079 mcm_info = &err_info->mcm_info;
1080 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1081 "%lld uncorrectable hardware errors detected in total in %s block\n",
1082 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1083 }
1084
1085 } else {
1086 if (is_de) {
1087 for_each_ras_error(err_node, err_data) {
1088 err_info = &err_node->err_info;
1089 mcm_info = &err_info->mcm_info;
1090 if (err_info->de_count) {
1091 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1092 "%lld new deferred hardware errors detected in %s block\n",
1093 mcm_info->socket_id,
1094 mcm_info->die_id,
1095 err_info->de_count,
1096 blk_name);
1097 }
1098 }
1099
1100 for_each_ras_error(err_node, &ras_mgr->err_data) {
1101 err_info = &err_node->err_info;
1102 mcm_info = &err_info->mcm_info;
1103 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1104 "%lld deferred hardware errors detected in total in %s block\n",
1105 mcm_info->socket_id, mcm_info->die_id,
1106 err_info->de_count, blk_name);
1107 }
1108 } else {
1109 for_each_ras_error(err_node, err_data) {
1110 err_info = &err_node->err_info;
1111 mcm_info = &err_info->mcm_info;
1112 if (err_info->ce_count) {
1113 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1114 "%lld new correctable hardware errors detected in %s block\n",
1115 mcm_info->socket_id,
1116 mcm_info->die_id,
1117 err_info->ce_count,
1118 blk_name);
1119 }
1120 }
1121
1122 for_each_ras_error(err_node, &ras_mgr->err_data) {
1123 err_info = &err_node->err_info;
1124 mcm_info = &err_info->mcm_info;
1125 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1126 "%lld correctable hardware errors detected in total in %s block\n",
1127 mcm_info->socket_id, mcm_info->die_id,
1128 err_info->ce_count, blk_name);
1129 }
1130 }
1131 }
1132 }
1133
err_data_has_source_info(struct ras_err_data * data)1134 static inline bool err_data_has_source_info(struct ras_err_data *data)
1135 {
1136 return !list_empty(&data->err_node_list);
1137 }
1138
amdgpu_ras_error_generate_report(struct amdgpu_device * adev,struct ras_query_if * query_if,struct ras_err_data * err_data,struct ras_query_context * qctx)1139 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1140 struct ras_query_if *query_if,
1141 struct ras_err_data *err_data,
1142 struct ras_query_context *qctx)
1143 {
1144 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1145 const char *blk_name = get_ras_block_str(&query_if->head);
1146 u64 event_id = qctx->evid.event_id;
1147
1148 if (err_data->ce_count) {
1149 if (err_data_has_source_info(err_data)) {
1150 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1151 blk_name, false, false);
1152 } else if (!adev->aid_mask &&
1153 adev->smuio.funcs &&
1154 adev->smuio.funcs->get_socket_id &&
1155 adev->smuio.funcs->get_die_id) {
1156 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1157 "%ld correctable hardware errors "
1158 "detected in %s block\n",
1159 adev->smuio.funcs->get_socket_id(adev),
1160 adev->smuio.funcs->get_die_id(adev),
1161 ras_mgr->err_data.ce_count,
1162 blk_name);
1163 } else {
1164 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1165 "detected in %s block\n",
1166 ras_mgr->err_data.ce_count,
1167 blk_name);
1168 }
1169 }
1170
1171 if (err_data->ue_count) {
1172 if (err_data_has_source_info(err_data)) {
1173 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1174 blk_name, true, false);
1175 } else if (!adev->aid_mask &&
1176 adev->smuio.funcs &&
1177 adev->smuio.funcs->get_socket_id &&
1178 adev->smuio.funcs->get_die_id) {
1179 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1180 "%ld uncorrectable hardware errors "
1181 "detected in %s block\n",
1182 adev->smuio.funcs->get_socket_id(adev),
1183 adev->smuio.funcs->get_die_id(adev),
1184 ras_mgr->err_data.ue_count,
1185 blk_name);
1186 } else {
1187 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1188 "detected in %s block\n",
1189 ras_mgr->err_data.ue_count,
1190 blk_name);
1191 }
1192 }
1193
1194 if (err_data->de_count) {
1195 if (err_data_has_source_info(err_data)) {
1196 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1197 blk_name, false, true);
1198 } else if (!adev->aid_mask &&
1199 adev->smuio.funcs &&
1200 adev->smuio.funcs->get_socket_id &&
1201 adev->smuio.funcs->get_die_id) {
1202 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1203 "%ld deferred hardware errors "
1204 "detected in %s block\n",
1205 adev->smuio.funcs->get_socket_id(adev),
1206 adev->smuio.funcs->get_die_id(adev),
1207 ras_mgr->err_data.de_count,
1208 blk_name);
1209 } else {
1210 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1211 "detected in %s block\n",
1212 ras_mgr->err_data.de_count,
1213 blk_name);
1214 }
1215 }
1216 }
1217
amdgpu_ras_virt_error_generate_report(struct amdgpu_device * adev,struct ras_query_if * query_if,struct ras_err_data * err_data,struct ras_query_context * qctx)1218 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1219 struct ras_query_if *query_if,
1220 struct ras_err_data *err_data,
1221 struct ras_query_context *qctx)
1222 {
1223 unsigned long new_ue, new_ce, new_de;
1224 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1225 const char *blk_name = get_ras_block_str(&query_if->head);
1226 u64 event_id = qctx->evid.event_id;
1227
1228 new_ce = err_data->ce_count - obj->err_data.ce_count;
1229 new_ue = err_data->ue_count - obj->err_data.ue_count;
1230 new_de = err_data->de_count - obj->err_data.de_count;
1231
1232 if (new_ce) {
1233 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1234 "detected in %s block\n",
1235 new_ce,
1236 blk_name);
1237 }
1238
1239 if (new_ue) {
1240 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1241 "detected in %s block\n",
1242 new_ue,
1243 blk_name);
1244 }
1245
1246 if (new_de) {
1247 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1248 "detected in %s block\n",
1249 new_de,
1250 blk_name);
1251 }
1252 }
1253
amdgpu_rasmgr_error_data_statistic_update(struct ras_manager * obj,struct ras_err_data * err_data)1254 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1255 {
1256 struct ras_err_node *err_node;
1257 struct ras_err_info *err_info;
1258
1259 if (err_data_has_source_info(err_data)) {
1260 for_each_ras_error(err_node, err_data) {
1261 err_info = &err_node->err_info;
1262 amdgpu_ras_error_statistic_de_count(&obj->err_data,
1263 &err_info->mcm_info, err_info->de_count);
1264 amdgpu_ras_error_statistic_ce_count(&obj->err_data,
1265 &err_info->mcm_info, err_info->ce_count);
1266 amdgpu_ras_error_statistic_ue_count(&obj->err_data,
1267 &err_info->mcm_info, err_info->ue_count);
1268 }
1269 } else {
1270 /* for legacy asic path which doesn't has error source info */
1271 obj->err_data.ue_count += err_data->ue_count;
1272 obj->err_data.ce_count += err_data->ce_count;
1273 obj->err_data.de_count += err_data->de_count;
1274 }
1275 }
1276
amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager * obj,struct ras_err_data * err_data)1277 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1278 struct ras_err_data *err_data)
1279 {
1280 /* Host reports absolute counts */
1281 obj->err_data.ue_count = err_data->ue_count;
1282 obj->err_data.ce_count = err_data->ce_count;
1283 obj->err_data.de_count = err_data->de_count;
1284 }
1285
get_ras_manager(struct amdgpu_device * adev,enum amdgpu_ras_block blk)1286 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1287 {
1288 struct ras_common_if head;
1289
1290 memset(&head, 0, sizeof(head));
1291 head.block = blk;
1292
1293 return amdgpu_ras_find_obj(adev, &head);
1294 }
1295
amdgpu_ras_bind_aca(struct amdgpu_device * adev,enum amdgpu_ras_block blk,const struct aca_info * aca_info,void * data)1296 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1297 const struct aca_info *aca_info, void *data)
1298 {
1299 struct ras_manager *obj;
1300
1301 /* in resume phase, no need to create aca fs node */
1302 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1303 return 0;
1304
1305 obj = get_ras_manager(adev, blk);
1306 if (!obj)
1307 return -EINVAL;
1308
1309 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1310 }
1311
amdgpu_ras_unbind_aca(struct amdgpu_device * adev,enum amdgpu_ras_block blk)1312 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1313 {
1314 struct ras_manager *obj;
1315
1316 obj = get_ras_manager(adev, blk);
1317 if (!obj)
1318 return -EINVAL;
1319
1320 amdgpu_aca_remove_handle(&obj->aca_handle);
1321
1322 return 0;
1323 }
1324
amdgpu_aca_log_ras_error_data(struct amdgpu_device * adev,enum amdgpu_ras_block blk,enum aca_error_type type,struct ras_err_data * err_data,struct ras_query_context * qctx)1325 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1326 enum aca_error_type type, struct ras_err_data *err_data,
1327 struct ras_query_context *qctx)
1328 {
1329 struct ras_manager *obj;
1330
1331 obj = get_ras_manager(adev, blk);
1332 if (!obj)
1333 return -EINVAL;
1334
1335 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
1336 }
1337
amdgpu_ras_aca_sysfs_read(struct device * dev,struct device_attribute * attr,struct aca_handle * handle,char * buf,void * data)1338 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1339 struct aca_handle *handle, char *buf, void *data)
1340 {
1341 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1342 struct ras_query_if info = {
1343 .head = obj->head,
1344 };
1345
1346 if (!amdgpu_ras_get_error_query_ready(obj->adev))
1347 return sysfs_emit(buf, "Query currently inaccessible\n");
1348
1349 if (amdgpu_ras_query_error_status(obj->adev, &info))
1350 return -EINVAL;
1351
1352 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1353 "ce", info.ce_count, "de", info.de_count);
1354 }
1355
amdgpu_ras_query_error_status_helper(struct amdgpu_device * adev,struct ras_query_if * info,struct ras_err_data * err_data,struct ras_query_context * qctx,unsigned int error_query_mode)1356 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1357 struct ras_query_if *info,
1358 struct ras_err_data *err_data,
1359 struct ras_query_context *qctx,
1360 unsigned int error_query_mode)
1361 {
1362 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1363 struct amdgpu_ras_block_object *block_obj = NULL;
1364 int ret;
1365
1366 if (blk == AMDGPU_RAS_BLOCK_COUNT)
1367 return -EINVAL;
1368
1369 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1370 return -EINVAL;
1371
1372 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1373 return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1374 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1375 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1376 amdgpu_ras_get_ecc_info(adev, err_data);
1377 } else {
1378 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1379 if (!block_obj || !block_obj->hw_ops) {
1380 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1381 get_ras_block_str(&info->head));
1382 return -EINVAL;
1383 }
1384
1385 if (block_obj->hw_ops->query_ras_error_count)
1386 block_obj->hw_ops->query_ras_error_count(adev, err_data);
1387
1388 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1389 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1390 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1391 if (block_obj->hw_ops->query_ras_error_status)
1392 block_obj->hw_ops->query_ras_error_status(adev);
1393 }
1394 }
1395 } else {
1396 if (amdgpu_aca_is_enabled(adev)) {
1397 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
1398 if (ret)
1399 return ret;
1400
1401 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
1402 if (ret)
1403 return ret;
1404
1405 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1406 if (ret)
1407 return ret;
1408 } else {
1409 /* FIXME: add code to check return value later */
1410 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1411 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1412 }
1413 }
1414
1415 return 0;
1416 }
1417
1418 /* query/inject/cure begin */
amdgpu_ras_query_error_status_with_event(struct amdgpu_device * adev,struct ras_query_if * info,enum ras_event_type type)1419 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1420 struct ras_query_if *info,
1421 enum ras_event_type type)
1422 {
1423 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1424 struct ras_err_data err_data;
1425 struct ras_query_context qctx;
1426 unsigned int error_query_mode;
1427 int ret;
1428
1429 if (!obj)
1430 return -EINVAL;
1431
1432 ret = amdgpu_ras_error_data_init(&err_data);
1433 if (ret)
1434 return ret;
1435
1436 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1437 return -EINVAL;
1438
1439 memset(&qctx, 0, sizeof(qctx));
1440 qctx.evid.type = type;
1441 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1442
1443 if (!down_read_trylock(&adev->reset_domain->sem)) {
1444 ret = -EIO;
1445 goto out_fini_err_data;
1446 }
1447
1448 ret = amdgpu_ras_query_error_status_helper(adev, info,
1449 &err_data,
1450 &qctx,
1451 error_query_mode);
1452 up_read(&adev->reset_domain->sem);
1453 if (ret)
1454 goto out_fini_err_data;
1455
1456 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1457 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1458 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1459 } else {
1460 /* Host provides absolute error counts. First generate the report
1461 * using the previous VF internal count against new host count.
1462 * Then Update VF internal count.
1463 */
1464 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1465 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1466 }
1467
1468 info->ue_count = obj->err_data.ue_count;
1469 info->ce_count = obj->err_data.ce_count;
1470 info->de_count = obj->err_data.de_count;
1471
1472 out_fini_err_data:
1473 amdgpu_ras_error_data_fini(&err_data);
1474
1475 return ret;
1476 }
1477
amdgpu_ras_query_error_status(struct amdgpu_device * adev,struct ras_query_if * info)1478 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1479 {
1480 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1481 }
1482
amdgpu_ras_reset_error_count(struct amdgpu_device * adev,enum amdgpu_ras_block block)1483 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1484 enum amdgpu_ras_block block)
1485 {
1486 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1487 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1488 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1489
1490 if (!block_obj || !block_obj->hw_ops) {
1491 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1492 ras_block_str(block));
1493 return -EOPNOTSUPP;
1494 }
1495
1496 if (!amdgpu_ras_is_supported(adev, block) ||
1497 !amdgpu_ras_get_aca_debug_mode(adev))
1498 return -EOPNOTSUPP;
1499
1500 /* skip ras error reset in gpu reset */
1501 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1502 ((smu_funcs && smu_funcs->set_debug_mode) ||
1503 (mca_funcs && mca_funcs->mca_set_debug_mode)))
1504 return -EOPNOTSUPP;
1505
1506 if (block_obj->hw_ops->reset_ras_error_count)
1507 block_obj->hw_ops->reset_ras_error_count(adev);
1508
1509 return 0;
1510 }
1511
amdgpu_ras_reset_error_status(struct amdgpu_device * adev,enum amdgpu_ras_block block)1512 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1513 enum amdgpu_ras_block block)
1514 {
1515 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1516
1517 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1518 return 0;
1519
1520 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1521 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1522 if (block_obj->hw_ops->reset_ras_error_status)
1523 block_obj->hw_ops->reset_ras_error_status(adev);
1524 }
1525
1526 return 0;
1527 }
1528
1529 /* wrapper of psp_ras_trigger_error */
amdgpu_ras_error_inject(struct amdgpu_device * adev,struct ras_inject_if * info)1530 int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1531 struct ras_inject_if *info)
1532 {
1533 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1534 struct ta_ras_trigger_error_input block_info = {
1535 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1536 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1537 .sub_block_index = info->head.sub_block_index,
1538 .address = info->address,
1539 .value = info->value,
1540 };
1541 int ret = -EINVAL;
1542 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1543 info->head.block,
1544 info->head.sub_block_index);
1545
1546 /* inject on guest isn't allowed, return success directly */
1547 if (amdgpu_sriov_vf(adev))
1548 return 0;
1549
1550 if (!obj)
1551 return -EINVAL;
1552
1553 if (!block_obj || !block_obj->hw_ops) {
1554 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1555 get_ras_block_str(&info->head));
1556 return -EINVAL;
1557 }
1558
1559 /* Calculate XGMI relative offset */
1560 if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1561 info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1562 block_info.address =
1563 amdgpu_xgmi_get_relative_phy_addr(adev,
1564 block_info.address);
1565 }
1566
1567 if (block_obj->hw_ops->ras_error_inject) {
1568 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1569 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1570 else /* Special ras_error_inject is defined (e.g: xgmi) */
1571 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1572 info->instance_mask);
1573 } else {
1574 /* default path */
1575 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1576 }
1577
1578 if (ret)
1579 dev_err(adev->dev, "ras inject %s failed %d\n",
1580 get_ras_block_str(&info->head), ret);
1581
1582 return ret;
1583 }
1584
1585 /**
1586 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1587 * @adev: pointer to AMD GPU device
1588 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1589 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1590 * @query_info: pointer to ras_query_if
1591 *
1592 * Return 0 for query success or do nothing, otherwise return an error
1593 * on failures
1594 */
amdgpu_ras_query_error_count_helper(struct amdgpu_device * adev,unsigned long * ce_count,unsigned long * ue_count,struct ras_query_if * query_info)1595 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1596 unsigned long *ce_count,
1597 unsigned long *ue_count,
1598 struct ras_query_if *query_info)
1599 {
1600 int ret;
1601
1602 if (!query_info)
1603 /* do nothing if query_info is not specified */
1604 return 0;
1605
1606 ret = amdgpu_ras_query_error_status(adev, query_info);
1607 if (ret)
1608 return ret;
1609
1610 *ce_count += query_info->ce_count;
1611 *ue_count += query_info->ue_count;
1612
1613 /* some hardware/IP supports read to clear
1614 * no need to explictly reset the err status after the query call */
1615 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1616 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1617 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1618 dev_warn(adev->dev,
1619 "Failed to reset error counter and error status\n");
1620 }
1621
1622 return 0;
1623 }
1624
1625 /**
1626 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1627 * @adev: pointer to AMD GPU device
1628 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1629 * @ue_count: pointer to an integer to be set to the count of uncorrectible
1630 * errors.
1631 * @query_info: pointer to ras_query_if if the query request is only for
1632 * specific ip block; if info is NULL, then the qurey request is for
1633 * all the ip blocks that support query ras error counters/status
1634 *
1635 * If set, @ce_count or @ue_count, count and return the corresponding
1636 * error counts in those integer pointers. Return 0 if the device
1637 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1638 */
amdgpu_ras_query_error_count(struct amdgpu_device * adev,unsigned long * ce_count,unsigned long * ue_count,struct ras_query_if * query_info)1639 int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1640 unsigned long *ce_count,
1641 unsigned long *ue_count,
1642 struct ras_query_if *query_info)
1643 {
1644 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1645 struct ras_manager *obj;
1646 unsigned long ce, ue;
1647 int ret;
1648
1649 if (!adev->ras_enabled || !con)
1650 return -EOPNOTSUPP;
1651
1652 /* Don't count since no reporting.
1653 */
1654 if (!ce_count && !ue_count)
1655 return 0;
1656
1657 ce = 0;
1658 ue = 0;
1659 if (!query_info) {
1660 /* query all the ip blocks that support ras query interface */
1661 list_for_each_entry(obj, &con->head, node) {
1662 struct ras_query_if info = {
1663 .head = obj->head,
1664 };
1665
1666 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1667 }
1668 } else {
1669 /* query specific ip block */
1670 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1671 }
1672
1673 if (ret)
1674 return ret;
1675
1676 if (ce_count)
1677 *ce_count = ce;
1678
1679 if (ue_count)
1680 *ue_count = ue;
1681
1682 return 0;
1683 }
1684 /* query/inject/cure end */
1685
1686
1687 /* sysfs begin */
1688
1689 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1690 struct ras_badpage **bps, unsigned int *count);
1691
amdgpu_ras_badpage_flags_str(unsigned int flags)1692 static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1693 {
1694 switch (flags) {
1695 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1696 return "R";
1697 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1698 return "P";
1699 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1700 default:
1701 return "F";
1702 }
1703 }
1704
1705 /**
1706 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1707 *
1708 * It allows user to read the bad pages of vram on the gpu through
1709 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1710 *
1711 * It outputs multiple lines, and each line stands for one gpu page.
1712 *
1713 * The format of one line is below,
1714 * gpu pfn : gpu page size : flags
1715 *
1716 * gpu pfn and gpu page size are printed in hex format.
1717 * flags can be one of below character,
1718 *
1719 * R: reserved, this gpu page is reserved and not able to use.
1720 *
1721 * P: pending for reserve, this gpu page is marked as bad, will be reserved
1722 * in next window of page_reserve.
1723 *
1724 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1725 *
1726 * Examples:
1727 *
1728 * .. code-block:: bash
1729 *
1730 * 0x00000001 : 0x00001000 : R
1731 * 0x00000002 : 0x00001000 : P
1732 *
1733 */
1734
amdgpu_ras_sysfs_badpages_read(struct file * f,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t ppos,size_t count)1735 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1736 struct kobject *kobj, struct bin_attribute *attr,
1737 char *buf, loff_t ppos, size_t count)
1738 {
1739 struct amdgpu_ras *con =
1740 container_of(attr, struct amdgpu_ras, badpages_attr);
1741 struct amdgpu_device *adev = con->adev;
1742 const unsigned int element_size =
1743 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1744 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1745 unsigned int end = div64_ul(ppos + count - 1, element_size);
1746 ssize_t s = 0;
1747 struct ras_badpage *bps = NULL;
1748 unsigned int bps_count = 0;
1749
1750 memset(buf, 0, count);
1751
1752 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1753 return 0;
1754
1755 for (; start < end && start < bps_count; start++)
1756 s += scnprintf(&buf[s], element_size + 1,
1757 "0x%08x : 0x%08x : %1s\n",
1758 bps[start].bp,
1759 bps[start].size,
1760 amdgpu_ras_badpage_flags_str(bps[start].flags));
1761
1762 kfree(bps);
1763
1764 return s;
1765 }
1766
amdgpu_ras_sysfs_features_read(struct device * dev,struct device_attribute * attr,char * buf)1767 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1768 struct device_attribute *attr, char *buf)
1769 {
1770 struct amdgpu_ras *con =
1771 container_of(attr, struct amdgpu_ras, features_attr);
1772
1773 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1774 }
1775
amdgpu_ras_sysfs_version_show(struct device * dev,struct device_attribute * attr,char * buf)1776 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1777 struct device_attribute *attr, char *buf)
1778 {
1779 struct amdgpu_ras *con =
1780 container_of(attr, struct amdgpu_ras, version_attr);
1781 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1782 }
1783
amdgpu_ras_sysfs_schema_show(struct device * dev,struct device_attribute * attr,char * buf)1784 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1785 struct device_attribute *attr, char *buf)
1786 {
1787 struct amdgpu_ras *con =
1788 container_of(attr, struct amdgpu_ras, schema_attr);
1789 return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1790 }
1791
1792 static struct {
1793 enum ras_event_type type;
1794 const char *name;
1795 } dump_event[] = {
1796 {RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1797 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1798 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1799 };
1800
amdgpu_ras_sysfs_event_state_show(struct device * dev,struct device_attribute * attr,char * buf)1801 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1802 struct device_attribute *attr, char *buf)
1803 {
1804 struct amdgpu_ras *con =
1805 container_of(attr, struct amdgpu_ras, event_state_attr);
1806 struct ras_event_manager *event_mgr = con->event_mgr;
1807 struct ras_event_state *event_state;
1808 int i, size = 0;
1809
1810 if (!event_mgr)
1811 return -EINVAL;
1812
1813 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1814 for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1815 event_state = &event_mgr->event_state[dump_event[i].type];
1816 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1817 dump_event[i].name,
1818 atomic64_read(&event_state->count),
1819 event_state->last_seqno);
1820 }
1821
1822 return (ssize_t)size;
1823 }
1824
amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device * adev)1825 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1826 {
1827 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1828
1829 if (adev->dev->kobj.sd)
1830 sysfs_remove_file_from_group(&adev->dev->kobj,
1831 &con->badpages_attr.attr,
1832 RAS_FS_NAME);
1833 }
1834
amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device * adev)1835 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
1836 {
1837 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1838 struct attribute *attrs[] = {
1839 &con->features_attr.attr,
1840 &con->version_attr.attr,
1841 &con->schema_attr.attr,
1842 &con->event_state_attr.attr,
1843 NULL
1844 };
1845 struct attribute_group group = {
1846 .name = RAS_FS_NAME,
1847 .attrs = attrs,
1848 };
1849
1850 if (adev->dev->kobj.sd)
1851 sysfs_remove_group(&adev->dev->kobj, &group);
1852
1853 return 0;
1854 }
1855
amdgpu_ras_sysfs_create(struct amdgpu_device * adev,struct ras_common_if * head)1856 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1857 struct ras_common_if *head)
1858 {
1859 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1860
1861 if (amdgpu_aca_is_enabled(adev))
1862 return 0;
1863
1864 if (!obj || obj->attr_inuse)
1865 return -EINVAL;
1866
1867 get_obj(obj);
1868
1869 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1870 "%s_err_count", head->name);
1871
1872 obj->sysfs_attr = (struct device_attribute){
1873 .attr = {
1874 .name = obj->fs_data.sysfs_name,
1875 .mode = S_IRUGO,
1876 },
1877 .show = amdgpu_ras_sysfs_read,
1878 };
1879 sysfs_attr_init(&obj->sysfs_attr.attr);
1880
1881 if (sysfs_add_file_to_group(&adev->dev->kobj,
1882 &obj->sysfs_attr.attr,
1883 RAS_FS_NAME)) {
1884 put_obj(obj);
1885 return -EINVAL;
1886 }
1887
1888 obj->attr_inuse = 1;
1889
1890 return 0;
1891 }
1892
amdgpu_ras_sysfs_remove(struct amdgpu_device * adev,struct ras_common_if * head)1893 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1894 struct ras_common_if *head)
1895 {
1896 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1897
1898 if (amdgpu_aca_is_enabled(adev))
1899 return 0;
1900
1901 if (!obj || !obj->attr_inuse)
1902 return -EINVAL;
1903
1904 if (adev->dev->kobj.sd)
1905 sysfs_remove_file_from_group(&adev->dev->kobj,
1906 &obj->sysfs_attr.attr,
1907 RAS_FS_NAME);
1908 obj->attr_inuse = 0;
1909 put_obj(obj);
1910
1911 return 0;
1912 }
1913
amdgpu_ras_sysfs_remove_all(struct amdgpu_device * adev)1914 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1915 {
1916 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1917 struct ras_manager *obj, *tmp;
1918
1919 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1920 amdgpu_ras_sysfs_remove(adev, &obj->head);
1921 }
1922
1923 if (amdgpu_bad_page_threshold != 0)
1924 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1925
1926 amdgpu_ras_sysfs_remove_dev_attr_node(adev);
1927
1928 return 0;
1929 }
1930 /* sysfs end */
1931
1932 /**
1933 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1934 *
1935 * Normally when there is an uncorrectable error, the driver will reset
1936 * the GPU to recover. However, in the event of an unrecoverable error,
1937 * the driver provides an interface to reboot the system automatically
1938 * in that event.
1939 *
1940 * The following file in debugfs provides that interface:
1941 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1942 *
1943 * Usage:
1944 *
1945 * .. code-block:: bash
1946 *
1947 * echo true > .../ras/auto_reboot
1948 *
1949 */
1950 /* debugfs begin */
amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device * adev)1951 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1952 {
1953 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1954 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1955 struct drm_minor *minor = adev_to_drm(adev)->primary;
1956 struct dentry *dir;
1957
1958 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1959 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1960 &amdgpu_ras_debugfs_ctrl_ops);
1961 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1962 &amdgpu_ras_debugfs_eeprom_ops);
1963 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1964 &con->bad_page_cnt_threshold);
1965 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1966 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1967 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1968 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1969 &amdgpu_ras_debugfs_eeprom_size_ops);
1970 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1971 S_IRUGO, dir, adev,
1972 &amdgpu_ras_debugfs_eeprom_table_ops);
1973 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1974
1975 /*
1976 * After one uncorrectable error happens, usually GPU recovery will
1977 * be scheduled. But due to the known problem in GPU recovery failing
1978 * to bring GPU back, below interface provides one direct way to
1979 * user to reboot system automatically in such case within
1980 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1981 * will never be called.
1982 */
1983 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1984
1985 /*
1986 * User could set this not to clean up hardware's error count register
1987 * of RAS IPs during ras recovery.
1988 */
1989 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1990 &con->disable_ras_err_cnt_harvest);
1991 return dir;
1992 }
1993
amdgpu_ras_debugfs_create(struct amdgpu_device * adev,struct ras_fs_if * head,struct dentry * dir)1994 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
1995 struct ras_fs_if *head,
1996 struct dentry *dir)
1997 {
1998 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
1999
2000 if (!obj || !dir)
2001 return;
2002
2003 get_obj(obj);
2004
2005 memcpy(obj->fs_data.debugfs_name,
2006 head->debugfs_name,
2007 sizeof(obj->fs_data.debugfs_name));
2008
2009 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2010 obj, &amdgpu_ras_debugfs_ops);
2011 }
2012
amdgpu_ras_aca_is_supported(struct amdgpu_device * adev)2013 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2014 {
2015 bool ret;
2016
2017 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2018 case IP_VERSION(13, 0, 6):
2019 case IP_VERSION(13, 0, 12):
2020 case IP_VERSION(13, 0, 14):
2021 ret = true;
2022 break;
2023 default:
2024 ret = false;
2025 break;
2026 }
2027
2028 return ret;
2029 }
2030
amdgpu_ras_debugfs_create_all(struct amdgpu_device * adev)2031 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2032 {
2033 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2034 struct dentry *dir;
2035 struct ras_manager *obj;
2036 struct ras_fs_if fs_info;
2037
2038 /*
2039 * it won't be called in resume path, no need to check
2040 * suspend and gpu reset status
2041 */
2042 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2043 return;
2044
2045 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2046
2047 list_for_each_entry(obj, &con->head, node) {
2048 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2049 (obj->attr_inuse == 1)) {
2050 sprintf(fs_info.debugfs_name, "%s_err_inject",
2051 get_ras_block_str(&obj->head));
2052 fs_info.head = obj->head;
2053 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
2054 }
2055 }
2056
2057 if (amdgpu_ras_aca_is_supported(adev)) {
2058 if (amdgpu_aca_is_enabled(adev))
2059 amdgpu_aca_smu_debugfs_init(adev, dir);
2060 else
2061 amdgpu_mca_smu_debugfs_init(adev, dir);
2062 }
2063 }
2064
2065 /* debugfs end */
2066
2067 /* ras fs */
2068 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2069 amdgpu_ras_sysfs_badpages_read, NULL, 0);
2070 static DEVICE_ATTR(features, S_IRUGO,
2071 amdgpu_ras_sysfs_features_read, NULL);
2072 static DEVICE_ATTR(version, 0444,
2073 amdgpu_ras_sysfs_version_show, NULL);
2074 static DEVICE_ATTR(schema, 0444,
2075 amdgpu_ras_sysfs_schema_show, NULL);
2076 static DEVICE_ATTR(event_state, 0444,
2077 amdgpu_ras_sysfs_event_state_show, NULL);
amdgpu_ras_fs_init(struct amdgpu_device * adev)2078 static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2079 {
2080 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2081 struct attribute_group group = {
2082 .name = RAS_FS_NAME,
2083 };
2084 struct attribute *attrs[] = {
2085 &con->features_attr.attr,
2086 &con->version_attr.attr,
2087 &con->schema_attr.attr,
2088 &con->event_state_attr.attr,
2089 NULL
2090 };
2091 struct bin_attribute *bin_attrs[] = {
2092 NULL,
2093 NULL,
2094 };
2095 int r;
2096
2097 group.attrs = attrs;
2098
2099 /* add features entry */
2100 con->features_attr = dev_attr_features;
2101 sysfs_attr_init(attrs[0]);
2102
2103 /* add version entry */
2104 con->version_attr = dev_attr_version;
2105 sysfs_attr_init(attrs[1]);
2106
2107 /* add schema entry */
2108 con->schema_attr = dev_attr_schema;
2109 sysfs_attr_init(attrs[2]);
2110
2111 /* add event_state entry */
2112 con->event_state_attr = dev_attr_event_state;
2113 sysfs_attr_init(attrs[3]);
2114
2115 if (amdgpu_bad_page_threshold != 0) {
2116 /* add bad_page_features entry */
2117 bin_attr_gpu_vram_bad_pages.private = NULL;
2118 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2119 bin_attrs[0] = &con->badpages_attr;
2120 group.bin_attrs = bin_attrs;
2121 sysfs_bin_attr_init(bin_attrs[0]);
2122 }
2123
2124 r = sysfs_create_group(&adev->dev->kobj, &group);
2125 if (r)
2126 dev_err(adev->dev, "Failed to create RAS sysfs group!");
2127
2128 return 0;
2129 }
2130
amdgpu_ras_fs_fini(struct amdgpu_device * adev)2131 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2132 {
2133 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2134 struct ras_manager *con_obj, *ip_obj, *tmp;
2135
2136 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2137 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2138 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2139 if (ip_obj)
2140 put_obj(ip_obj);
2141 }
2142 }
2143
2144 amdgpu_ras_sysfs_remove_all(adev);
2145 return 0;
2146 }
2147 /* ras fs end */
2148
2149 /* ih begin */
2150
2151 /* For the hardware that cannot enable bif ring for both ras_controller_irq
2152 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2153 * register to check whether the interrupt is triggered or not, and properly
2154 * ack the interrupt if it is there
2155 */
amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device * adev)2156 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2157 {
2158 /* Fatal error events are handled on host side */
2159 if (amdgpu_sriov_vf(adev))
2160 return;
2161 /**
2162 * If the current interrupt is caused by a non-fatal RAS error, skip
2163 * check for fatal error. For fatal errors, FED status of all devices
2164 * in XGMI hive gets set when the first device gets fatal error
2165 * interrupt. The error gets propagated to other devices as well, so
2166 * make sure to ack the interrupt regardless of FED status.
2167 */
2168 if (!amdgpu_ras_get_fed_status(adev) &&
2169 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2170 return;
2171
2172 if (adev->nbio.ras &&
2173 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2174 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2175
2176 if (adev->nbio.ras &&
2177 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2178 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2179 }
2180
amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2181 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2182 struct amdgpu_iv_entry *entry)
2183 {
2184 bool poison_stat = false;
2185 struct amdgpu_device *adev = obj->adev;
2186 struct amdgpu_ras_block_object *block_obj =
2187 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
2188 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2189 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2190 u64 event_id;
2191 int ret;
2192
2193 if (!block_obj || !con)
2194 return;
2195
2196 ret = amdgpu_ras_mark_ras_event(adev, type);
2197 if (ret)
2198 return;
2199
2200 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
2201 /* both query_poison_status and handle_poison_consumption are optional,
2202 * but at least one of them should be implemented if we need poison
2203 * consumption handler
2204 */
2205 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2206 poison_stat = block_obj->hw_ops->query_poison_status(adev);
2207 if (!poison_stat) {
2208 /* Not poison consumption interrupt, no need to handle it */
2209 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2210 block_obj->ras_comm.name);
2211
2212 return;
2213 }
2214 }
2215
2216 amdgpu_umc_poison_handler(adev, obj->head.block, 0);
2217
2218 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2219 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2220
2221 /* gpu reset is fallback for failed and default cases.
2222 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2223 */
2224 if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2225 event_id = amdgpu_ras_acquire_event_id(adev, type);
2226 RAS_EVENT_LOG(adev, event_id,
2227 "GPU reset for %s RAS poison consumption is issued!\n",
2228 block_obj->ras_comm.name);
2229 amdgpu_ras_reset_gpu(adev);
2230 }
2231
2232 if (!poison_stat)
2233 amdgpu_gfx_poison_consumption_handler(adev, entry);
2234 }
2235
amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2236 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2237 struct amdgpu_iv_entry *entry)
2238 {
2239 struct amdgpu_device *adev = obj->adev;
2240 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2241 u64 event_id;
2242 int ret;
2243
2244 ret = amdgpu_ras_mark_ras_event(adev, type);
2245 if (ret)
2246 return;
2247
2248 event_id = amdgpu_ras_acquire_event_id(adev, type);
2249 RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2250
2251 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2252 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2253
2254 atomic_inc(&con->page_retirement_req_cnt);
2255 atomic_inc(&con->poison_creation_count);
2256
2257 wake_up(&con->page_retirement_wq);
2258 }
2259 }
2260
amdgpu_ras_interrupt_umc_handler(struct ras_manager * obj,struct amdgpu_iv_entry * entry)2261 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2262 struct amdgpu_iv_entry *entry)
2263 {
2264 struct ras_ih_data *data = &obj->ih_data;
2265 struct ras_err_data err_data;
2266 int ret;
2267
2268 if (!data->cb)
2269 return;
2270
2271 ret = amdgpu_ras_error_data_init(&err_data);
2272 if (ret)
2273 return;
2274
2275 /* Let IP handle its data, maybe we need get the output
2276 * from the callback to update the error type/count, etc
2277 */
2278 amdgpu_ras_set_fed(obj->adev, true);
2279 ret = data->cb(obj->adev, &err_data, entry);
2280 /* ue will trigger an interrupt, and in that case
2281 * we need do a reset to recovery the whole system.
2282 * But leave IP do that recovery, here we just dispatch
2283 * the error.
2284 */
2285 if (ret == AMDGPU_RAS_SUCCESS) {
2286 /* these counts could be left as 0 if
2287 * some blocks do not count error number
2288 */
2289 obj->err_data.ue_count += err_data.ue_count;
2290 obj->err_data.ce_count += err_data.ce_count;
2291 obj->err_data.de_count += err_data.de_count;
2292 }
2293
2294 amdgpu_ras_error_data_fini(&err_data);
2295 }
2296
amdgpu_ras_interrupt_handler(struct ras_manager * obj)2297 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2298 {
2299 struct ras_ih_data *data = &obj->ih_data;
2300 struct amdgpu_iv_entry entry;
2301
2302 while (data->rptr != data->wptr) {
2303 rmb();
2304 memcpy(&entry, &data->ring[data->rptr],
2305 data->element_size);
2306
2307 wmb();
2308 data->rptr = (data->aligned_element_size +
2309 data->rptr) % data->ring_size;
2310
2311 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2312 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2313 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
2314 else
2315 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
2316 } else {
2317 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2318 amdgpu_ras_interrupt_umc_handler(obj, &entry);
2319 else
2320 dev_warn(obj->adev->dev,
2321 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
2322 }
2323 }
2324 }
2325
amdgpu_ras_interrupt_process_handler(struct work_struct * work)2326 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2327 {
2328 struct ras_ih_data *data =
2329 container_of(work, struct ras_ih_data, ih_work);
2330 struct ras_manager *obj =
2331 container_of(data, struct ras_manager, ih_data);
2332
2333 amdgpu_ras_interrupt_handler(obj);
2334 }
2335
amdgpu_ras_interrupt_dispatch(struct amdgpu_device * adev,struct ras_dispatch_if * info)2336 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2337 struct ras_dispatch_if *info)
2338 {
2339 struct ras_manager *obj;
2340 struct ras_ih_data *data;
2341
2342 obj = amdgpu_ras_find_obj(adev, &info->head);
2343 if (!obj)
2344 return -EINVAL;
2345
2346 data = &obj->ih_data;
2347
2348 if (data->inuse == 0)
2349 return 0;
2350
2351 /* Might be overflow... */
2352 memcpy(&data->ring[data->wptr], info->entry,
2353 data->element_size);
2354
2355 wmb();
2356 data->wptr = (data->aligned_element_size +
2357 data->wptr) % data->ring_size;
2358
2359 schedule_work(&data->ih_work);
2360
2361 return 0;
2362 }
2363
amdgpu_ras_interrupt_remove_handler(struct amdgpu_device * adev,struct ras_common_if * head)2364 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2365 struct ras_common_if *head)
2366 {
2367 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2368 struct ras_ih_data *data;
2369
2370 if (!obj)
2371 return -EINVAL;
2372
2373 data = &obj->ih_data;
2374 if (data->inuse == 0)
2375 return 0;
2376
2377 cancel_work_sync(&data->ih_work);
2378
2379 kfree(data->ring);
2380 memset(data, 0, sizeof(*data));
2381 put_obj(obj);
2382
2383 return 0;
2384 }
2385
amdgpu_ras_interrupt_add_handler(struct amdgpu_device * adev,struct ras_common_if * head)2386 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2387 struct ras_common_if *head)
2388 {
2389 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2390 struct ras_ih_data *data;
2391 struct amdgpu_ras_block_object *ras_obj;
2392
2393 if (!obj) {
2394 /* in case we registe the IH before enable ras feature */
2395 obj = amdgpu_ras_create_obj(adev, head);
2396 if (!obj)
2397 return -EINVAL;
2398 } else
2399 get_obj(obj);
2400
2401 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2402
2403 data = &obj->ih_data;
2404 /* add the callback.etc */
2405 *data = (struct ras_ih_data) {
2406 .inuse = 0,
2407 .cb = ras_obj->ras_cb,
2408 .element_size = sizeof(struct amdgpu_iv_entry),
2409 .rptr = 0,
2410 .wptr = 0,
2411 };
2412
2413 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2414
2415 data->aligned_element_size = ALIGN(data->element_size, 8);
2416 /* the ring can store 64 iv entries. */
2417 data->ring_size = 64 * data->aligned_element_size;
2418 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2419 if (!data->ring) {
2420 put_obj(obj);
2421 return -ENOMEM;
2422 }
2423
2424 /* IH is ready */
2425 data->inuse = 1;
2426
2427 return 0;
2428 }
2429
amdgpu_ras_interrupt_remove_all(struct amdgpu_device * adev)2430 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2431 {
2432 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2433 struct ras_manager *obj, *tmp;
2434
2435 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2436 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
2437 }
2438
2439 return 0;
2440 }
2441 /* ih end */
2442
2443 /* traversal all IPs except NBIO to query error counter */
amdgpu_ras_log_on_err_counter(struct amdgpu_device * adev,enum ras_event_type type)2444 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2445 {
2446 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2447 struct ras_manager *obj;
2448
2449 if (!adev->ras_enabled || !con)
2450 return;
2451
2452 list_for_each_entry(obj, &con->head, node) {
2453 struct ras_query_if info = {
2454 .head = obj->head,
2455 };
2456
2457 /*
2458 * PCIE_BIF IP has one different isr by ras controller
2459 * interrupt, the specific ras counter query will be
2460 * done in that isr. So skip such block from common
2461 * sync flood interrupt isr calling.
2462 */
2463 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2464 continue;
2465
2466 /*
2467 * this is a workaround for aldebaran, skip send msg to
2468 * smu to get ecc_info table due to smu handle get ecc
2469 * info table failed temporarily.
2470 * should be removed until smu fix handle ecc_info table.
2471 */
2472 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2473 (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2474 IP_VERSION(13, 0, 2)))
2475 continue;
2476
2477 amdgpu_ras_query_error_status_with_event(adev, &info, type);
2478
2479 if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2480 IP_VERSION(11, 0, 2) &&
2481 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2482 IP_VERSION(11, 0, 4) &&
2483 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2484 IP_VERSION(13, 0, 0)) {
2485 if (amdgpu_ras_reset_error_status(adev, info.head.block))
2486 dev_warn(adev->dev, "Failed to reset error counter and error status");
2487 }
2488 }
2489 }
2490
2491 /* Parse RdRspStatus and WrRspStatus */
amdgpu_ras_error_status_query(struct amdgpu_device * adev,struct ras_query_if * info)2492 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2493 struct ras_query_if *info)
2494 {
2495 struct amdgpu_ras_block_object *block_obj;
2496 /*
2497 * Only two block need to query read/write
2498 * RspStatus at current state
2499 */
2500 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2501 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2502 return;
2503
2504 block_obj = amdgpu_ras_get_ras_block(adev,
2505 info->head.block,
2506 info->head.sub_block_index);
2507
2508 if (!block_obj || !block_obj->hw_ops) {
2509 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2510 get_ras_block_str(&info->head));
2511 return;
2512 }
2513
2514 if (block_obj->hw_ops->query_ras_error_status)
2515 block_obj->hw_ops->query_ras_error_status(adev);
2516
2517 }
2518
amdgpu_ras_query_err_status(struct amdgpu_device * adev)2519 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2520 {
2521 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2522 struct ras_manager *obj;
2523
2524 if (!adev->ras_enabled || !con)
2525 return;
2526
2527 list_for_each_entry(obj, &con->head, node) {
2528 struct ras_query_if info = {
2529 .head = obj->head,
2530 };
2531
2532 amdgpu_ras_error_status_query(adev, &info);
2533 }
2534 }
2535
2536 /* recovery begin */
2537
2538 /* return 0 on success.
2539 * caller need free bps.
2540 */
amdgpu_ras_badpages_read(struct amdgpu_device * adev,struct ras_badpage ** bps,unsigned int * count)2541 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2542 struct ras_badpage **bps, unsigned int *count)
2543 {
2544 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2545 struct ras_err_handler_data *data;
2546 int i = 0;
2547 int ret = 0, status;
2548
2549 if (!con || !con->eh_data || !bps || !count)
2550 return -EINVAL;
2551
2552 mutex_lock(&con->recovery_lock);
2553 data = con->eh_data;
2554 if (!data || data->count == 0) {
2555 *bps = NULL;
2556 ret = -EINVAL;
2557 goto out;
2558 }
2559
2560 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2561 if (!*bps) {
2562 ret = -ENOMEM;
2563 goto out;
2564 }
2565
2566 for (; i < data->count; i++) {
2567 (*bps)[i] = (struct ras_badpage){
2568 .bp = data->bps[i].retired_page,
2569 .size = AMDGPU_GPU_PAGE_SIZE,
2570 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2571 };
2572 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2573 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
2574 if (status == -EBUSY)
2575 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2576 else if (status == -ENOENT)
2577 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2578 }
2579
2580 *count = data->count;
2581 out:
2582 mutex_unlock(&con->recovery_lock);
2583 return ret;
2584 }
2585
amdgpu_ras_set_fed_all(struct amdgpu_device * adev,struct amdgpu_hive_info * hive,bool status)2586 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2587 struct amdgpu_hive_info *hive, bool status)
2588 {
2589 struct amdgpu_device *tmp_adev;
2590
2591 if (hive) {
2592 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2593 amdgpu_ras_set_fed(tmp_adev, status);
2594 } else {
2595 amdgpu_ras_set_fed(adev, status);
2596 }
2597 }
2598
amdgpu_ras_in_recovery(struct amdgpu_device * adev)2599 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2600 {
2601 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2602 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2603 int hive_ras_recovery = 0;
2604
2605 if (hive) {
2606 hive_ras_recovery = atomic_read(&hive->ras_recovery);
2607 amdgpu_put_xgmi_hive(hive);
2608 }
2609
2610 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2611 return true;
2612
2613 return false;
2614 }
2615
amdgpu_ras_get_fatal_error_event(struct amdgpu_device * adev)2616 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2617 {
2618 if (amdgpu_ras_intr_triggered())
2619 return RAS_EVENT_TYPE_FATAL;
2620 else
2621 return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2622 }
2623
amdgpu_ras_do_recovery(struct work_struct * work)2624 static void amdgpu_ras_do_recovery(struct work_struct *work)
2625 {
2626 struct amdgpu_ras *ras =
2627 container_of(work, struct amdgpu_ras, recovery_work);
2628 struct amdgpu_device *remote_adev = NULL;
2629 struct amdgpu_device *adev = ras->adev;
2630 struct list_head device_list, *device_list_handle = NULL;
2631 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2632 enum ras_event_type type;
2633
2634 if (hive) {
2635 atomic_set(&hive->ras_recovery, 1);
2636
2637 /* If any device which is part of the hive received RAS fatal
2638 * error interrupt, set fatal error status on all. This
2639 * condition will need a recovery, and flag will be cleared
2640 * as part of recovery.
2641 */
2642 list_for_each_entry(remote_adev, &hive->device_list,
2643 gmc.xgmi.head)
2644 if (amdgpu_ras_get_fed_status(remote_adev)) {
2645 amdgpu_ras_set_fed_all(adev, hive, true);
2646 break;
2647 }
2648 }
2649 if (!ras->disable_ras_err_cnt_harvest) {
2650
2651 /* Build list of devices to query RAS related errors */
2652 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2653 device_list_handle = &hive->device_list;
2654 } else {
2655 INIT_LIST_HEAD(&device_list);
2656 list_add_tail(&adev->gmc.xgmi.head, &device_list);
2657 device_list_handle = &device_list;
2658 }
2659
2660 type = amdgpu_ras_get_fatal_error_event(adev);
2661 list_for_each_entry(remote_adev,
2662 device_list_handle, gmc.xgmi.head) {
2663 amdgpu_ras_query_err_status(remote_adev);
2664 amdgpu_ras_log_on_err_counter(remote_adev, type);
2665 }
2666
2667 }
2668
2669 if (amdgpu_device_should_recover_gpu(ras->adev)) {
2670 struct amdgpu_reset_context reset_context;
2671 memset(&reset_context, 0, sizeof(reset_context));
2672
2673 reset_context.method = AMD_RESET_METHOD_NONE;
2674 reset_context.reset_req_dev = adev;
2675 reset_context.src = AMDGPU_RESET_SRC_RAS;
2676 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
2677
2678 /* Perform full reset in fatal error mode */
2679 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2680 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2681 else {
2682 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2683
2684 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2685 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2686 reset_context.method = AMD_RESET_METHOD_MODE2;
2687 }
2688
2689 /* Fatal error occurs in poison mode, mode1 reset is used to
2690 * recover gpu.
2691 */
2692 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2693 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2694 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2695
2696 psp_fatal_error_recovery_quirk(&adev->psp);
2697 }
2698 }
2699
2700 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2701 }
2702 atomic_set(&ras->in_recovery, 0);
2703 if (hive) {
2704 atomic_set(&hive->ras_recovery, 0);
2705 amdgpu_put_xgmi_hive(hive);
2706 }
2707 }
2708
2709 /* alloc/realloc bps array */
amdgpu_ras_realloc_eh_data_space(struct amdgpu_device * adev,struct ras_err_handler_data * data,int pages)2710 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2711 struct ras_err_handler_data *data, int pages)
2712 {
2713 unsigned int old_space = data->count + data->space_left;
2714 unsigned int new_space = old_space + pages;
2715 unsigned int align_space = ALIGN(new_space, 512);
2716 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2717
2718 if (!bps) {
2719 return -ENOMEM;
2720 }
2721
2722 if (data->bps) {
2723 memcpy(bps, data->bps,
2724 data->count * sizeof(*data->bps));
2725 kfree(data->bps);
2726 }
2727
2728 data->bps = bps;
2729 data->space_left += align_space - old_space;
2730 return 0;
2731 }
2732
amdgpu_ras_mca2pa_by_idx(struct amdgpu_device * adev,struct eeprom_table_record * bps,struct ras_err_data * err_data)2733 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
2734 struct eeprom_table_record *bps,
2735 struct ras_err_data *err_data)
2736 {
2737 struct ta_ras_query_address_input addr_in;
2738 uint32_t socket = 0;
2739 int ret = 0;
2740
2741 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2742 socket = adev->smuio.funcs->get_socket_id(adev);
2743
2744 /* reinit err_data */
2745 err_data->err_addr_cnt = 0;
2746 err_data->err_addr_len = adev->umc.retire_unit;
2747
2748 memset(&addr_in, 0, sizeof(addr_in));
2749 addr_in.ma.err_addr = bps->address;
2750 addr_in.ma.socket_id = socket;
2751 addr_in.ma.ch_inst = bps->mem_channel;
2752 /* tell RAS TA the node instance is not used */
2753 addr_in.ma.node_inst = TA_RAS_INV_NODE;
2754
2755 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2756 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2757 &addr_in, NULL, false);
2758
2759 return ret;
2760 }
2761
amdgpu_ras_mca2pa(struct amdgpu_device * adev,struct eeprom_table_record * bps,struct ras_err_data * err_data)2762 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2763 struct eeprom_table_record *bps,
2764 struct ras_err_data *err_data)
2765 {
2766 struct ta_ras_query_address_input addr_in;
2767 uint32_t die_id, socket = 0;
2768
2769 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2770 socket = adev->smuio.funcs->get_socket_id(adev);
2771
2772 /* although die id is gotten from PA in nps1 mode, the id is
2773 * fitable for any nps mode
2774 */
2775 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2776 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2777 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2778 else
2779 return -EINVAL;
2780
2781 /* reinit err_data */
2782 err_data->err_addr_cnt = 0;
2783 err_data->err_addr_len = adev->umc.retire_unit;
2784
2785 memset(&addr_in, 0, sizeof(addr_in));
2786 addr_in.ma.err_addr = bps->address;
2787 addr_in.ma.ch_inst = bps->mem_channel;
2788 addr_in.ma.umc_inst = bps->mcumc_id;
2789 addr_in.ma.node_inst = die_id;
2790 addr_in.ma.socket_id = socket;
2791
2792 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2793 return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2794 &addr_in, NULL, false);
2795 else
2796 return -EINVAL;
2797 }
2798
2799 /* it deal with vram only. */
amdgpu_ras_add_bad_pages(struct amdgpu_device * adev,struct eeprom_table_record * bps,int pages,bool from_rom)2800 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2801 struct eeprom_table_record *bps, int pages, bool from_rom)
2802 {
2803 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2804 struct ras_err_handler_data *data;
2805 struct ras_err_data err_data;
2806 struct eeprom_table_record *err_rec;
2807 struct amdgpu_ras_eeprom_control *control =
2808 &adev->psp.ras_context.ras->eeprom_control;
2809 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
2810 int ret = 0;
2811 uint32_t i, j, loop_cnt = 1;
2812 bool find_pages_per_pa = false;
2813
2814 if (!con || !con->eh_data || !bps || pages <= 0)
2815 return 0;
2816
2817 if (from_rom) {
2818 err_data.err_addr =
2819 kcalloc(adev->umc.retire_unit,
2820 sizeof(struct eeprom_table_record), GFP_KERNEL);
2821 if (!err_data.err_addr) {
2822 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
2823 ret = -ENOMEM;
2824 goto out;
2825 }
2826
2827 err_rec = err_data.err_addr;
2828 loop_cnt = adev->umc.retire_unit;
2829 if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2830 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
2831 }
2832
2833 mutex_lock(&con->recovery_lock);
2834 data = con->eh_data;
2835 if (!data) {
2836 /* Returning 0 as the absence of eh_data is acceptable */
2837 goto free;
2838 }
2839
2840 for (i = 0; i < pages; i++) {
2841 if (from_rom &&
2842 control->rec_type == AMDGPU_RAS_EEPROM_REC_MCA) {
2843 if (!find_pages_per_pa) {
2844 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2845 if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2846 /* may use old RAS TA, use PA to find pages in
2847 * one row
2848 */
2849 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2850 bps[i].retired_page <<
2851 AMDGPU_GPU_PAGE_SHIFT)) {
2852 ret = -EINVAL;
2853 goto free;
2854 } else {
2855 find_pages_per_pa = true;
2856 }
2857 } else {
2858 /* unsupported cases */
2859 ret = -EOPNOTSUPP;
2860 goto free;
2861 }
2862 }
2863 } else {
2864 if (amdgpu_umc_pages_in_a_row(adev, &err_data,
2865 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
2866 ret = -EINVAL;
2867 goto free;
2868 }
2869 }
2870 } else {
2871 if (from_rom && !find_pages_per_pa) {
2872 if (bps[i].retired_page & UMC_CHANNEL_IDX_V2) {
2873 /* bad page in any NPS mode in eeprom */
2874 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[i], &err_data)) {
2875 ret = -EINVAL;
2876 goto free;
2877 }
2878 } else {
2879 /* legacy bad page in eeprom, generated only in
2880 * NPS1 mode
2881 */
2882 if (amdgpu_ras_mca2pa(adev, &bps[i], &err_data)) {
2883 /* old RAS TA or ASICs which don't support to
2884 * convert addrss via mca address
2885 */
2886 if (!i && nps == AMDGPU_NPS1_PARTITION_MODE) {
2887 find_pages_per_pa = true;
2888 err_rec = &bps[i];
2889 loop_cnt = 1;
2890 } else {
2891 /* non-nps1 mode, old RAS TA
2892 * can't support it
2893 */
2894 ret = -EOPNOTSUPP;
2895 goto free;
2896 }
2897 }
2898 }
2899
2900 if (!find_pages_per_pa)
2901 i += (adev->umc.retire_unit - 1);
2902 } else {
2903 err_rec = &bps[i];
2904 }
2905 }
2906
2907 for (j = 0; j < loop_cnt; j++) {
2908 if (amdgpu_ras_check_bad_page_unlock(con,
2909 err_rec[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2910 continue;
2911
2912 if (!data->space_left &&
2913 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2914 ret = -ENOMEM;
2915 goto free;
2916 }
2917
2918 amdgpu_ras_reserve_page(adev, err_rec[j].retired_page);
2919
2920 memcpy(&data->bps[data->count], &(err_rec[j]),
2921 sizeof(struct eeprom_table_record));
2922 data->count++;
2923 data->space_left--;
2924 }
2925 }
2926
2927 free:
2928 if (from_rom)
2929 kfree(err_data.err_addr);
2930 out:
2931 mutex_unlock(&con->recovery_lock);
2932
2933 return ret;
2934 }
2935
2936 /*
2937 * write error record array to eeprom, the function should be
2938 * protected by recovery_lock
2939 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
2940 */
amdgpu_ras_save_bad_pages(struct amdgpu_device * adev,unsigned long * new_cnt)2941 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2942 unsigned long *new_cnt)
2943 {
2944 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2945 struct ras_err_handler_data *data;
2946 struct amdgpu_ras_eeprom_control *control;
2947 int save_count, unit_num, bad_page_num, i;
2948
2949 if (!con || !con->eh_data) {
2950 if (new_cnt)
2951 *new_cnt = 0;
2952
2953 return 0;
2954 }
2955
2956 mutex_lock(&con->recovery_lock);
2957 control = &con->eeprom_control;
2958 data = con->eh_data;
2959 bad_page_num = control->ras_num_bad_pages;
2960 save_count = data->count - bad_page_num;
2961 mutex_unlock(&con->recovery_lock);
2962
2963 unit_num = save_count / adev->umc.retire_unit;
2964 if (new_cnt)
2965 *new_cnt = unit_num;
2966
2967 /* only new entries are saved */
2968 if (save_count > 0) {
2969 if (control->rec_type == AMDGPU_RAS_EEPROM_REC_PA) {
2970 if (amdgpu_ras_eeprom_append(control,
2971 &data->bps[control->ras_num_recs],
2972 save_count)) {
2973 dev_err(adev->dev, "Failed to save EEPROM table data!");
2974 return -EIO;
2975 }
2976 } else {
2977 for (i = 0; i < unit_num; i++) {
2978 if (amdgpu_ras_eeprom_append(control,
2979 &data->bps[bad_page_num + i * adev->umc.retire_unit],
2980 1)) {
2981 dev_err(adev->dev, "Failed to save EEPROM table data!");
2982 return -EIO;
2983 }
2984 }
2985 }
2986
2987 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2988 }
2989
2990 return 0;
2991 }
2992
2993 /*
2994 * read error record array in eeprom and reserve enough space for
2995 * storing new bad pages
2996 */
amdgpu_ras_load_bad_pages(struct amdgpu_device * adev)2997 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2998 {
2999 struct amdgpu_ras_eeprom_control *control =
3000 &adev->psp.ras_context.ras->eeprom_control;
3001 struct eeprom_table_record *bps;
3002 int ret;
3003
3004 /* no bad page record, skip eeprom access */
3005 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3006 return 0;
3007
3008 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3009 if (!bps)
3010 return -ENOMEM;
3011
3012 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
3013 if (ret) {
3014 dev_err(adev->dev, "Failed to load EEPROM table records!");
3015 } else {
3016 if (control->ras_num_recs > 1 &&
3017 adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3018 if ((bps[0].address == bps[1].address) &&
3019 (bps[0].mem_channel == bps[1].mem_channel))
3020 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3021 else
3022 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3023 }
3024
3025 ret = amdgpu_ras_eeprom_check(control);
3026 if (ret)
3027 goto out;
3028
3029 /* HW not usable */
3030 if (amdgpu_ras_is_rma(adev)) {
3031 ret = -EHWPOISON;
3032 goto out;
3033 }
3034
3035 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
3036 }
3037
3038 out:
3039 kfree(bps);
3040 return ret;
3041 }
3042
amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras * con,uint64_t addr)3043 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3044 uint64_t addr)
3045 {
3046 struct ras_err_handler_data *data = con->eh_data;
3047 int i;
3048
3049 addr >>= AMDGPU_GPU_PAGE_SHIFT;
3050 for (i = 0; i < data->count; i++)
3051 if (addr == data->bps[i].retired_page)
3052 return true;
3053
3054 return false;
3055 }
3056
3057 /*
3058 * check if an address belongs to bad page
3059 *
3060 * Note: this check is only for umc block
3061 */
amdgpu_ras_check_bad_page(struct amdgpu_device * adev,uint64_t addr)3062 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3063 uint64_t addr)
3064 {
3065 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3066 bool ret = false;
3067
3068 if (!con || !con->eh_data)
3069 return ret;
3070
3071 mutex_lock(&con->recovery_lock);
3072 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3073 mutex_unlock(&con->recovery_lock);
3074 return ret;
3075 }
3076
amdgpu_ras_validate_threshold(struct amdgpu_device * adev,uint32_t max_count)3077 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3078 uint32_t max_count)
3079 {
3080 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3081
3082 /*
3083 * Justification of value bad_page_cnt_threshold in ras structure
3084 *
3085 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
3086 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
3087 * scenarios accordingly.
3088 *
3089 * Bad page retirement enablement:
3090 * - If amdgpu_bad_page_threshold = -2,
3091 * bad_page_cnt_threshold = typical value by formula.
3092 *
3093 * - When the value from user is 0 < amdgpu_bad_page_threshold <
3094 * max record length in eeprom, use it directly.
3095 *
3096 * Bad page retirement disablement:
3097 * - If amdgpu_bad_page_threshold = 0, bad page retirement
3098 * functionality is disabled, and bad_page_cnt_threshold will
3099 * take no effect.
3100 */
3101
3102 if (amdgpu_bad_page_threshold < 0) {
3103 u64 val = adev->gmc.mc_vram_size;
3104
3105 do_div(val, RAS_BAD_PAGE_COVER);
3106 con->bad_page_cnt_threshold = min(lower_32_bits(val),
3107 max_count);
3108 } else {
3109 con->bad_page_cnt_threshold = min_t(int, max_count,
3110 amdgpu_bad_page_threshold);
3111 }
3112 }
3113
amdgpu_ras_put_poison_req(struct amdgpu_device * adev,enum amdgpu_ras_block block,uint16_t pasid,pasid_notify pasid_fn,void * data,uint32_t reset)3114 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3115 enum amdgpu_ras_block block, uint16_t pasid,
3116 pasid_notify pasid_fn, void *data, uint32_t reset)
3117 {
3118 int ret = 0;
3119 struct ras_poison_msg poison_msg;
3120 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3121
3122 memset(&poison_msg, 0, sizeof(poison_msg));
3123 poison_msg.block = block;
3124 poison_msg.pasid = pasid;
3125 poison_msg.reset = reset;
3126 poison_msg.pasid_fn = pasid_fn;
3127 poison_msg.data = data;
3128
3129 ret = kfifo_put(&con->poison_fifo, poison_msg);
3130 if (!ret) {
3131 dev_err(adev->dev, "Poison message fifo is full!\n");
3132 return -ENOSPC;
3133 }
3134
3135 return 0;
3136 }
3137
amdgpu_ras_get_poison_req(struct amdgpu_device * adev,struct ras_poison_msg * poison_msg)3138 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3139 struct ras_poison_msg *poison_msg)
3140 {
3141 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3142
3143 return kfifo_get(&con->poison_fifo, poison_msg);
3144 }
3145
amdgpu_ras_ecc_log_init(struct ras_ecc_log_info * ecc_log)3146 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3147 {
3148 mutex_init(&ecc_log->lock);
3149
3150 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3151 ecc_log->de_queried_count = 0;
3152 ecc_log->prev_de_queried_count = 0;
3153 }
3154
amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info * ecc_log)3155 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3156 {
3157 struct radix_tree_iter iter;
3158 void __rcu **slot;
3159 struct ras_ecc_err *ecc_err;
3160
3161 mutex_lock(&ecc_log->lock);
3162 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3163 ecc_err = radix_tree_deref_slot(slot);
3164 kfree(ecc_err->err_pages.pfn);
3165 kfree(ecc_err);
3166 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3167 }
3168 mutex_unlock(&ecc_log->lock);
3169
3170 mutex_destroy(&ecc_log->lock);
3171 ecc_log->de_queried_count = 0;
3172 ecc_log->prev_de_queried_count = 0;
3173 }
3174
amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras * con,uint32_t delayed_ms)3175 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3176 uint32_t delayed_ms)
3177 {
3178 int ret;
3179
3180 mutex_lock(&con->umc_ecc_log.lock);
3181 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3182 UMC_ECC_NEW_DETECTED_TAG);
3183 mutex_unlock(&con->umc_ecc_log.lock);
3184
3185 if (ret)
3186 schedule_delayed_work(&con->page_retirement_dwork,
3187 msecs_to_jiffies(delayed_ms));
3188
3189 return ret ? true : false;
3190 }
3191
amdgpu_ras_do_page_retirement(struct work_struct * work)3192 static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3193 {
3194 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3195 page_retirement_dwork.work);
3196 struct amdgpu_device *adev = con->adev;
3197 struct ras_err_data err_data;
3198 unsigned long err_cnt;
3199
3200 /* If gpu reset is ongoing, delay retiring the bad pages */
3201 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3202 amdgpu_ras_schedule_retirement_dwork(con,
3203 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3204 return;
3205 }
3206
3207 amdgpu_ras_error_data_init(&err_data);
3208
3209 amdgpu_umc_handle_bad_pages(adev, &err_data);
3210 err_cnt = err_data.err_addr_cnt;
3211
3212 amdgpu_ras_error_data_fini(&err_data);
3213
3214 if (err_cnt && amdgpu_ras_is_rma(adev))
3215 amdgpu_ras_reset_gpu(adev);
3216
3217 amdgpu_ras_schedule_retirement_dwork(con,
3218 AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3219 }
3220
amdgpu_ras_poison_creation_handler(struct amdgpu_device * adev,uint32_t poison_creation_count)3221 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3222 uint32_t poison_creation_count)
3223 {
3224 int ret = 0;
3225 struct ras_ecc_log_info *ecc_log;
3226 struct ras_query_if info;
3227 uint32_t timeout = 0;
3228 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3229 uint64_t de_queried_count;
3230 uint32_t new_detect_count, total_detect_count;
3231 uint32_t need_query_count = poison_creation_count;
3232 bool query_data_timeout = false;
3233 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3234
3235 memset(&info, 0, sizeof(info));
3236 info.head.block = AMDGPU_RAS_BLOCK__UMC;
3237
3238 ecc_log = &ras->umc_ecc_log;
3239 total_detect_count = 0;
3240 do {
3241 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
3242 if (ret)
3243 return ret;
3244
3245 de_queried_count = ecc_log->de_queried_count;
3246 if (de_queried_count > ecc_log->prev_de_queried_count) {
3247 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3248 ecc_log->prev_de_queried_count = de_queried_count;
3249 timeout = 0;
3250 } else {
3251 new_detect_count = 0;
3252 }
3253
3254 if (new_detect_count) {
3255 total_detect_count += new_detect_count;
3256 } else {
3257 if (!timeout && need_query_count)
3258 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3259
3260 if (timeout) {
3261 if (!--timeout) {
3262 query_data_timeout = true;
3263 break;
3264 }
3265 msleep(1);
3266 }
3267 }
3268 } while (total_detect_count < need_query_count);
3269
3270 if (query_data_timeout) {
3271 dev_warn(adev->dev, "Can't find deferred error! count: %u\n",
3272 (need_query_count - total_detect_count));
3273 return -ENOENT;
3274 }
3275
3276 if (total_detect_count)
3277 schedule_delayed_work(&ras->page_retirement_dwork, 0);
3278
3279 return 0;
3280 }
3281
amdgpu_ras_clear_poison_fifo(struct amdgpu_device * adev)3282 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3283 {
3284 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3285 struct ras_poison_msg msg;
3286 int ret;
3287
3288 do {
3289 ret = kfifo_get(&con->poison_fifo, &msg);
3290 } while (ret);
3291 }
3292
amdgpu_ras_poison_consumption_handler(struct amdgpu_device * adev,uint32_t msg_count,uint32_t * gpu_reset)3293 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3294 uint32_t msg_count, uint32_t *gpu_reset)
3295 {
3296 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3297 uint32_t reset_flags = 0, reset = 0;
3298 struct ras_poison_msg msg;
3299 int ret, i;
3300
3301 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3302
3303 for (i = 0; i < msg_count; i++) {
3304 ret = amdgpu_ras_get_poison_req(adev, &msg);
3305 if (!ret)
3306 continue;
3307
3308 if (msg.pasid_fn)
3309 msg.pasid_fn(adev, msg.pasid, msg.data);
3310
3311 reset_flags |= msg.reset;
3312 }
3313
3314 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3315 if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3316 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3317 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3318 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3319 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3320 else
3321 reset = reset_flags;
3322
3323 flush_delayed_work(&con->page_retirement_dwork);
3324
3325 con->gpu_reset_flags |= reset;
3326 amdgpu_ras_reset_gpu(adev);
3327
3328 *gpu_reset = reset;
3329
3330 /* Wait for gpu recovery to complete */
3331 flush_work(&con->recovery_work);
3332 }
3333
3334 return 0;
3335 }
3336
amdgpu_ras_page_retirement_thread(void * param)3337 static int amdgpu_ras_page_retirement_thread(void *param)
3338 {
3339 struct amdgpu_device *adev = (struct amdgpu_device *)param;
3340 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3341 uint32_t poison_creation_count, msg_count;
3342 uint32_t gpu_reset;
3343 int ret;
3344
3345 while (!kthread_should_stop()) {
3346
3347 wait_event_interruptible(con->page_retirement_wq,
3348 kthread_should_stop() ||
3349 atomic_read(&con->page_retirement_req_cnt));
3350
3351 if (kthread_should_stop())
3352 break;
3353
3354 gpu_reset = 0;
3355
3356 do {
3357 poison_creation_count = atomic_read(&con->poison_creation_count);
3358 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3359 if (ret == -EIO)
3360 break;
3361
3362 if (poison_creation_count) {
3363 atomic_sub(poison_creation_count, &con->poison_creation_count);
3364 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3365 }
3366 } while (atomic_read(&con->poison_creation_count));
3367
3368 if (ret != -EIO) {
3369 msg_count = kfifo_len(&con->poison_fifo);
3370 if (msg_count) {
3371 ret = amdgpu_ras_poison_consumption_handler(adev,
3372 msg_count, &gpu_reset);
3373 if ((ret != -EIO) &&
3374 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3375 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3376 }
3377 }
3378
3379 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3380 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3381 /* Clear poison creation request */
3382 atomic_set(&con->poison_creation_count, 0);
3383
3384 /* Clear poison fifo */
3385 amdgpu_ras_clear_poison_fifo(adev);
3386
3387 /* Clear all poison requests */
3388 atomic_set(&con->page_retirement_req_cnt, 0);
3389
3390 if (ret == -EIO) {
3391 /* Wait for mode-1 reset to complete */
3392 down_read(&adev->reset_domain->sem);
3393 up_read(&adev->reset_domain->sem);
3394 }
3395
3396 /* Wake up work to save bad pages to eeprom */
3397 schedule_delayed_work(&con->page_retirement_dwork, 0);
3398 } else if (gpu_reset) {
3399 /* gpu just completed mode-2 reset or other reset */
3400 /* Clear poison consumption messages cached in fifo */
3401 msg_count = kfifo_len(&con->poison_fifo);
3402 if (msg_count) {
3403 amdgpu_ras_clear_poison_fifo(adev);
3404 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3405 }
3406
3407 /* Wake up work to save bad pages to eeprom */
3408 schedule_delayed_work(&con->page_retirement_dwork, 0);
3409 }
3410 }
3411
3412 return 0;
3413 }
3414
amdgpu_ras_init_badpage_info(struct amdgpu_device * adev)3415 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3416 {
3417 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3418 struct amdgpu_ras_eeprom_control *control;
3419 int ret;
3420
3421 if (!con || amdgpu_sriov_vf(adev))
3422 return 0;
3423
3424 control = &con->eeprom_control;
3425 ret = amdgpu_ras_eeprom_init(control);
3426 if (ret)
3427 return ret;
3428
3429 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3430 control->rec_type = AMDGPU_RAS_EEPROM_REC_PA;
3431
3432 /* default status is MCA storage */
3433 if (control->ras_num_recs <= 1 &&
3434 adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3435 control->rec_type = AMDGPU_RAS_EEPROM_REC_MCA;
3436
3437 if (control->ras_num_recs) {
3438 ret = amdgpu_ras_load_bad_pages(adev);
3439 if (ret)
3440 return ret;
3441
3442 amdgpu_dpm_send_hbm_bad_pages_num(
3443 adev, control->ras_num_bad_pages);
3444
3445 if (con->update_channel_flag == true) {
3446 amdgpu_dpm_send_hbm_bad_channel_flag(
3447 adev, control->bad_channel_bitmap);
3448 con->update_channel_flag = false;
3449 }
3450 }
3451
3452 return ret;
3453 }
3454
amdgpu_ras_recovery_init(struct amdgpu_device * adev,bool init_bp_info)3455 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3456 {
3457 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3458 struct ras_err_handler_data **data;
3459 u32 max_eeprom_records_count = 0;
3460 int ret;
3461
3462 if (!con || amdgpu_sriov_vf(adev))
3463 return 0;
3464
3465 /* Allow access to RAS EEPROM via debugfs, when the ASIC
3466 * supports RAS and debugfs is enabled, but when
3467 * adev->ras_enabled is unset, i.e. when "ras_enable"
3468 * module parameter is set to 0.
3469 */
3470 con->adev = adev;
3471
3472 if (!adev->ras_enabled)
3473 return 0;
3474
3475 data = &con->eh_data;
3476 *data = kzalloc(sizeof(**data), GFP_KERNEL);
3477 if (!*data) {
3478 ret = -ENOMEM;
3479 goto out;
3480 }
3481
3482 mutex_init(&con->recovery_lock);
3483 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3484 atomic_set(&con->in_recovery, 0);
3485 con->eeprom_control.bad_channel_bitmap = 0;
3486
3487 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
3488 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
3489
3490 if (init_bp_info) {
3491 ret = amdgpu_ras_init_badpage_info(adev);
3492 if (ret)
3493 goto free;
3494 }
3495
3496 mutex_init(&con->page_rsv_lock);
3497 INIT_KFIFO(con->poison_fifo);
3498 mutex_init(&con->page_retirement_lock);
3499 init_waitqueue_head(&con->page_retirement_wq);
3500 atomic_set(&con->page_retirement_req_cnt, 0);
3501 atomic_set(&con->poison_creation_count, 0);
3502 con->page_retirement_thread =
3503 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3504 if (IS_ERR(con->page_retirement_thread)) {
3505 con->page_retirement_thread = NULL;
3506 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3507 }
3508
3509 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3510 amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
3511 #ifdef CONFIG_X86_MCE_AMD
3512 if ((adev->asic_type == CHIP_ALDEBARAN) &&
3513 (adev->gmc.xgmi.connected_to_cpu))
3514 amdgpu_register_bad_pages_mca_notifier(adev);
3515 #endif
3516 return 0;
3517
3518 free:
3519 kfree((*data)->bps);
3520 kfree(*data);
3521 con->eh_data = NULL;
3522 out:
3523 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3524
3525 /*
3526 * Except error threshold exceeding case, other failure cases in this
3527 * function would not fail amdgpu driver init.
3528 */
3529 if (!amdgpu_ras_is_rma(adev))
3530 ret = 0;
3531 else
3532 ret = -EINVAL;
3533
3534 return ret;
3535 }
3536
amdgpu_ras_recovery_fini(struct amdgpu_device * adev)3537 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3538 {
3539 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3540 struct ras_err_handler_data *data = con->eh_data;
3541 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3542 bool ret;
3543
3544 /* recovery_init failed to init it, fini is useless */
3545 if (!data)
3546 return 0;
3547
3548 /* Save all cached bad pages to eeprom */
3549 do {
3550 flush_delayed_work(&con->page_retirement_dwork);
3551 ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3552 } while (ret && max_flush_timeout--);
3553
3554 if (con->page_retirement_thread)
3555 kthread_stop(con->page_retirement_thread);
3556
3557 atomic_set(&con->page_retirement_req_cnt, 0);
3558 atomic_set(&con->poison_creation_count, 0);
3559
3560 mutex_destroy(&con->page_rsv_lock);
3561
3562 cancel_work_sync(&con->recovery_work);
3563
3564 cancel_delayed_work_sync(&con->page_retirement_dwork);
3565
3566 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3567
3568 mutex_lock(&con->recovery_lock);
3569 con->eh_data = NULL;
3570 kfree(data->bps);
3571 kfree(data);
3572 mutex_unlock(&con->recovery_lock);
3573
3574 return 0;
3575 }
3576 /* recovery end */
3577
amdgpu_ras_asic_supported(struct amdgpu_device * adev)3578 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3579 {
3580 if (amdgpu_sriov_vf(adev)) {
3581 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3582 case IP_VERSION(13, 0, 2):
3583 case IP_VERSION(13, 0, 6):
3584 case IP_VERSION(13, 0, 12):
3585 case IP_VERSION(13, 0, 14):
3586 return true;
3587 default:
3588 return false;
3589 }
3590 }
3591
3592 if (adev->asic_type == CHIP_IP_DISCOVERY) {
3593 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3594 case IP_VERSION(13, 0, 0):
3595 case IP_VERSION(13, 0, 6):
3596 case IP_VERSION(13, 0, 10):
3597 case IP_VERSION(13, 0, 12):
3598 case IP_VERSION(13, 0, 14):
3599 case IP_VERSION(14, 0, 3):
3600 return true;
3601 default:
3602 return false;
3603 }
3604 }
3605
3606 return adev->asic_type == CHIP_VEGA10 ||
3607 adev->asic_type == CHIP_VEGA20 ||
3608 adev->asic_type == CHIP_ARCTURUS ||
3609 adev->asic_type == CHIP_ALDEBARAN ||
3610 adev->asic_type == CHIP_SIENNA_CICHLID;
3611 }
3612
3613 /*
3614 * this is workaround for vega20 workstation sku,
3615 * force enable gfx ras, ignore vbios gfx ras flag
3616 * due to GC EDC can not write
3617 */
amdgpu_ras_get_quirks(struct amdgpu_device * adev)3618 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
3619 {
3620 struct atom_context *ctx = adev->mode_info.atom_context;
3621
3622 if (!ctx)
3623 return;
3624
3625 if (strnstr(ctx->vbios_pn, "D16406",
3626 sizeof(ctx->vbios_pn)) ||
3627 strnstr(ctx->vbios_pn, "D36002",
3628 sizeof(ctx->vbios_pn)))
3629 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
3630 }
3631
3632 /* Query ras capablity via atomfirmware interface */
amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device * adev)3633 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3634 {
3635 /* mem_ecc cap */
3636 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3637 dev_info(adev->dev, "MEM ECC is active.\n");
3638 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3639 1 << AMDGPU_RAS_BLOCK__DF);
3640 } else {
3641 dev_info(adev->dev, "MEM ECC is not presented.\n");
3642 }
3643
3644 /* sram_ecc cap */
3645 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3646 dev_info(adev->dev, "SRAM ECC is active.\n");
3647 if (!amdgpu_sriov_vf(adev))
3648 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3649 1 << AMDGPU_RAS_BLOCK__DF);
3650 else
3651 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3652 1 << AMDGPU_RAS_BLOCK__SDMA |
3653 1 << AMDGPU_RAS_BLOCK__GFX);
3654
3655 /*
3656 * VCN/JPEG RAS can be supported on both bare metal and
3657 * SRIOV environment
3658 */
3659 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3660 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
3661 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3))
3662 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3663 1 << AMDGPU_RAS_BLOCK__JPEG);
3664 else
3665 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3666 1 << AMDGPU_RAS_BLOCK__JPEG);
3667
3668 /*
3669 * XGMI RAS is not supported if xgmi num physical nodes
3670 * is zero
3671 */
3672 if (!adev->gmc.xgmi.num_physical_nodes)
3673 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3674 } else {
3675 dev_info(adev->dev, "SRAM ECC is not presented.\n");
3676 }
3677 }
3678
3679 /* Query poison mode from umc/df IP callbacks */
amdgpu_ras_query_poison_mode(struct amdgpu_device * adev)3680 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3681 {
3682 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3683 bool df_poison, umc_poison;
3684
3685 /* poison setting is useless on SRIOV guest */
3686 if (amdgpu_sriov_vf(adev) || !con)
3687 return;
3688
3689 /* Init poison supported flag, the default value is false */
3690 if (adev->gmc.xgmi.connected_to_cpu ||
3691 adev->gmc.is_app_apu) {
3692 /* enabled by default when GPU is connected to CPU */
3693 con->poison_supported = true;
3694 } else if (adev->df.funcs &&
3695 adev->df.funcs->query_ras_poison_mode &&
3696 adev->umc.ras &&
3697 adev->umc.ras->query_ras_poison_mode) {
3698 df_poison =
3699 adev->df.funcs->query_ras_poison_mode(adev);
3700 umc_poison =
3701 adev->umc.ras->query_ras_poison_mode(adev);
3702
3703 /* Only poison is set in both DF and UMC, we can support it */
3704 if (df_poison && umc_poison)
3705 con->poison_supported = true;
3706 else if (df_poison != umc_poison)
3707 dev_warn(adev->dev,
3708 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3709 df_poison, umc_poison);
3710 }
3711 }
3712
3713 /*
3714 * check hardware's ras ability which will be saved in hw_supported.
3715 * if hardware does not support ras, we can skip some ras initializtion and
3716 * forbid some ras operations from IP.
3717 * if software itself, say boot parameter, limit the ras ability. We still
3718 * need allow IP do some limited operations, like disable. In such case,
3719 * we have to initialize ras as normal. but need check if operation is
3720 * allowed or not in each function.
3721 */
amdgpu_ras_check_supported(struct amdgpu_device * adev)3722 static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
3723 {
3724 adev->ras_hw_enabled = adev->ras_enabled = 0;
3725
3726 if (!amdgpu_ras_asic_supported(adev))
3727 return;
3728
3729 if (amdgpu_sriov_vf(adev)) {
3730 if (amdgpu_virt_get_ras_capability(adev))
3731 goto init_ras_enabled_flag;
3732 }
3733
3734 /* query ras capability from psp */
3735 if (amdgpu_psp_get_ras_capability(&adev->psp))
3736 goto init_ras_enabled_flag;
3737
3738 /* query ras capablity from bios */
3739 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3740 amdgpu_ras_query_ras_capablity_from_vbios(adev);
3741 } else {
3742 /* driver only manages a few IP blocks RAS feature
3743 * when GPU is connected cpu through XGMI */
3744 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
3745 1 << AMDGPU_RAS_BLOCK__SDMA |
3746 1 << AMDGPU_RAS_BLOCK__MMHUB);
3747 }
3748
3749 /* apply asic specific settings (vega20 only for now) */
3750 amdgpu_ras_get_quirks(adev);
3751
3752 /* query poison mode from umc/df ip callback */
3753 amdgpu_ras_query_poison_mode(adev);
3754
3755 init_ras_enabled_flag:
3756 /* hw_supported needs to be aligned with RAS block mask. */
3757 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
3758
3759 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3760 adev->ras_hw_enabled & amdgpu_ras_mask;
3761
3762 /* aca is disabled by default */
3763 adev->aca.is_enabled = false;
3764
3765 /* bad page feature is not applicable to specific app platform */
3766 if (adev->gmc.is_app_apu &&
3767 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3768 amdgpu_bad_page_threshold = 0;
3769 }
3770
amdgpu_ras_counte_dw(struct work_struct * work)3771 static void amdgpu_ras_counte_dw(struct work_struct *work)
3772 {
3773 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3774 ras_counte_delay_work.work);
3775 struct amdgpu_device *adev = con->adev;
3776 struct drm_device *dev = adev_to_drm(adev);
3777 unsigned long ce_count, ue_count;
3778 int res;
3779
3780 res = pm_runtime_get_sync(dev->dev);
3781 if (res < 0)
3782 goto Out;
3783
3784 /* Cache new values.
3785 */
3786 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
3787 atomic_set(&con->ras_ce_count, ce_count);
3788 atomic_set(&con->ras_ue_count, ue_count);
3789 }
3790
3791 pm_runtime_mark_last_busy(dev->dev);
3792 Out:
3793 pm_runtime_put_autosuspend(dev->dev);
3794 }
3795
amdgpu_get_ras_schema(struct amdgpu_device * adev)3796 static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3797 {
3798 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3799 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3800 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3801 AMDGPU_RAS_ERROR__PARITY;
3802 }
3803
ras_event_mgr_init(struct ras_event_manager * mgr)3804 static void ras_event_mgr_init(struct ras_event_manager *mgr)
3805 {
3806 struct ras_event_state *event_state;
3807 int i;
3808
3809 memset(mgr, 0, sizeof(*mgr));
3810 atomic64_set(&mgr->seqno, 0);
3811
3812 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3813 event_state = &mgr->event_state[i];
3814 event_state->last_seqno = RAS_EVENT_INVALID_ID;
3815 atomic64_set(&event_state->count, 0);
3816 }
3817 }
3818
amdgpu_ras_event_mgr_init(struct amdgpu_device * adev)3819 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3820 {
3821 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3822 struct amdgpu_hive_info *hive;
3823
3824 if (!ras)
3825 return;
3826
3827 hive = amdgpu_get_xgmi_hive(adev);
3828 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3829
3830 /* init event manager with node 0 on xgmi system */
3831 if (!amdgpu_reset_in_recovery(adev)) {
3832 if (!hive || adev->gmc.xgmi.node_id == 0)
3833 ras_event_mgr_init(ras->event_mgr);
3834 }
3835
3836 if (hive)
3837 amdgpu_put_xgmi_hive(hive);
3838 }
3839
amdgpu_ras_init_reserved_vram_size(struct amdgpu_device * adev)3840 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3841 {
3842 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3843
3844 if (!con || (adev->flags & AMD_IS_APU))
3845 return;
3846
3847 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3848 case IP_VERSION(13, 0, 2):
3849 case IP_VERSION(13, 0, 6):
3850 case IP_VERSION(13, 0, 12):
3851 case IP_VERSION(13, 0, 14):
3852 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE;
3853 break;
3854 default:
3855 break;
3856 }
3857 }
3858
amdgpu_ras_init(struct amdgpu_device * adev)3859 int amdgpu_ras_init(struct amdgpu_device *adev)
3860 {
3861 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3862 int r;
3863
3864 if (con)
3865 return 0;
3866
3867 con = kzalloc(sizeof(*con) +
3868 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3869 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
3870 GFP_KERNEL);
3871 if (!con)
3872 return -ENOMEM;
3873
3874 con->adev = adev;
3875 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3876 atomic_set(&con->ras_ce_count, 0);
3877 atomic_set(&con->ras_ue_count, 0);
3878
3879 con->objs = (struct ras_manager *)(con + 1);
3880
3881 amdgpu_ras_set_context(adev, con);
3882
3883 amdgpu_ras_check_supported(adev);
3884
3885 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
3886 /* set gfx block ras context feature for VEGA20 Gaming
3887 * send ras disable cmd to ras ta during ras late init.
3888 */
3889 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
3890 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3891
3892 return 0;
3893 }
3894
3895 r = 0;
3896 goto release_con;
3897 }
3898
3899 con->update_channel_flag = false;
3900 con->features = 0;
3901 con->schema = 0;
3902 INIT_LIST_HEAD(&con->head);
3903 /* Might need get this flag from vbios. */
3904 con->flags = RAS_DEFAULT_FLAGS;
3905
3906 /* initialize nbio ras function ahead of any other
3907 * ras functions so hardware fatal error interrupt
3908 * can be enabled as early as possible */
3909 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
3910 case IP_VERSION(7, 4, 0):
3911 case IP_VERSION(7, 4, 1):
3912 case IP_VERSION(7, 4, 4):
3913 if (!adev->gmc.xgmi.connected_to_cpu)
3914 adev->nbio.ras = &nbio_v7_4_ras;
3915 break;
3916 case IP_VERSION(4, 3, 0):
3917 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3918 /* unlike other generation of nbio ras,
3919 * nbio v4_3 only support fatal error interrupt
3920 * to inform software that DF is freezed due to
3921 * system fatal error event. driver should not
3922 * enable nbio ras in such case. Instead,
3923 * check DF RAS */
3924 adev->nbio.ras = &nbio_v4_3_ras;
3925 break;
3926 case IP_VERSION(6, 3, 1):
3927 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3928 /* unlike other generation of nbio ras,
3929 * nbif v6_3_1 only support fatal error interrupt
3930 * to inform software that DF is freezed due to
3931 * system fatal error event. driver should not
3932 * enable nbio ras in such case. Instead,
3933 * check DF RAS
3934 */
3935 adev->nbio.ras = &nbif_v6_3_1_ras;
3936 break;
3937 case IP_VERSION(7, 9, 0):
3938 case IP_VERSION(7, 9, 1):
3939 if (!adev->gmc.is_app_apu)
3940 adev->nbio.ras = &nbio_v7_9_ras;
3941 break;
3942 default:
3943 /* nbio ras is not available */
3944 break;
3945 }
3946
3947 /* nbio ras block needs to be enabled ahead of other ras blocks
3948 * to handle fatal error */
3949 r = amdgpu_nbio_ras_sw_init(adev);
3950 if (r)
3951 return r;
3952
3953 if (adev->nbio.ras &&
3954 adev->nbio.ras->init_ras_controller_interrupt) {
3955 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
3956 if (r)
3957 goto release_con;
3958 }
3959
3960 if (adev->nbio.ras &&
3961 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
3962 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
3963 if (r)
3964 goto release_con;
3965 }
3966
3967 /* Packed socket_id to ras feature mask bits[31:29] */
3968 if (adev->smuio.funcs &&
3969 adev->smuio.funcs->get_socket_id)
3970 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
3971 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
3972
3973 /* Get RAS schema for particular SOC */
3974 con->schema = amdgpu_get_ras_schema(adev);
3975
3976 amdgpu_ras_init_reserved_vram_size(adev);
3977
3978 if (amdgpu_ras_fs_init(adev)) {
3979 r = -EINVAL;
3980 goto release_con;
3981 }
3982
3983 if (amdgpu_ras_aca_is_supported(adev)) {
3984 if (amdgpu_aca_is_enabled(adev))
3985 r = amdgpu_aca_init(adev);
3986 else
3987 r = amdgpu_mca_init(adev);
3988 if (r)
3989 goto release_con;
3990 }
3991
3992 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
3993 "hardware ability[%x] ras_mask[%x]\n",
3994 adev->ras_hw_enabled, adev->ras_enabled);
3995
3996 return 0;
3997 release_con:
3998 amdgpu_ras_set_context(adev, NULL);
3999 kfree(con);
4000
4001 return r;
4002 }
4003
amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device * adev)4004 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4005 {
4006 if (adev->gmc.xgmi.connected_to_cpu ||
4007 adev->gmc.is_app_apu)
4008 return 1;
4009 return 0;
4010 }
4011
amdgpu_persistent_edc_harvesting(struct amdgpu_device * adev,struct ras_common_if * ras_block)4012 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4013 struct ras_common_if *ras_block)
4014 {
4015 struct ras_query_if info = {
4016 .head = *ras_block,
4017 };
4018
4019 if (!amdgpu_persistent_edc_harvesting_supported(adev))
4020 return 0;
4021
4022 if (amdgpu_ras_query_error_status(adev, &info) != 0)
4023 DRM_WARN("RAS init harvest failure");
4024
4025 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4026 DRM_WARN("RAS init harvest reset failure");
4027
4028 return 0;
4029 }
4030
amdgpu_ras_is_poison_mode_supported(struct amdgpu_device * adev)4031 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4032 {
4033 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4034
4035 if (!con)
4036 return false;
4037
4038 return con->poison_supported;
4039 }
4040
4041 /* helper function to handle common stuff in ip late init phase */
amdgpu_ras_block_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)4042 int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4043 struct ras_common_if *ras_block)
4044 {
4045 struct amdgpu_ras_block_object *ras_obj = NULL;
4046 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4047 struct ras_query_if *query_info;
4048 unsigned long ue_count, ce_count;
4049 int r;
4050
4051 /* disable RAS feature per IP block if it is not supported */
4052 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4053 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4054 return 0;
4055 }
4056
4057 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4058 if (r) {
4059 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4060 /* in resume phase, if fail to enable ras,
4061 * clean up all ras fs nodes, and disable ras */
4062 goto cleanup;
4063 } else
4064 return r;
4065 }
4066
4067 /* check for errors on warm reset edc persisant supported ASIC */
4068 amdgpu_persistent_edc_harvesting(adev, ras_block);
4069
4070 /* in resume phase, no need to create ras fs node */
4071 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4072 return 0;
4073
4074 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4075 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4076 (ras_obj->hw_ops->query_poison_status ||
4077 ras_obj->hw_ops->handle_poison_consumption))) {
4078 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
4079 if (r)
4080 goto cleanup;
4081 }
4082
4083 if (ras_obj->hw_ops &&
4084 (ras_obj->hw_ops->query_ras_error_count ||
4085 ras_obj->hw_ops->query_ras_error_status)) {
4086 r = amdgpu_ras_sysfs_create(adev, ras_block);
4087 if (r)
4088 goto interrupt;
4089
4090 /* Those are the cached values at init.
4091 */
4092 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4093 if (!query_info)
4094 return -ENOMEM;
4095 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4096
4097 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4098 atomic_set(&con->ras_ce_count, ce_count);
4099 atomic_set(&con->ras_ue_count, ue_count);
4100 }
4101
4102 kfree(query_info);
4103 }
4104
4105 return 0;
4106
4107 interrupt:
4108 if (ras_obj->ras_cb)
4109 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4110 cleanup:
4111 amdgpu_ras_feature_enable(adev, ras_block, 0);
4112 return r;
4113 }
4114
amdgpu_ras_block_late_init_default(struct amdgpu_device * adev,struct ras_common_if * ras_block)4115 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4116 struct ras_common_if *ras_block)
4117 {
4118 return amdgpu_ras_block_late_init(adev, ras_block);
4119 }
4120
4121 /* helper function to remove ras fs node and interrupt handler */
amdgpu_ras_block_late_fini(struct amdgpu_device * adev,struct ras_common_if * ras_block)4122 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4123 struct ras_common_if *ras_block)
4124 {
4125 struct amdgpu_ras_block_object *ras_obj;
4126 if (!ras_block)
4127 return;
4128
4129 amdgpu_ras_sysfs_remove(adev, ras_block);
4130
4131 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4132 if (ras_obj->ras_cb)
4133 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4134 }
4135
amdgpu_ras_block_late_fini_default(struct amdgpu_device * adev,struct ras_common_if * ras_block)4136 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4137 struct ras_common_if *ras_block)
4138 {
4139 return amdgpu_ras_block_late_fini(adev, ras_block);
4140 }
4141
4142 /* do some init work after IP late init as dependence.
4143 * and it runs in resume/gpu reset/booting up cases.
4144 */
amdgpu_ras_resume(struct amdgpu_device * adev)4145 void amdgpu_ras_resume(struct amdgpu_device *adev)
4146 {
4147 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4148 struct ras_manager *obj, *tmp;
4149
4150 if (!adev->ras_enabled || !con) {
4151 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
4152 amdgpu_release_ras_context(adev);
4153
4154 return;
4155 }
4156
4157 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4158 /* Set up all other IPs which are not implemented. There is a
4159 * tricky thing that IP's actual ras error type should be
4160 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4161 * ERROR_NONE make sense anyway.
4162 */
4163 amdgpu_ras_enable_all_features(adev, 1);
4164
4165 /* We enable ras on all hw_supported block, but as boot
4166 * parameter might disable some of them and one or more IP has
4167 * not implemented yet. So we disable them on behalf.
4168 */
4169 list_for_each_entry_safe(obj, tmp, &con->head, node) {
4170 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4171 amdgpu_ras_feature_enable(adev, &obj->head, 0);
4172 /* there should be no any reference. */
4173 WARN_ON(alive_obj(obj));
4174 }
4175 }
4176 }
4177 }
4178
amdgpu_ras_suspend(struct amdgpu_device * adev)4179 void amdgpu_ras_suspend(struct amdgpu_device *adev)
4180 {
4181 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4182
4183 if (!adev->ras_enabled || !con)
4184 return;
4185
4186 amdgpu_ras_disable_all_features(adev, 0);
4187 /* Make sure all ras objects are disabled. */
4188 if (AMDGPU_RAS_GET_FEATURES(con->features))
4189 amdgpu_ras_disable_all_features(adev, 1);
4190 }
4191
amdgpu_ras_late_init(struct amdgpu_device * adev)4192 int amdgpu_ras_late_init(struct amdgpu_device *adev)
4193 {
4194 struct amdgpu_ras_block_list *node, *tmp;
4195 struct amdgpu_ras_block_object *obj;
4196 int r;
4197
4198 amdgpu_ras_event_mgr_init(adev);
4199
4200 if (amdgpu_ras_aca_is_supported(adev)) {
4201 if (amdgpu_reset_in_recovery(adev)) {
4202 if (amdgpu_aca_is_enabled(adev))
4203 r = amdgpu_aca_reset(adev);
4204 else
4205 r = amdgpu_mca_reset(adev);
4206 if (r)
4207 return r;
4208 }
4209
4210 if (!amdgpu_sriov_vf(adev)) {
4211 if (amdgpu_aca_is_enabled(adev))
4212 amdgpu_ras_set_aca_debug_mode(adev, false);
4213 else
4214 amdgpu_ras_set_mca_debug_mode(adev, false);
4215 }
4216 }
4217
4218 /* Guest side doesn't need init ras feature */
4219 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4220 return 0;
4221
4222 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4223 obj = node->ras_obj;
4224 if (!obj) {
4225 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4226 continue;
4227 }
4228
4229 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4230 continue;
4231
4232 if (obj->ras_late_init) {
4233 r = obj->ras_late_init(adev, &obj->ras_comm);
4234 if (r) {
4235 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4236 obj->ras_comm.name, r);
4237 return r;
4238 }
4239 } else
4240 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
4241 }
4242
4243 return 0;
4244 }
4245
4246 /* do some fini work before IP fini as dependence */
amdgpu_ras_pre_fini(struct amdgpu_device * adev)4247 int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4248 {
4249 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4250
4251 if (!adev->ras_enabled || !con)
4252 return 0;
4253
4254
4255 /* Need disable ras on all IPs here before ip [hw/sw]fini */
4256 if (AMDGPU_RAS_GET_FEATURES(con->features))
4257 amdgpu_ras_disable_all_features(adev, 0);
4258 amdgpu_ras_recovery_fini(adev);
4259 return 0;
4260 }
4261
amdgpu_ras_fini(struct amdgpu_device * adev)4262 int amdgpu_ras_fini(struct amdgpu_device *adev)
4263 {
4264 struct amdgpu_ras_block_list *ras_node, *tmp;
4265 struct amdgpu_ras_block_object *obj = NULL;
4266 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4267
4268 if (!adev->ras_enabled || !con)
4269 return 0;
4270
4271 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4272 if (ras_node->ras_obj) {
4273 obj = ras_node->ras_obj;
4274 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4275 obj->ras_fini)
4276 obj->ras_fini(adev, &obj->ras_comm);
4277 else
4278 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
4279 }
4280
4281 /* Clear ras blocks from ras_list and free ras block list node */
4282 list_del(&ras_node->node);
4283 kfree(ras_node);
4284 }
4285
4286 amdgpu_ras_fs_fini(adev);
4287 amdgpu_ras_interrupt_remove_all(adev);
4288
4289 if (amdgpu_ras_aca_is_supported(adev)) {
4290 if (amdgpu_aca_is_enabled(adev))
4291 amdgpu_aca_fini(adev);
4292 else
4293 amdgpu_mca_fini(adev);
4294 }
4295
4296 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4297
4298 if (AMDGPU_RAS_GET_FEATURES(con->features))
4299 amdgpu_ras_disable_all_features(adev, 0);
4300
4301 cancel_delayed_work_sync(&con->ras_counte_delay_work);
4302
4303 amdgpu_ras_set_context(adev, NULL);
4304 kfree(con);
4305
4306 return 0;
4307 }
4308
amdgpu_ras_get_fed_status(struct amdgpu_device * adev)4309 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4310 {
4311 struct amdgpu_ras *ras;
4312
4313 ras = amdgpu_ras_get_context(adev);
4314 if (!ras)
4315 return false;
4316
4317 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4318 }
4319
amdgpu_ras_set_fed(struct amdgpu_device * adev,bool status)4320 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4321 {
4322 struct amdgpu_ras *ras;
4323
4324 ras = amdgpu_ras_get_context(adev);
4325 if (ras) {
4326 if (status)
4327 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4328 else
4329 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4330 }
4331 }
4332
amdgpu_ras_clear_err_state(struct amdgpu_device * adev)4333 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4334 {
4335 struct amdgpu_ras *ras;
4336
4337 ras = amdgpu_ras_get_context(adev);
4338 if (ras)
4339 ras->ras_err_state = 0;
4340 }
4341
amdgpu_ras_set_err_poison(struct amdgpu_device * adev,enum amdgpu_ras_block block)4342 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4343 enum amdgpu_ras_block block)
4344 {
4345 struct amdgpu_ras *ras;
4346
4347 ras = amdgpu_ras_get_context(adev);
4348 if (ras)
4349 set_bit(block, &ras->ras_err_state);
4350 }
4351
amdgpu_ras_is_err_state(struct amdgpu_device * adev,int block)4352 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4353 {
4354 struct amdgpu_ras *ras;
4355
4356 ras = amdgpu_ras_get_context(adev);
4357 if (ras) {
4358 if (block == AMDGPU_RAS_BLOCK__ANY)
4359 return (ras->ras_err_state != 0);
4360 else
4361 return test_bit(block, &ras->ras_err_state) ||
4362 test_bit(AMDGPU_RAS_BLOCK__LAST,
4363 &ras->ras_err_state);
4364 }
4365
4366 return false;
4367 }
4368
__get_ras_event_mgr(struct amdgpu_device * adev)4369 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4370 {
4371 struct amdgpu_ras *ras;
4372
4373 ras = amdgpu_ras_get_context(adev);
4374 if (!ras)
4375 return NULL;
4376
4377 return ras->event_mgr;
4378 }
4379
amdgpu_ras_mark_ras_event_caller(struct amdgpu_device * adev,enum ras_event_type type,const void * caller)4380 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4381 const void *caller)
4382 {
4383 struct ras_event_manager *event_mgr;
4384 struct ras_event_state *event_state;
4385 int ret = 0;
4386
4387 if (type >= RAS_EVENT_TYPE_COUNT) {
4388 ret = -EINVAL;
4389 goto out;
4390 }
4391
4392 event_mgr = __get_ras_event_mgr(adev);
4393 if (!event_mgr) {
4394 ret = -EINVAL;
4395 goto out;
4396 }
4397
4398 event_state = &event_mgr->event_state[type];
4399 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4400 atomic64_inc(&event_state->count);
4401
4402 out:
4403 if (ret && caller)
4404 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4405 (int)type, caller, ret);
4406
4407 return ret;
4408 }
4409
amdgpu_ras_acquire_event_id(struct amdgpu_device * adev,enum ras_event_type type)4410 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4411 {
4412 struct ras_event_manager *event_mgr;
4413 u64 id;
4414
4415 if (type >= RAS_EVENT_TYPE_COUNT)
4416 return RAS_EVENT_INVALID_ID;
4417
4418 switch (type) {
4419 case RAS_EVENT_TYPE_FATAL:
4420 case RAS_EVENT_TYPE_POISON_CREATION:
4421 case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4422 event_mgr = __get_ras_event_mgr(adev);
4423 if (!event_mgr)
4424 return RAS_EVENT_INVALID_ID;
4425
4426 id = event_mgr->event_state[type].last_seqno;
4427 break;
4428 case RAS_EVENT_TYPE_INVALID:
4429 default:
4430 id = RAS_EVENT_INVALID_ID;
4431 break;
4432 }
4433
4434 return id;
4435 }
4436
amdgpu_ras_global_ras_isr(struct amdgpu_device * adev)4437 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4438 {
4439 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
4440 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4441 enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4442 u64 event_id;
4443
4444 if (amdgpu_ras_mark_ras_event(adev, type))
4445 return;
4446
4447 event_id = amdgpu_ras_acquire_event_id(adev, type);
4448
4449 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4450 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4451
4452 amdgpu_ras_set_fed(adev, true);
4453 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4454 amdgpu_ras_reset_gpu(adev);
4455 }
4456 }
4457
amdgpu_ras_need_emergency_restart(struct amdgpu_device * adev)4458 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4459 {
4460 if (adev->asic_type == CHIP_VEGA20 &&
4461 adev->pm.fw_version <= 0x283400) {
4462 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4463 amdgpu_ras_intr_triggered();
4464 }
4465
4466 return false;
4467 }
4468
amdgpu_release_ras_context(struct amdgpu_device * adev)4469 void amdgpu_release_ras_context(struct amdgpu_device *adev)
4470 {
4471 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4472
4473 if (!con)
4474 return;
4475
4476 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4477 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4478 amdgpu_ras_set_context(adev, NULL);
4479 kfree(con);
4480 }
4481 }
4482
4483 #ifdef CONFIG_X86_MCE_AMD
find_adev(uint32_t node_id)4484 static struct amdgpu_device *find_adev(uint32_t node_id)
4485 {
4486 int i;
4487 struct amdgpu_device *adev = NULL;
4488
4489 for (i = 0; i < mce_adev_list.num_gpu; i++) {
4490 adev = mce_adev_list.devs[i];
4491
4492 if (adev && adev->gmc.xgmi.connected_to_cpu &&
4493 adev->gmc.xgmi.physical_node_id == node_id)
4494 break;
4495 adev = NULL;
4496 }
4497
4498 return adev;
4499 }
4500
4501 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
4502 #define GET_UMC_INST(m) (((m) >> 21) & 0x7)
4503 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4504 #define GPU_ID_OFFSET 8
4505
amdgpu_bad_page_notifier(struct notifier_block * nb,unsigned long val,void * data)4506 static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4507 unsigned long val, void *data)
4508 {
4509 struct mce *m = (struct mce *)data;
4510 struct amdgpu_device *adev = NULL;
4511 uint32_t gpu_id = 0;
4512 uint32_t umc_inst = 0, ch_inst = 0;
4513
4514 /*
4515 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4516 * and error occurred in DramECC (Extended error code = 0) then only
4517 * process the error, else bail out.
4518 */
4519 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
4520 (XEC(m->status, 0x3f) == 0x0)))
4521 return NOTIFY_DONE;
4522
4523 /*
4524 * If it is correctable error, return.
4525 */
4526 if (mce_is_correctable(m))
4527 return NOTIFY_OK;
4528
4529 /*
4530 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4531 */
4532 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4533
4534 adev = find_adev(gpu_id);
4535 if (!adev) {
4536 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4537 gpu_id);
4538 return NOTIFY_DONE;
4539 }
4540
4541 /*
4542 * If it is uncorrectable error, then find out UMC instance and
4543 * channel index.
4544 */
4545 umc_inst = GET_UMC_INST(m->ipid);
4546 ch_inst = GET_CHAN_INDEX(m->ipid);
4547
4548 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4549 umc_inst, ch_inst);
4550
4551 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4552 return NOTIFY_OK;
4553 else
4554 return NOTIFY_DONE;
4555 }
4556
4557 static struct notifier_block amdgpu_bad_page_nb = {
4558 .notifier_call = amdgpu_bad_page_notifier,
4559 .priority = MCE_PRIO_UC,
4560 };
4561
amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device * adev)4562 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4563 {
4564 /*
4565 * Add the adev to the mce_adev_list.
4566 * During mode2 reset, amdgpu device is temporarily
4567 * removed from the mgpu_info list which can cause
4568 * page retirement to fail.
4569 * Use this list instead of mgpu_info to find the amdgpu
4570 * device on which the UMC error was reported.
4571 */
4572 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4573
4574 /*
4575 * Register the x86 notifier only once
4576 * with MCE subsystem.
4577 */
4578 if (notifier_registered == false) {
4579 mce_register_decode_chain(&amdgpu_bad_page_nb);
4580 notifier_registered = true;
4581 }
4582 }
4583 #endif
4584
amdgpu_ras_get_context(struct amdgpu_device * adev)4585 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
4586 {
4587 if (!adev)
4588 return NULL;
4589
4590 return adev->psp.ras_context.ras;
4591 }
4592
amdgpu_ras_set_context(struct amdgpu_device * adev,struct amdgpu_ras * ras_con)4593 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
4594 {
4595 if (!adev)
4596 return -EINVAL;
4597
4598 adev->psp.ras_context.ras = ras_con;
4599 return 0;
4600 }
4601
4602 /* check if ras is supported on block, say, sdma, gfx */
amdgpu_ras_is_supported(struct amdgpu_device * adev,unsigned int block)4603 int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4604 unsigned int block)
4605 {
4606 int ret = 0;
4607 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4608
4609 if (block >= AMDGPU_RAS_BLOCK_COUNT)
4610 return 0;
4611
4612 ret = ras && (adev->ras_enabled & (1 << block));
4613
4614 /* For the special asic with mem ecc enabled but sram ecc
4615 * not enabled, even if the ras block is not supported on
4616 * .ras_enabled, if the asic supports poison mode and the
4617 * ras block has ras configuration, it can be considered
4618 * that the ras block supports ras function.
4619 */
4620 if (!ret &&
4621 (block == AMDGPU_RAS_BLOCK__GFX ||
4622 block == AMDGPU_RAS_BLOCK__SDMA ||
4623 block == AMDGPU_RAS_BLOCK__VCN ||
4624 block == AMDGPU_RAS_BLOCK__JPEG) &&
4625 (amdgpu_ras_mask & (1 << block)) &&
4626 amdgpu_ras_is_poison_mode_supported(adev) &&
4627 amdgpu_ras_get_ras_block(adev, block, 0))
4628 ret = 1;
4629
4630 return ret;
4631 }
4632
amdgpu_ras_reset_gpu(struct amdgpu_device * adev)4633 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4634 {
4635 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4636
4637 /* mode1 is the only selection for RMA status */
4638 if (amdgpu_ras_is_rma(adev)) {
4639 ras->gpu_reset_flags = 0;
4640 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4641 }
4642
4643 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4644 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4645 int hive_ras_recovery = 0;
4646
4647 if (hive) {
4648 hive_ras_recovery = atomic_read(&hive->ras_recovery);
4649 amdgpu_put_xgmi_hive(hive);
4650 }
4651 /* In the case of multiple GPUs, after a GPU has started
4652 * resetting all GPUs on hive, other GPUs do not need to
4653 * trigger GPU reset again.
4654 */
4655 if (!hive_ras_recovery)
4656 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4657 else
4658 atomic_set(&ras->in_recovery, 0);
4659 } else {
4660 flush_work(&ras->recovery_work);
4661 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4662 }
4663
4664 return 0;
4665 }
4666
amdgpu_ras_set_mca_debug_mode(struct amdgpu_device * adev,bool enable)4667 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
4668 {
4669 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4670 int ret = 0;
4671
4672 if (con) {
4673 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4674 if (!ret)
4675 con->is_aca_debug_mode = enable;
4676 }
4677
4678 return ret;
4679 }
4680
amdgpu_ras_set_aca_debug_mode(struct amdgpu_device * adev,bool enable)4681 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4682 {
4683 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4684 int ret = 0;
4685
4686 if (con) {
4687 if (amdgpu_aca_is_enabled(adev))
4688 ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4689 else
4690 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4691 if (!ret)
4692 con->is_aca_debug_mode = enable;
4693 }
4694
4695 return ret;
4696 }
4697
amdgpu_ras_get_aca_debug_mode(struct amdgpu_device * adev)4698 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
4699 {
4700 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4701 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4702 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4703
4704 if (!con)
4705 return false;
4706
4707 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4708 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4709 return con->is_aca_debug_mode;
4710 else
4711 return true;
4712 }
4713
amdgpu_ras_get_error_query_mode(struct amdgpu_device * adev,unsigned int * error_query_mode)4714 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4715 unsigned int *error_query_mode)
4716 {
4717 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4718 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4719 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4720
4721 if (!con) {
4722 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4723 return false;
4724 }
4725
4726 if (amdgpu_sriov_vf(adev)) {
4727 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4728 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
4729 *error_query_mode =
4730 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
4731 } else {
4732 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
4733 }
4734
4735 return true;
4736 }
4737
4738 /* Register each ip ras block into amdgpu ras */
amdgpu_ras_register_ras_block(struct amdgpu_device * adev,struct amdgpu_ras_block_object * ras_block_obj)4739 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
4740 struct amdgpu_ras_block_object *ras_block_obj)
4741 {
4742 struct amdgpu_ras_block_list *ras_node;
4743 if (!adev || !ras_block_obj)
4744 return -EINVAL;
4745
4746 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4747 if (!ras_node)
4748 return -ENOMEM;
4749
4750 INIT_LIST_HEAD(&ras_node->node);
4751 ras_node->ras_obj = ras_block_obj;
4752 list_add_tail(&ras_node->node, &adev->ras_list);
4753
4754 return 0;
4755 }
4756
amdgpu_ras_get_error_type_name(uint32_t err_type,char * err_type_name)4757 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4758 {
4759 if (!err_type_name)
4760 return;
4761
4762 switch (err_type) {
4763 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4764 sprintf(err_type_name, "correctable");
4765 break;
4766 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4767 sprintf(err_type_name, "uncorrectable");
4768 break;
4769 default:
4770 sprintf(err_type_name, "unknown");
4771 break;
4772 }
4773 }
4774
amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_entry,uint32_t instance,uint32_t * memory_id)4775 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4776 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4777 uint32_t instance,
4778 uint32_t *memory_id)
4779 {
4780 uint32_t err_status_lo_data, err_status_lo_offset;
4781
4782 if (!reg_entry)
4783 return false;
4784
4785 err_status_lo_offset =
4786 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4787 reg_entry->seg_lo, reg_entry->reg_lo);
4788 err_status_lo_data = RREG32(err_status_lo_offset);
4789
4790 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4791 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4792 return false;
4793
4794 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4795
4796 return true;
4797 }
4798
amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_entry,uint32_t instance,unsigned long * err_cnt)4799 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4800 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4801 uint32_t instance,
4802 unsigned long *err_cnt)
4803 {
4804 uint32_t err_status_hi_data, err_status_hi_offset;
4805
4806 if (!reg_entry)
4807 return false;
4808
4809 err_status_hi_offset =
4810 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4811 reg_entry->seg_hi, reg_entry->reg_hi);
4812 err_status_hi_data = RREG32(err_status_hi_offset);
4813
4814 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4815 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
4816 /* keep the check here in case we need to refer to the result later */
4817 dev_dbg(adev->dev, "Invalid err_info field\n");
4818
4819 /* read err count */
4820 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4821
4822 return true;
4823 }
4824
amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_list,uint32_t reg_list_size,const struct amdgpu_ras_memory_id_entry * mem_list,uint32_t mem_list_size,uint32_t instance,uint32_t err_type,unsigned long * err_count)4825 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4826 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4827 uint32_t reg_list_size,
4828 const struct amdgpu_ras_memory_id_entry *mem_list,
4829 uint32_t mem_list_size,
4830 uint32_t instance,
4831 uint32_t err_type,
4832 unsigned long *err_count)
4833 {
4834 uint32_t memory_id;
4835 unsigned long err_cnt;
4836 char err_type_name[16];
4837 uint32_t i, j;
4838
4839 for (i = 0; i < reg_list_size; i++) {
4840 /* query memory_id from err_status_lo */
4841 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i],
4842 instance, &memory_id))
4843 continue;
4844
4845 /* query err_cnt from err_status_hi */
4846 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i],
4847 instance, &err_cnt) ||
4848 !err_cnt)
4849 continue;
4850
4851 *err_count += err_cnt;
4852
4853 /* log the errors */
4854 amdgpu_ras_get_error_type_name(err_type, err_type_name);
4855 if (!mem_list) {
4856 /* memory_list is not supported */
4857 dev_info(adev->dev,
4858 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4859 err_cnt, err_type_name,
4860 reg_list[i].block_name,
4861 instance, memory_id);
4862 } else {
4863 for (j = 0; j < mem_list_size; j++) {
4864 if (memory_id == mem_list[j].memory_id) {
4865 dev_info(adev->dev,
4866 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4867 err_cnt, err_type_name,
4868 reg_list[i].block_name,
4869 instance, mem_list[j].name);
4870 break;
4871 }
4872 }
4873 }
4874 }
4875 }
4876
amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device * adev,const struct amdgpu_ras_err_status_reg_entry * reg_list,uint32_t reg_list_size,uint32_t instance)4877 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4878 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4879 uint32_t reg_list_size,
4880 uint32_t instance)
4881 {
4882 uint32_t err_status_lo_offset, err_status_hi_offset;
4883 uint32_t i;
4884
4885 for (i = 0; i < reg_list_size; i++) {
4886 err_status_lo_offset =
4887 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4888 reg_list[i].seg_lo, reg_list[i].reg_lo);
4889 err_status_hi_offset =
4890 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4891 reg_list[i].seg_hi, reg_list[i].reg_hi);
4892 WREG32(err_status_lo_offset, 0);
4893 WREG32(err_status_hi_offset, 0);
4894 }
4895 }
4896
amdgpu_ras_error_data_init(struct ras_err_data * err_data)4897 int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
4898 {
4899 memset(err_data, 0, sizeof(*err_data));
4900
4901 INIT_LIST_HEAD(&err_data->err_node_list);
4902
4903 return 0;
4904 }
4905
amdgpu_ras_error_node_release(struct ras_err_node * err_node)4906 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
4907 {
4908 if (!err_node)
4909 return;
4910
4911 list_del(&err_node->node);
4912 kvfree(err_node);
4913 }
4914
amdgpu_ras_error_data_fini(struct ras_err_data * err_data)4915 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
4916 {
4917 struct ras_err_node *err_node, *tmp;
4918
4919 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
4920 amdgpu_ras_error_node_release(err_node);
4921 }
4922
amdgpu_ras_error_find_node_by_id(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info)4923 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
4924 struct amdgpu_smuio_mcm_config_info *mcm_info)
4925 {
4926 struct ras_err_node *err_node;
4927 struct amdgpu_smuio_mcm_config_info *ref_id;
4928
4929 if (!err_data || !mcm_info)
4930 return NULL;
4931
4932 for_each_ras_error(err_node, err_data) {
4933 ref_id = &err_node->err_info.mcm_info;
4934
4935 if (mcm_info->socket_id == ref_id->socket_id &&
4936 mcm_info->die_id == ref_id->die_id)
4937 return err_node;
4938 }
4939
4940 return NULL;
4941 }
4942
amdgpu_ras_error_node_new(void)4943 static struct ras_err_node *amdgpu_ras_error_node_new(void)
4944 {
4945 struct ras_err_node *err_node;
4946
4947 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
4948 if (!err_node)
4949 return NULL;
4950
4951 INIT_LIST_HEAD(&err_node->node);
4952
4953 return err_node;
4954 }
4955
ras_err_info_cmp(void * priv,const struct list_head * a,const struct list_head * b)4956 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
4957 {
4958 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
4959 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
4960 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
4961 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
4962
4963 if (unlikely(infoa->socket_id != infob->socket_id))
4964 return infoa->socket_id - infob->socket_id;
4965 else
4966 return infoa->die_id - infob->die_id;
4967
4968 return 0;
4969 }
4970
amdgpu_ras_error_get_info(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info)4971 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
4972 struct amdgpu_smuio_mcm_config_info *mcm_info)
4973 {
4974 struct ras_err_node *err_node;
4975
4976 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
4977 if (err_node)
4978 return &err_node->err_info;
4979
4980 err_node = amdgpu_ras_error_node_new();
4981 if (!err_node)
4982 return NULL;
4983
4984 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
4985
4986 err_data->err_list_count++;
4987 list_add_tail(&err_node->node, &err_data->err_node_list);
4988 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
4989
4990 return &err_node->err_info;
4991 }
4992
amdgpu_ras_error_statistic_ue_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)4993 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
4994 struct amdgpu_smuio_mcm_config_info *mcm_info,
4995 u64 count)
4996 {
4997 struct ras_err_info *err_info;
4998
4999 if (!err_data || !mcm_info)
5000 return -EINVAL;
5001
5002 if (!count)
5003 return 0;
5004
5005 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5006 if (!err_info)
5007 return -EINVAL;
5008
5009 err_info->ue_count += count;
5010 err_data->ue_count += count;
5011
5012 return 0;
5013 }
5014
amdgpu_ras_error_statistic_ce_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)5015 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5016 struct amdgpu_smuio_mcm_config_info *mcm_info,
5017 u64 count)
5018 {
5019 struct ras_err_info *err_info;
5020
5021 if (!err_data || !mcm_info)
5022 return -EINVAL;
5023
5024 if (!count)
5025 return 0;
5026
5027 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5028 if (!err_info)
5029 return -EINVAL;
5030
5031 err_info->ce_count += count;
5032 err_data->ce_count += count;
5033
5034 return 0;
5035 }
5036
amdgpu_ras_error_statistic_de_count(struct ras_err_data * err_data,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 count)5037 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5038 struct amdgpu_smuio_mcm_config_info *mcm_info,
5039 u64 count)
5040 {
5041 struct ras_err_info *err_info;
5042
5043 if (!err_data || !mcm_info)
5044 return -EINVAL;
5045
5046 if (!count)
5047 return 0;
5048
5049 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5050 if (!err_info)
5051 return -EINVAL;
5052
5053 err_info->de_count += count;
5054 err_data->de_count += count;
5055
5056 return 0;
5057 }
5058
5059 #define mmMP0_SMN_C2PMSG_92 0x1609C
5060 #define mmMP0_SMN_C2PMSG_126 0x160BE
amdgpu_ras_boot_time_error_reporting(struct amdgpu_device * adev,u32 instance)5061 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5062 u32 instance)
5063 {
5064 u32 socket_id, aid_id, hbm_id;
5065 u32 fw_status;
5066 u32 boot_error;
5067 u64 reg_addr;
5068
5069 /* The pattern for smn addressing in other SOC could be different from
5070 * the one for aqua_vanjaram. We should revisit the code if the pattern
5071 * is changed. In such case, replace the aqua_vanjaram implementation
5072 * with more common helper */
5073 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5074 aqua_vanjaram_encode_ext_smn_addressing(instance);
5075 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5076
5077 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5078 aqua_vanjaram_encode_ext_smn_addressing(instance);
5079 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5080
5081 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5082 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5083 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5084
5085 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5086 dev_info(adev->dev,
5087 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5088 socket_id, aid_id, hbm_id, fw_status);
5089
5090 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5091 dev_info(adev->dev,
5092 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5093 socket_id, aid_id, fw_status);
5094
5095 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5096 dev_info(adev->dev,
5097 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5098 socket_id, aid_id, fw_status);
5099
5100 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5101 dev_info(adev->dev,
5102 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5103 socket_id, aid_id, fw_status);
5104
5105 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5106 dev_info(adev->dev,
5107 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5108 socket_id, aid_id, fw_status);
5109
5110 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5111 dev_info(adev->dev,
5112 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5113 socket_id, aid_id, fw_status);
5114
5115 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5116 dev_info(adev->dev,
5117 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5118 socket_id, aid_id, hbm_id, fw_status);
5119
5120 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5121 dev_info(adev->dev,
5122 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5123 socket_id, aid_id, hbm_id, fw_status);
5124
5125 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5126 dev_info(adev->dev,
5127 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5128 socket_id, aid_id, fw_status);
5129
5130 if (AMDGPU_RAS_GPU_ERR_UNKNOWN(boot_error))
5131 dev_info(adev->dev,
5132 "socket: %d, aid: %d, fw_status: 0x%x, unknown boot time errors\n",
5133 socket_id, aid_id, fw_status);
5134 }
5135
amdgpu_ras_boot_error_detected(struct amdgpu_device * adev,u32 instance)5136 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5137 u32 instance)
5138 {
5139 u64 reg_addr;
5140 u32 reg_data;
5141 int retry_loop;
5142
5143 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5144 aqua_vanjaram_encode_ext_smn_addressing(instance);
5145
5146 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5147 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5148 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5149 return false;
5150 else
5151 msleep(1);
5152 }
5153
5154 return true;
5155 }
5156
amdgpu_ras_query_boot_status(struct amdgpu_device * adev,u32 num_instances)5157 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5158 {
5159 u32 i;
5160
5161 for (i = 0; i < num_instances; i++) {
5162 if (amdgpu_ras_boot_error_detected(adev, i))
5163 amdgpu_ras_boot_time_error_reporting(adev, i);
5164 }
5165 }
5166
amdgpu_ras_reserve_page(struct amdgpu_device * adev,uint64_t pfn)5167 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5168 {
5169 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5170 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5171 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5172 int ret = 0;
5173
5174 mutex_lock(&con->page_rsv_lock);
5175 ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5176 if (ret == -ENOENT)
5177 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5178 mutex_unlock(&con->page_rsv_lock);
5179
5180 return ret;
5181 }
5182
amdgpu_ras_event_log_print(struct amdgpu_device * adev,u64 event_id,const char * fmt,...)5183 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5184 const char *fmt, ...)
5185 {
5186 struct va_format vaf;
5187 va_list args;
5188
5189 va_start(args, fmt);
5190 vaf.fmt = fmt;
5191 vaf.va = &args;
5192
5193 if (RAS_EVENT_ID_IS_VALID(event_id))
5194 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5195 else
5196 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5197
5198 va_end(args);
5199 }
5200
amdgpu_ras_is_rma(struct amdgpu_device * adev)5201 bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5202 {
5203 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5204
5205 if (!con)
5206 return false;
5207
5208 return con->is_rma;
5209 }
5210