1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 45 #ifdef CONFIG_X86_MCE_AMD 46 #include <asm/mce.h> 47 48 static bool notifier_registered; 49 #endif 50 static const char *RAS_FS_NAME = "ras"; 51 52 const char *ras_error_string[] = { 53 "none", 54 "parity", 55 "single_correctable", 56 "multi_uncorrectable", 57 "poison", 58 }; 59 60 const char *ras_block_string[] = { 61 "umc", 62 "sdma", 63 "gfx", 64 "mmhub", 65 "athub", 66 "pcie_bif", 67 "hdp", 68 "xgmi_wafl", 69 "df", 70 "smn", 71 "sem", 72 "mp0", 73 "mp1", 74 "fuse", 75 "mca", 76 "vcn", 77 "jpeg", 78 "ih", 79 "mpio", 80 "mmsch", 81 }; 82 83 const char *ras_mca_block_string[] = { 84 "mca_mp0", 85 "mca_mp1", 86 "mca_mpio", 87 "mca_iohc", 88 }; 89 90 struct amdgpu_ras_block_list { 91 /* ras block link */ 92 struct list_head node; 93 94 struct amdgpu_ras_block_object *ras_obj; 95 }; 96 97 const char *get_ras_block_str(struct ras_common_if *ras_block) 98 { 99 if (!ras_block) 100 return "NULL"; 101 102 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 103 ras_block->block >= ARRAY_SIZE(ras_block_string)) 104 return "OUT OF RANGE"; 105 106 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 107 return ras_mca_block_string[ras_block->sub_block_index]; 108 109 return ras_block_string[ras_block->block]; 110 } 111 112 #define ras_block_str(_BLOCK_) \ 113 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 114 115 #define ras_err_str(i) (ras_error_string[ffs(i)]) 116 117 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 118 119 /* inject address is 52 bits */ 120 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 121 122 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 123 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 124 125 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms 126 127 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 128 129 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 130 131 enum amdgpu_ras_retire_page_reservation { 132 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 133 AMDGPU_RAS_RETIRE_PAGE_PENDING, 134 AMDGPU_RAS_RETIRE_PAGE_FAULT, 135 }; 136 137 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 138 139 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 140 uint64_t addr); 141 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 142 uint64_t addr); 143 #ifdef CONFIG_X86_MCE_AMD 144 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 145 struct mce_notifier_adev_list { 146 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 147 int num_gpu; 148 }; 149 static struct mce_notifier_adev_list mce_adev_list; 150 #endif 151 152 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 153 { 154 if (adev && amdgpu_ras_get_context(adev)) 155 amdgpu_ras_get_context(adev)->error_query_ready = ready; 156 } 157 158 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 159 { 160 if (adev && amdgpu_ras_get_context(adev)) 161 return amdgpu_ras_get_context(adev)->error_query_ready; 162 163 return false; 164 } 165 166 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 167 { 168 struct ras_err_data err_data; 169 struct eeprom_table_record err_rec; 170 int ret; 171 172 if ((address >= adev->gmc.mc_vram_size) || 173 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 174 dev_warn(adev->dev, 175 "RAS WARN: input address 0x%llx is invalid.\n", 176 address); 177 return -EINVAL; 178 } 179 180 if (amdgpu_ras_check_bad_page(adev, address)) { 181 dev_warn(adev->dev, 182 "RAS WARN: 0x%llx has already been marked as bad page!\n", 183 address); 184 return 0; 185 } 186 187 ret = amdgpu_ras_error_data_init(&err_data); 188 if (ret) 189 return ret; 190 191 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 192 err_data.err_addr = &err_rec; 193 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 194 195 if (amdgpu_bad_page_threshold != 0) { 196 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 197 err_data.err_addr_cnt, false); 198 amdgpu_ras_save_bad_pages(adev, NULL); 199 } 200 201 amdgpu_ras_error_data_fini(&err_data); 202 203 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 204 dev_warn(adev->dev, "Clear EEPROM:\n"); 205 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 206 207 return 0; 208 } 209 210 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 211 size_t size, loff_t *pos) 212 { 213 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 214 struct ras_query_if info = { 215 .head = obj->head, 216 }; 217 ssize_t s; 218 char val[128]; 219 220 if (amdgpu_ras_query_error_status(obj->adev, &info)) 221 return -EINVAL; 222 223 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 224 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 225 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 226 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 227 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 228 } 229 230 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 231 "ue", info.ue_count, 232 "ce", info.ce_count); 233 if (*pos >= s) 234 return 0; 235 236 s -= *pos; 237 s = min_t(u64, s, size); 238 239 240 if (copy_to_user(buf, &val[*pos], s)) 241 return -EINVAL; 242 243 *pos += s; 244 245 return s; 246 } 247 248 static const struct file_operations amdgpu_ras_debugfs_ops = { 249 .owner = THIS_MODULE, 250 .read = amdgpu_ras_debugfs_read, 251 .write = NULL, 252 .llseek = default_llseek 253 }; 254 255 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 256 { 257 int i; 258 259 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 260 *block_id = i; 261 if (strcmp(name, ras_block_string[i]) == 0) 262 return 0; 263 } 264 return -EINVAL; 265 } 266 267 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 268 const char __user *buf, size_t size, 269 loff_t *pos, struct ras_debug_if *data) 270 { 271 ssize_t s = min_t(u64, 64, size); 272 char str[65]; 273 char block_name[33]; 274 char err[9] = "ue"; 275 int op = -1; 276 int block_id; 277 uint32_t sub_block; 278 u64 address, value; 279 /* default value is 0 if the mask is not set by user */ 280 u32 instance_mask = 0; 281 282 if (*pos) 283 return -EINVAL; 284 *pos = size; 285 286 memset(str, 0, sizeof(str)); 287 memset(data, 0, sizeof(*data)); 288 289 if (copy_from_user(str, buf, s)) 290 return -EINVAL; 291 292 if (sscanf(str, "disable %32s", block_name) == 1) 293 op = 0; 294 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 295 op = 1; 296 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 297 op = 2; 298 else if (strstr(str, "retire_page") != NULL) 299 op = 3; 300 else if (str[0] && str[1] && str[2] && str[3]) 301 /* ascii string, but commands are not matched. */ 302 return -EINVAL; 303 304 if (op != -1) { 305 if (op == 3) { 306 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 307 sscanf(str, "%*s %llu", &address) != 1) 308 return -EINVAL; 309 310 data->op = op; 311 data->inject.address = address; 312 313 return 0; 314 } 315 316 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 317 return -EINVAL; 318 319 data->head.block = block_id; 320 /* only ue, ce and poison errors are supported */ 321 if (!memcmp("ue", err, 2)) 322 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 323 else if (!memcmp("ce", err, 2)) 324 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 325 else if (!memcmp("poison", err, 6)) 326 data->head.type = AMDGPU_RAS_ERROR__POISON; 327 else 328 return -EINVAL; 329 330 data->op = op; 331 332 if (op == 2) { 333 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 334 &sub_block, &address, &value, &instance_mask) != 4 && 335 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 336 &sub_block, &address, &value, &instance_mask) != 4 && 337 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 338 &sub_block, &address, &value) != 3 && 339 sscanf(str, "%*s %*s %*s %u %llu %llu", 340 &sub_block, &address, &value) != 3) 341 return -EINVAL; 342 data->head.sub_block_index = sub_block; 343 data->inject.address = address; 344 data->inject.value = value; 345 data->inject.instance_mask = instance_mask; 346 } 347 } else { 348 if (size < sizeof(*data)) 349 return -EINVAL; 350 351 if (copy_from_user(data, buf, sizeof(*data))) 352 return -EINVAL; 353 } 354 355 return 0; 356 } 357 358 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 359 struct ras_debug_if *data) 360 { 361 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 362 uint32_t mask, inst_mask = data->inject.instance_mask; 363 364 /* no need to set instance mask if there is only one instance */ 365 if (num_xcc <= 1 && inst_mask) { 366 data->inject.instance_mask = 0; 367 dev_dbg(adev->dev, 368 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 369 inst_mask); 370 371 return; 372 } 373 374 switch (data->head.block) { 375 case AMDGPU_RAS_BLOCK__GFX: 376 mask = GENMASK(num_xcc - 1, 0); 377 break; 378 case AMDGPU_RAS_BLOCK__SDMA: 379 mask = GENMASK(adev->sdma.num_instances - 1, 0); 380 break; 381 case AMDGPU_RAS_BLOCK__VCN: 382 case AMDGPU_RAS_BLOCK__JPEG: 383 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 384 break; 385 default: 386 mask = inst_mask; 387 break; 388 } 389 390 /* remove invalid bits in instance mask */ 391 data->inject.instance_mask &= mask; 392 if (inst_mask != data->inject.instance_mask) 393 dev_dbg(adev->dev, 394 "Adjust RAS inject mask 0x%x to 0x%x\n", 395 inst_mask, data->inject.instance_mask); 396 } 397 398 /** 399 * DOC: AMDGPU RAS debugfs control interface 400 * 401 * The control interface accepts struct ras_debug_if which has two members. 402 * 403 * First member: ras_debug_if::head or ras_debug_if::inject. 404 * 405 * head is used to indicate which IP block will be under control. 406 * 407 * head has four members, they are block, type, sub_block_index, name. 408 * block: which IP will be under control. 409 * type: what kind of error will be enabled/disabled/injected. 410 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 411 * name: the name of IP. 412 * 413 * inject has three more members than head, they are address, value and mask. 414 * As their names indicate, inject operation will write the 415 * value to the address. 416 * 417 * The second member: struct ras_debug_if::op. 418 * It has three kinds of operations. 419 * 420 * - 0: disable RAS on the block. Take ::head as its data. 421 * - 1: enable RAS on the block. Take ::head as its data. 422 * - 2: inject errors on the block. Take ::inject as its data. 423 * 424 * How to use the interface? 425 * 426 * In a program 427 * 428 * Copy the struct ras_debug_if in your code and initialize it. 429 * Write the struct to the control interface. 430 * 431 * From shell 432 * 433 * .. code-block:: bash 434 * 435 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 436 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 437 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 438 * 439 * Where N, is the card which you want to affect. 440 * 441 * "disable" requires only the block. 442 * "enable" requires the block and error type. 443 * "inject" requires the block, error type, address, and value. 444 * 445 * The block is one of: umc, sdma, gfx, etc. 446 * see ras_block_string[] for details 447 * 448 * The error type is one of: ue, ce and poison where, 449 * ue is multi-uncorrectable 450 * ce is single-correctable 451 * poison is poison 452 * 453 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 454 * The address and value are hexadecimal numbers, leading 0x is optional. 455 * The mask means instance mask, is optional, default value is 0x1. 456 * 457 * For instance, 458 * 459 * .. code-block:: bash 460 * 461 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 462 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 463 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 464 * 465 * How to check the result of the operation? 466 * 467 * To check disable/enable, see "ras" features at, 468 * /sys/class/drm/card[0/1/2...]/device/ras/features 469 * 470 * To check inject, see the corresponding error count at, 471 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 472 * 473 * .. note:: 474 * Operations are only allowed on blocks which are supported. 475 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 476 * to see which blocks support RAS on a particular asic. 477 * 478 */ 479 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 480 const char __user *buf, 481 size_t size, loff_t *pos) 482 { 483 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 484 struct ras_debug_if data; 485 int ret = 0; 486 487 if (!amdgpu_ras_get_error_query_ready(adev)) { 488 dev_warn(adev->dev, "RAS WARN: error injection " 489 "currently inaccessible\n"); 490 return size; 491 } 492 493 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 494 if (ret) 495 return ret; 496 497 if (data.op == 3) { 498 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 499 if (!ret) 500 return size; 501 else 502 return ret; 503 } 504 505 if (!amdgpu_ras_is_supported(adev, data.head.block)) 506 return -EINVAL; 507 508 switch (data.op) { 509 case 0: 510 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 511 break; 512 case 1: 513 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 514 break; 515 case 2: 516 if ((data.inject.address >= adev->gmc.mc_vram_size && 517 adev->gmc.mc_vram_size) || 518 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 519 dev_warn(adev->dev, "RAS WARN: input address " 520 "0x%llx is invalid.", 521 data.inject.address); 522 ret = -EINVAL; 523 break; 524 } 525 526 /* umc ce/ue error injection for a bad page is not allowed */ 527 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) && 528 amdgpu_ras_check_bad_page(adev, data.inject.address)) { 529 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has " 530 "already been marked as bad!\n", 531 data.inject.address); 532 break; 533 } 534 535 amdgpu_ras_instance_mask_check(adev, &data); 536 537 /* data.inject.address is offset instead of absolute gpu address */ 538 ret = amdgpu_ras_error_inject(adev, &data.inject); 539 break; 540 default: 541 ret = -EINVAL; 542 break; 543 } 544 545 if (ret) 546 return ret; 547 548 return size; 549 } 550 551 /** 552 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 553 * 554 * Some boards contain an EEPROM which is used to persistently store a list of 555 * bad pages which experiences ECC errors in vram. This interface provides 556 * a way to reset the EEPROM, e.g., after testing error injection. 557 * 558 * Usage: 559 * 560 * .. code-block:: bash 561 * 562 * echo 1 > ../ras/ras_eeprom_reset 563 * 564 * will reset EEPROM table to 0 entries. 565 * 566 */ 567 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 568 const char __user *buf, 569 size_t size, loff_t *pos) 570 { 571 struct amdgpu_device *adev = 572 (struct amdgpu_device *)file_inode(f)->i_private; 573 int ret; 574 575 ret = amdgpu_ras_eeprom_reset_table( 576 &(amdgpu_ras_get_context(adev)->eeprom_control)); 577 578 if (!ret) { 579 /* Something was written to EEPROM. 580 */ 581 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 582 return size; 583 } else { 584 return ret; 585 } 586 } 587 588 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 589 .owner = THIS_MODULE, 590 .read = NULL, 591 .write = amdgpu_ras_debugfs_ctrl_write, 592 .llseek = default_llseek 593 }; 594 595 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 596 .owner = THIS_MODULE, 597 .read = NULL, 598 .write = amdgpu_ras_debugfs_eeprom_write, 599 .llseek = default_llseek 600 }; 601 602 /** 603 * DOC: AMDGPU RAS sysfs Error Count Interface 604 * 605 * It allows the user to read the error count for each IP block on the gpu through 606 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 607 * 608 * It outputs the multiple lines which report the uncorrected (ue) and corrected 609 * (ce) error counts. 610 * 611 * The format of one line is below, 612 * 613 * [ce|ue]: count 614 * 615 * Example: 616 * 617 * .. code-block:: bash 618 * 619 * ue: 0 620 * ce: 1 621 * 622 */ 623 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 624 struct device_attribute *attr, char *buf) 625 { 626 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 627 struct ras_query_if info = { 628 .head = obj->head, 629 }; 630 631 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 632 return sysfs_emit(buf, "Query currently inaccessible\n"); 633 634 if (amdgpu_ras_query_error_status(obj->adev, &info)) 635 return -EINVAL; 636 637 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 638 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 639 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 640 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 641 } 642 643 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 644 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 645 "ce", info.ce_count, "de", info.de_count); 646 else 647 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 648 "ce", info.ce_count); 649 } 650 651 /* obj begin */ 652 653 #define get_obj(obj) do { (obj)->use++; } while (0) 654 #define alive_obj(obj) ((obj)->use) 655 656 static inline void put_obj(struct ras_manager *obj) 657 { 658 if (obj && (--obj->use == 0)) { 659 list_del(&obj->node); 660 amdgpu_ras_error_data_fini(&obj->err_data); 661 } 662 663 if (obj && (obj->use < 0)) 664 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 665 } 666 667 /* make one obj and return it. */ 668 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 669 struct ras_common_if *head) 670 { 671 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 672 struct ras_manager *obj; 673 674 if (!adev->ras_enabled || !con) 675 return NULL; 676 677 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 678 return NULL; 679 680 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 681 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 682 return NULL; 683 684 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 685 } else 686 obj = &con->objs[head->block]; 687 688 /* already exist. return obj? */ 689 if (alive_obj(obj)) 690 return NULL; 691 692 if (amdgpu_ras_error_data_init(&obj->err_data)) 693 return NULL; 694 695 obj->head = *head; 696 obj->adev = adev; 697 list_add(&obj->node, &con->head); 698 get_obj(obj); 699 700 return obj; 701 } 702 703 /* return an obj equal to head, or the first when head is NULL */ 704 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 705 struct ras_common_if *head) 706 { 707 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 708 struct ras_manager *obj; 709 int i; 710 711 if (!adev->ras_enabled || !con) 712 return NULL; 713 714 if (head) { 715 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 716 return NULL; 717 718 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 719 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 720 return NULL; 721 722 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 723 } else 724 obj = &con->objs[head->block]; 725 726 if (alive_obj(obj)) 727 return obj; 728 } else { 729 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 730 obj = &con->objs[i]; 731 if (alive_obj(obj)) 732 return obj; 733 } 734 } 735 736 return NULL; 737 } 738 /* obj end */ 739 740 /* feature ctl begin */ 741 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 742 struct ras_common_if *head) 743 { 744 return adev->ras_hw_enabled & BIT(head->block); 745 } 746 747 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 748 struct ras_common_if *head) 749 { 750 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 751 752 return con->features & BIT(head->block); 753 } 754 755 /* 756 * if obj is not created, then create one. 757 * set feature enable flag. 758 */ 759 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 760 struct ras_common_if *head, int enable) 761 { 762 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 763 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 764 765 /* If hardware does not support ras, then do not create obj. 766 * But if hardware support ras, we can create the obj. 767 * Ras framework checks con->hw_supported to see if it need do 768 * corresponding initialization. 769 * IP checks con->support to see if it need disable ras. 770 */ 771 if (!amdgpu_ras_is_feature_allowed(adev, head)) 772 return 0; 773 774 if (enable) { 775 if (!obj) { 776 obj = amdgpu_ras_create_obj(adev, head); 777 if (!obj) 778 return -EINVAL; 779 } else { 780 /* In case we create obj somewhere else */ 781 get_obj(obj); 782 } 783 con->features |= BIT(head->block); 784 } else { 785 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 786 con->features &= ~BIT(head->block); 787 put_obj(obj); 788 } 789 } 790 791 return 0; 792 } 793 794 /* wrapper of psp_ras_enable_features */ 795 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 796 struct ras_common_if *head, bool enable) 797 { 798 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 799 union ta_ras_cmd_input *info; 800 int ret; 801 802 if (!con) 803 return -EINVAL; 804 805 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 806 /* For gfx ip, regardless of feature support status, */ 807 /* Force issue enable or disable ras feature commands */ 808 if (head->block != AMDGPU_RAS_BLOCK__GFX && 809 !amdgpu_ras_is_feature_allowed(adev, head)) 810 return 0; 811 812 /* Only enable gfx ras feature from host side */ 813 if (head->block == AMDGPU_RAS_BLOCK__GFX && 814 !amdgpu_sriov_vf(adev) && 815 !amdgpu_ras_intr_triggered()) { 816 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 817 if (!info) 818 return -ENOMEM; 819 820 if (!enable) { 821 info->disable_features = (struct ta_ras_disable_features_input) { 822 .block_id = amdgpu_ras_block_to_ta(head->block), 823 .error_type = amdgpu_ras_error_to_ta(head->type), 824 }; 825 } else { 826 info->enable_features = (struct ta_ras_enable_features_input) { 827 .block_id = amdgpu_ras_block_to_ta(head->block), 828 .error_type = amdgpu_ras_error_to_ta(head->type), 829 }; 830 } 831 832 ret = psp_ras_enable_features(&adev->psp, info, enable); 833 if (ret) { 834 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 835 enable ? "enable":"disable", 836 get_ras_block_str(head), 837 amdgpu_ras_is_poison_mode_supported(adev), ret); 838 kfree(info); 839 return ret; 840 } 841 842 kfree(info); 843 } 844 845 /* setup the obj */ 846 __amdgpu_ras_feature_enable(adev, head, enable); 847 848 return 0; 849 } 850 851 /* Only used in device probe stage and called only once. */ 852 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 853 struct ras_common_if *head, bool enable) 854 { 855 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 856 int ret; 857 858 if (!con) 859 return -EINVAL; 860 861 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 862 if (enable) { 863 /* There is no harm to issue a ras TA cmd regardless of 864 * the currecnt ras state. 865 * If current state == target state, it will do nothing 866 * But sometimes it requests driver to reset and repost 867 * with error code -EAGAIN. 868 */ 869 ret = amdgpu_ras_feature_enable(adev, head, 1); 870 /* With old ras TA, we might fail to enable ras. 871 * Log it and just setup the object. 872 * TODO need remove this WA in the future. 873 */ 874 if (ret == -EINVAL) { 875 ret = __amdgpu_ras_feature_enable(adev, head, 1); 876 if (!ret) 877 dev_info(adev->dev, 878 "RAS INFO: %s setup object\n", 879 get_ras_block_str(head)); 880 } 881 } else { 882 /* setup the object then issue a ras TA disable cmd.*/ 883 ret = __amdgpu_ras_feature_enable(adev, head, 1); 884 if (ret) 885 return ret; 886 887 /* gfx block ras disable cmd must send to ras-ta */ 888 if (head->block == AMDGPU_RAS_BLOCK__GFX) 889 con->features |= BIT(head->block); 890 891 ret = amdgpu_ras_feature_enable(adev, head, 0); 892 893 /* clean gfx block ras features flag */ 894 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 895 con->features &= ~BIT(head->block); 896 } 897 } else 898 ret = amdgpu_ras_feature_enable(adev, head, enable); 899 900 return ret; 901 } 902 903 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 904 bool bypass) 905 { 906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 907 struct ras_manager *obj, *tmp; 908 909 list_for_each_entry_safe(obj, tmp, &con->head, node) { 910 /* bypass psp. 911 * aka just release the obj and corresponding flags 912 */ 913 if (bypass) { 914 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 915 break; 916 } else { 917 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 918 break; 919 } 920 } 921 922 return con->features; 923 } 924 925 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 926 bool bypass) 927 { 928 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 929 int i; 930 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 931 932 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 933 struct ras_common_if head = { 934 .block = i, 935 .type = default_ras_type, 936 .sub_block_index = 0, 937 }; 938 939 if (i == AMDGPU_RAS_BLOCK__MCA) 940 continue; 941 942 if (bypass) { 943 /* 944 * bypass psp. vbios enable ras for us. 945 * so just create the obj 946 */ 947 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 948 break; 949 } else { 950 if (amdgpu_ras_feature_enable(adev, &head, 1)) 951 break; 952 } 953 } 954 955 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 956 struct ras_common_if head = { 957 .block = AMDGPU_RAS_BLOCK__MCA, 958 .type = default_ras_type, 959 .sub_block_index = i, 960 }; 961 962 if (bypass) { 963 /* 964 * bypass psp. vbios enable ras for us. 965 * so just create the obj 966 */ 967 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 968 break; 969 } else { 970 if (amdgpu_ras_feature_enable(adev, &head, 1)) 971 break; 972 } 973 } 974 975 return con->features; 976 } 977 /* feature ctl end */ 978 979 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 980 enum amdgpu_ras_block block) 981 { 982 if (!block_obj) 983 return -EINVAL; 984 985 if (block_obj->ras_comm.block == block) 986 return 0; 987 988 return -EINVAL; 989 } 990 991 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 992 enum amdgpu_ras_block block, uint32_t sub_block_index) 993 { 994 struct amdgpu_ras_block_list *node, *tmp; 995 struct amdgpu_ras_block_object *obj; 996 997 if (block >= AMDGPU_RAS_BLOCK__LAST) 998 return NULL; 999 1000 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1001 if (!node->ras_obj) { 1002 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1003 continue; 1004 } 1005 1006 obj = node->ras_obj; 1007 if (obj->ras_block_match) { 1008 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1009 return obj; 1010 } else { 1011 if (amdgpu_ras_block_match_default(obj, block) == 0) 1012 return obj; 1013 } 1014 } 1015 1016 return NULL; 1017 } 1018 1019 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1020 { 1021 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1022 int ret = 0; 1023 1024 /* 1025 * choosing right query method according to 1026 * whether smu support query error information 1027 */ 1028 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1029 if (ret == -EOPNOTSUPP) { 1030 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1032 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1033 1034 /* umc query_ras_error_address is also responsible for clearing 1035 * error status 1036 */ 1037 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1039 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1040 } else if (!ret) { 1041 if (adev->umc.ras && 1042 adev->umc.ras->ecc_info_query_ras_error_count) 1043 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1044 1045 if (adev->umc.ras && 1046 adev->umc.ras->ecc_info_query_ras_error_address) 1047 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1048 } 1049 } 1050 1051 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1052 struct ras_manager *ras_mgr, 1053 struct ras_err_data *err_data, 1054 struct ras_query_context *qctx, 1055 const char *blk_name, 1056 bool is_ue, 1057 bool is_de) 1058 { 1059 struct amdgpu_smuio_mcm_config_info *mcm_info; 1060 struct ras_err_node *err_node; 1061 struct ras_err_info *err_info; 1062 u64 event_id = qctx->evid.event_id; 1063 1064 if (is_ue) { 1065 for_each_ras_error(err_node, err_data) { 1066 err_info = &err_node->err_info; 1067 mcm_info = &err_info->mcm_info; 1068 if (err_info->ue_count) { 1069 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1070 "%lld new uncorrectable hardware errors detected in %s block\n", 1071 mcm_info->socket_id, 1072 mcm_info->die_id, 1073 err_info->ue_count, 1074 blk_name); 1075 } 1076 } 1077 1078 for_each_ras_error(err_node, &ras_mgr->err_data) { 1079 err_info = &err_node->err_info; 1080 mcm_info = &err_info->mcm_info; 1081 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1082 "%lld uncorrectable hardware errors detected in total in %s block\n", 1083 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1084 } 1085 1086 } else { 1087 if (is_de) { 1088 for_each_ras_error(err_node, err_data) { 1089 err_info = &err_node->err_info; 1090 mcm_info = &err_info->mcm_info; 1091 if (err_info->de_count) { 1092 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1093 "%lld new deferred hardware errors detected in %s block\n", 1094 mcm_info->socket_id, 1095 mcm_info->die_id, 1096 err_info->de_count, 1097 blk_name); 1098 } 1099 } 1100 1101 for_each_ras_error(err_node, &ras_mgr->err_data) { 1102 err_info = &err_node->err_info; 1103 mcm_info = &err_info->mcm_info; 1104 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1105 "%lld deferred hardware errors detected in total in %s block\n", 1106 mcm_info->socket_id, mcm_info->die_id, 1107 err_info->de_count, blk_name); 1108 } 1109 } else { 1110 if (adev->debug_disable_ce_logs) 1111 return; 1112 1113 for_each_ras_error(err_node, err_data) { 1114 err_info = &err_node->err_info; 1115 mcm_info = &err_info->mcm_info; 1116 if (err_info->ce_count) { 1117 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1118 "%lld new correctable hardware errors detected in %s block\n", 1119 mcm_info->socket_id, 1120 mcm_info->die_id, 1121 err_info->ce_count, 1122 blk_name); 1123 } 1124 } 1125 1126 for_each_ras_error(err_node, &ras_mgr->err_data) { 1127 err_info = &err_node->err_info; 1128 mcm_info = &err_info->mcm_info; 1129 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1130 "%lld correctable hardware errors detected in total in %s block\n", 1131 mcm_info->socket_id, mcm_info->die_id, 1132 err_info->ce_count, blk_name); 1133 } 1134 } 1135 } 1136 } 1137 1138 static inline bool err_data_has_source_info(struct ras_err_data *data) 1139 { 1140 return !list_empty(&data->err_node_list); 1141 } 1142 1143 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1144 struct ras_query_if *query_if, 1145 struct ras_err_data *err_data, 1146 struct ras_query_context *qctx) 1147 { 1148 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1149 const char *blk_name = get_ras_block_str(&query_if->head); 1150 u64 event_id = qctx->evid.event_id; 1151 1152 if (err_data->ce_count) { 1153 if (err_data_has_source_info(err_data)) { 1154 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1155 blk_name, false, false); 1156 } else if (!adev->aid_mask && 1157 adev->smuio.funcs && 1158 adev->smuio.funcs->get_socket_id && 1159 adev->smuio.funcs->get_die_id) { 1160 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1161 "%ld correctable hardware errors " 1162 "detected in %s block\n", 1163 adev->smuio.funcs->get_socket_id(adev), 1164 adev->smuio.funcs->get_die_id(adev), 1165 ras_mgr->err_data.ce_count, 1166 blk_name); 1167 } else { 1168 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1169 "detected in %s block\n", 1170 ras_mgr->err_data.ce_count, 1171 blk_name); 1172 } 1173 } 1174 1175 if (err_data->ue_count) { 1176 if (err_data_has_source_info(err_data)) { 1177 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1178 blk_name, true, false); 1179 } else if (!adev->aid_mask && 1180 adev->smuio.funcs && 1181 adev->smuio.funcs->get_socket_id && 1182 adev->smuio.funcs->get_die_id) { 1183 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1184 "%ld uncorrectable hardware errors " 1185 "detected in %s block\n", 1186 adev->smuio.funcs->get_socket_id(adev), 1187 adev->smuio.funcs->get_die_id(adev), 1188 ras_mgr->err_data.ue_count, 1189 blk_name); 1190 } else { 1191 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1192 "detected in %s block\n", 1193 ras_mgr->err_data.ue_count, 1194 blk_name); 1195 } 1196 } 1197 1198 if (err_data->de_count) { 1199 if (err_data_has_source_info(err_data)) { 1200 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1201 blk_name, false, true); 1202 } else if (!adev->aid_mask && 1203 adev->smuio.funcs && 1204 adev->smuio.funcs->get_socket_id && 1205 adev->smuio.funcs->get_die_id) { 1206 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1207 "%ld deferred hardware errors " 1208 "detected in %s block\n", 1209 adev->smuio.funcs->get_socket_id(adev), 1210 adev->smuio.funcs->get_die_id(adev), 1211 ras_mgr->err_data.de_count, 1212 blk_name); 1213 } else { 1214 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1215 "detected in %s block\n", 1216 ras_mgr->err_data.de_count, 1217 blk_name); 1218 } 1219 } 1220 } 1221 1222 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1223 struct ras_query_if *query_if, 1224 struct ras_err_data *err_data, 1225 struct ras_query_context *qctx) 1226 { 1227 unsigned long new_ue, new_ce, new_de; 1228 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1229 const char *blk_name = get_ras_block_str(&query_if->head); 1230 u64 event_id = qctx->evid.event_id; 1231 1232 new_ce = err_data->ce_count - obj->err_data.ce_count; 1233 new_ue = err_data->ue_count - obj->err_data.ue_count; 1234 new_de = err_data->de_count - obj->err_data.de_count; 1235 1236 if (new_ce) { 1237 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1238 "detected in %s block\n", 1239 new_ce, 1240 blk_name); 1241 } 1242 1243 if (new_ue) { 1244 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1245 "detected in %s block\n", 1246 new_ue, 1247 blk_name); 1248 } 1249 1250 if (new_de) { 1251 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1252 "detected in %s block\n", 1253 new_de, 1254 blk_name); 1255 } 1256 } 1257 1258 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1259 { 1260 struct ras_err_node *err_node; 1261 struct ras_err_info *err_info; 1262 1263 if (err_data_has_source_info(err_data)) { 1264 for_each_ras_error(err_node, err_data) { 1265 err_info = &err_node->err_info; 1266 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1267 &err_info->mcm_info, err_info->de_count); 1268 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1269 &err_info->mcm_info, err_info->ce_count); 1270 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1271 &err_info->mcm_info, err_info->ue_count); 1272 } 1273 } else { 1274 /* for legacy asic path which doesn't has error source info */ 1275 obj->err_data.ue_count += err_data->ue_count; 1276 obj->err_data.ce_count += err_data->ce_count; 1277 obj->err_data.de_count += err_data->de_count; 1278 } 1279 } 1280 1281 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1282 struct ras_err_data *err_data) 1283 { 1284 /* Host reports absolute counts */ 1285 obj->err_data.ue_count = err_data->ue_count; 1286 obj->err_data.ce_count = err_data->ce_count; 1287 obj->err_data.de_count = err_data->de_count; 1288 } 1289 1290 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1291 { 1292 struct ras_common_if head; 1293 1294 memset(&head, 0, sizeof(head)); 1295 head.block = blk; 1296 1297 return amdgpu_ras_find_obj(adev, &head); 1298 } 1299 1300 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1301 const struct aca_info *aca_info, void *data) 1302 { 1303 struct ras_manager *obj; 1304 1305 /* in resume phase, no need to create aca fs node */ 1306 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1307 return 0; 1308 1309 obj = get_ras_manager(adev, blk); 1310 if (!obj) 1311 return -EINVAL; 1312 1313 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1314 } 1315 1316 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1317 { 1318 struct ras_manager *obj; 1319 1320 obj = get_ras_manager(adev, blk); 1321 if (!obj) 1322 return -EINVAL; 1323 1324 amdgpu_aca_remove_handle(&obj->aca_handle); 1325 1326 return 0; 1327 } 1328 1329 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1330 enum aca_error_type type, struct ras_err_data *err_data, 1331 struct ras_query_context *qctx) 1332 { 1333 struct ras_manager *obj; 1334 1335 obj = get_ras_manager(adev, blk); 1336 if (!obj) 1337 return -EINVAL; 1338 1339 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1340 } 1341 1342 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1343 struct aca_handle *handle, char *buf, void *data) 1344 { 1345 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1346 struct ras_query_if info = { 1347 .head = obj->head, 1348 }; 1349 1350 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1351 return sysfs_emit(buf, "Query currently inaccessible\n"); 1352 1353 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1354 return -EINVAL; 1355 1356 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1357 "ce", info.ce_count, "de", info.de_count); 1358 } 1359 1360 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1361 struct ras_query_if *info, 1362 struct ras_err_data *err_data, 1363 struct ras_query_context *qctx, 1364 unsigned int error_query_mode) 1365 { 1366 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1367 struct amdgpu_ras_block_object *block_obj = NULL; 1368 int ret; 1369 1370 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1371 return -EINVAL; 1372 1373 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1374 return -EINVAL; 1375 1376 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1377 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1378 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1379 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1380 amdgpu_ras_get_ecc_info(adev, err_data); 1381 } else { 1382 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1383 if (!block_obj || !block_obj->hw_ops) { 1384 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1385 get_ras_block_str(&info->head)); 1386 return -EINVAL; 1387 } 1388 1389 if (block_obj->hw_ops->query_ras_error_count) 1390 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1391 1392 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1393 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1394 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1395 if (block_obj->hw_ops->query_ras_error_status) 1396 block_obj->hw_ops->query_ras_error_status(adev); 1397 } 1398 } 1399 } else { 1400 if (amdgpu_aca_is_enabled(adev)) { 1401 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1402 if (ret) 1403 return ret; 1404 1405 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1406 if (ret) 1407 return ret; 1408 1409 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1410 if (ret) 1411 return ret; 1412 } else { 1413 /* FIXME: add code to check return value later */ 1414 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1415 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1416 } 1417 } 1418 1419 return 0; 1420 } 1421 1422 /* query/inject/cure begin */ 1423 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1424 struct ras_query_if *info, 1425 enum ras_event_type type) 1426 { 1427 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1428 struct ras_err_data err_data; 1429 struct ras_query_context qctx; 1430 unsigned int error_query_mode; 1431 int ret; 1432 1433 if (!obj) 1434 return -EINVAL; 1435 1436 ret = amdgpu_ras_error_data_init(&err_data); 1437 if (ret) 1438 return ret; 1439 1440 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1441 return -EINVAL; 1442 1443 memset(&qctx, 0, sizeof(qctx)); 1444 qctx.evid.type = type; 1445 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1446 1447 if (!down_read_trylock(&adev->reset_domain->sem)) { 1448 ret = -EIO; 1449 goto out_fini_err_data; 1450 } 1451 1452 ret = amdgpu_ras_query_error_status_helper(adev, info, 1453 &err_data, 1454 &qctx, 1455 error_query_mode); 1456 up_read(&adev->reset_domain->sem); 1457 if (ret) 1458 goto out_fini_err_data; 1459 1460 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1461 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1462 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1463 } else { 1464 /* Host provides absolute error counts. First generate the report 1465 * using the previous VF internal count against new host count. 1466 * Then Update VF internal count. 1467 */ 1468 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1469 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1470 } 1471 1472 info->ue_count = obj->err_data.ue_count; 1473 info->ce_count = obj->err_data.ce_count; 1474 info->de_count = obj->err_data.de_count; 1475 1476 out_fini_err_data: 1477 amdgpu_ras_error_data_fini(&err_data); 1478 1479 return ret; 1480 } 1481 1482 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1483 { 1484 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1485 } 1486 1487 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1488 enum amdgpu_ras_block block) 1489 { 1490 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1491 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1492 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1493 1494 if (!block_obj || !block_obj->hw_ops) { 1495 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1496 ras_block_str(block)); 1497 return -EOPNOTSUPP; 1498 } 1499 1500 if (!amdgpu_ras_is_supported(adev, block) || 1501 !amdgpu_ras_get_aca_debug_mode(adev)) 1502 return -EOPNOTSUPP; 1503 1504 if (amdgpu_sriov_vf(adev)) 1505 return -EOPNOTSUPP; 1506 1507 /* skip ras error reset in gpu reset */ 1508 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1509 ((smu_funcs && smu_funcs->set_debug_mode) || 1510 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1511 return -EOPNOTSUPP; 1512 1513 if (block_obj->hw_ops->reset_ras_error_count) 1514 block_obj->hw_ops->reset_ras_error_count(adev); 1515 1516 return 0; 1517 } 1518 1519 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1520 enum amdgpu_ras_block block) 1521 { 1522 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1523 1524 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1525 return 0; 1526 1527 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1528 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1529 if (block_obj->hw_ops->reset_ras_error_status) 1530 block_obj->hw_ops->reset_ras_error_status(adev); 1531 } 1532 1533 return 0; 1534 } 1535 1536 /* wrapper of psp_ras_trigger_error */ 1537 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1538 struct ras_inject_if *info) 1539 { 1540 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1541 struct ta_ras_trigger_error_input block_info = { 1542 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1543 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1544 .sub_block_index = info->head.sub_block_index, 1545 .address = info->address, 1546 .value = info->value, 1547 }; 1548 int ret = -EINVAL; 1549 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1550 info->head.block, 1551 info->head.sub_block_index); 1552 1553 /* inject on guest isn't allowed, return success directly */ 1554 if (amdgpu_sriov_vf(adev)) 1555 return 0; 1556 1557 if (!obj) 1558 return -EINVAL; 1559 1560 if (!block_obj || !block_obj->hw_ops) { 1561 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1562 get_ras_block_str(&info->head)); 1563 return -EINVAL; 1564 } 1565 1566 /* Calculate XGMI relative offset */ 1567 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1568 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1569 block_info.address = 1570 amdgpu_xgmi_get_relative_phy_addr(adev, 1571 block_info.address); 1572 } 1573 1574 if (block_obj->hw_ops->ras_error_inject) { 1575 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1576 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1577 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1578 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1579 info->instance_mask); 1580 } else { 1581 /* default path */ 1582 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1583 } 1584 1585 if (ret) 1586 dev_err(adev->dev, "ras inject %s failed %d\n", 1587 get_ras_block_str(&info->head), ret); 1588 1589 return ret; 1590 } 1591 1592 /** 1593 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1594 * @adev: pointer to AMD GPU device 1595 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1596 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1597 * @query_info: pointer to ras_query_if 1598 * 1599 * Return 0 for query success or do nothing, otherwise return an error 1600 * on failures 1601 */ 1602 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1603 unsigned long *ce_count, 1604 unsigned long *ue_count, 1605 struct ras_query_if *query_info) 1606 { 1607 int ret; 1608 1609 if (!query_info) 1610 /* do nothing if query_info is not specified */ 1611 return 0; 1612 1613 ret = amdgpu_ras_query_error_status(adev, query_info); 1614 if (ret) 1615 return ret; 1616 1617 *ce_count += query_info->ce_count; 1618 *ue_count += query_info->ue_count; 1619 1620 /* some hardware/IP supports read to clear 1621 * no need to explictly reset the err status after the query call */ 1622 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1623 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1624 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1625 dev_warn(adev->dev, 1626 "Failed to reset error counter and error status\n"); 1627 } 1628 1629 return 0; 1630 } 1631 1632 /** 1633 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1634 * @adev: pointer to AMD GPU device 1635 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1636 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1637 * errors. 1638 * @query_info: pointer to ras_query_if if the query request is only for 1639 * specific ip block; if info is NULL, then the qurey request is for 1640 * all the ip blocks that support query ras error counters/status 1641 * 1642 * If set, @ce_count or @ue_count, count and return the corresponding 1643 * error counts in those integer pointers. Return 0 if the device 1644 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1645 */ 1646 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1647 unsigned long *ce_count, 1648 unsigned long *ue_count, 1649 struct ras_query_if *query_info) 1650 { 1651 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1652 struct ras_manager *obj; 1653 unsigned long ce, ue; 1654 int ret; 1655 1656 if (!adev->ras_enabled || !con) 1657 return -EOPNOTSUPP; 1658 1659 /* Don't count since no reporting. 1660 */ 1661 if (!ce_count && !ue_count) 1662 return 0; 1663 1664 ce = 0; 1665 ue = 0; 1666 if (!query_info) { 1667 /* query all the ip blocks that support ras query interface */ 1668 list_for_each_entry(obj, &con->head, node) { 1669 struct ras_query_if info = { 1670 .head = obj->head, 1671 }; 1672 1673 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1674 } 1675 } else { 1676 /* query specific ip block */ 1677 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1678 } 1679 1680 if (ret) 1681 return ret; 1682 1683 if (ce_count) 1684 *ce_count = ce; 1685 1686 if (ue_count) 1687 *ue_count = ue; 1688 1689 return 0; 1690 } 1691 /* query/inject/cure end */ 1692 1693 1694 /* sysfs begin */ 1695 1696 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1697 struct ras_badpage **bps, unsigned int *count); 1698 1699 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1700 { 1701 switch (flags) { 1702 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1703 return "R"; 1704 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1705 return "P"; 1706 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1707 default: 1708 return "F"; 1709 } 1710 } 1711 1712 /** 1713 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1714 * 1715 * It allows user to read the bad pages of vram on the gpu through 1716 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1717 * 1718 * It outputs multiple lines, and each line stands for one gpu page. 1719 * 1720 * The format of one line is below, 1721 * gpu pfn : gpu page size : flags 1722 * 1723 * gpu pfn and gpu page size are printed in hex format. 1724 * flags can be one of below character, 1725 * 1726 * R: reserved, this gpu page is reserved and not able to use. 1727 * 1728 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1729 * in next window of page_reserve. 1730 * 1731 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1732 * 1733 * Examples: 1734 * 1735 * .. code-block:: bash 1736 * 1737 * 0x00000001 : 0x00001000 : R 1738 * 0x00000002 : 0x00001000 : P 1739 * 1740 */ 1741 1742 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1743 struct kobject *kobj, const struct bin_attribute *attr, 1744 char *buf, loff_t ppos, size_t count) 1745 { 1746 struct amdgpu_ras *con = 1747 container_of(attr, struct amdgpu_ras, badpages_attr); 1748 struct amdgpu_device *adev = con->adev; 1749 const unsigned int element_size = 1750 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1751 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1752 unsigned int end = div64_ul(ppos + count - 1, element_size); 1753 ssize_t s = 0; 1754 struct ras_badpage *bps = NULL; 1755 unsigned int bps_count = 0; 1756 1757 memset(buf, 0, count); 1758 1759 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1760 return 0; 1761 1762 for (; start < end && start < bps_count; start++) 1763 s += scnprintf(&buf[s], element_size + 1, 1764 "0x%08x : 0x%08x : %1s\n", 1765 bps[start].bp, 1766 bps[start].size, 1767 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1768 1769 kfree(bps); 1770 1771 return s; 1772 } 1773 1774 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1775 struct device_attribute *attr, char *buf) 1776 { 1777 struct amdgpu_ras *con = 1778 container_of(attr, struct amdgpu_ras, features_attr); 1779 1780 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1781 } 1782 1783 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1784 struct device_attribute *attr, char *buf) 1785 { 1786 struct amdgpu_ras *con = 1787 container_of(attr, struct amdgpu_ras, version_attr); 1788 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version); 1789 } 1790 1791 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1792 struct device_attribute *attr, char *buf) 1793 { 1794 struct amdgpu_ras *con = 1795 container_of(attr, struct amdgpu_ras, schema_attr); 1796 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1797 } 1798 1799 static struct { 1800 enum ras_event_type type; 1801 const char *name; 1802 } dump_event[] = { 1803 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 1804 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 1805 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 1806 }; 1807 1808 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 1809 struct device_attribute *attr, char *buf) 1810 { 1811 struct amdgpu_ras *con = 1812 container_of(attr, struct amdgpu_ras, event_state_attr); 1813 struct ras_event_manager *event_mgr = con->event_mgr; 1814 struct ras_event_state *event_state; 1815 int i, size = 0; 1816 1817 if (!event_mgr) 1818 return -EINVAL; 1819 1820 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 1821 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 1822 event_state = &event_mgr->event_state[dump_event[i].type]; 1823 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 1824 dump_event[i].name, 1825 atomic64_read(&event_state->count), 1826 event_state->last_seqno); 1827 } 1828 1829 return (ssize_t)size; 1830 } 1831 1832 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1833 { 1834 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1835 1836 if (adev->dev->kobj.sd) 1837 sysfs_remove_file_from_group(&adev->dev->kobj, 1838 &con->badpages_attr.attr, 1839 RAS_FS_NAME); 1840 } 1841 1842 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 1843 { 1844 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1845 struct attribute *attrs[] = { 1846 &con->features_attr.attr, 1847 &con->version_attr.attr, 1848 &con->schema_attr.attr, 1849 &con->event_state_attr.attr, 1850 NULL 1851 }; 1852 struct attribute_group group = { 1853 .name = RAS_FS_NAME, 1854 .attrs = attrs, 1855 }; 1856 1857 if (adev->dev->kobj.sd) 1858 sysfs_remove_group(&adev->dev->kobj, &group); 1859 1860 return 0; 1861 } 1862 1863 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1864 struct ras_common_if *head) 1865 { 1866 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1867 1868 if (amdgpu_aca_is_enabled(adev)) 1869 return 0; 1870 1871 if (!obj || obj->attr_inuse) 1872 return -EINVAL; 1873 1874 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 1875 return 0; 1876 1877 get_obj(obj); 1878 1879 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1880 "%s_err_count", head->name); 1881 1882 obj->sysfs_attr = (struct device_attribute){ 1883 .attr = { 1884 .name = obj->fs_data.sysfs_name, 1885 .mode = S_IRUGO, 1886 }, 1887 .show = amdgpu_ras_sysfs_read, 1888 }; 1889 sysfs_attr_init(&obj->sysfs_attr.attr); 1890 1891 if (sysfs_add_file_to_group(&adev->dev->kobj, 1892 &obj->sysfs_attr.attr, 1893 RAS_FS_NAME)) { 1894 put_obj(obj); 1895 return -EINVAL; 1896 } 1897 1898 obj->attr_inuse = 1; 1899 1900 return 0; 1901 } 1902 1903 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1904 struct ras_common_if *head) 1905 { 1906 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1907 1908 if (amdgpu_aca_is_enabled(adev)) 1909 return 0; 1910 1911 if (!obj || !obj->attr_inuse) 1912 return -EINVAL; 1913 1914 if (adev->dev->kobj.sd) 1915 sysfs_remove_file_from_group(&adev->dev->kobj, 1916 &obj->sysfs_attr.attr, 1917 RAS_FS_NAME); 1918 obj->attr_inuse = 0; 1919 put_obj(obj); 1920 1921 return 0; 1922 } 1923 1924 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1925 { 1926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1927 struct ras_manager *obj, *tmp; 1928 1929 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1930 amdgpu_ras_sysfs_remove(adev, &obj->head); 1931 } 1932 1933 if (amdgpu_bad_page_threshold != 0) 1934 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1935 1936 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 1937 1938 return 0; 1939 } 1940 /* sysfs end */ 1941 1942 /** 1943 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 1944 * 1945 * Normally when there is an uncorrectable error, the driver will reset 1946 * the GPU to recover. However, in the event of an unrecoverable error, 1947 * the driver provides an interface to reboot the system automatically 1948 * in that event. 1949 * 1950 * The following file in debugfs provides that interface: 1951 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 1952 * 1953 * Usage: 1954 * 1955 * .. code-block:: bash 1956 * 1957 * echo true > .../ras/auto_reboot 1958 * 1959 */ 1960 /* debugfs begin */ 1961 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 1962 { 1963 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1964 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 1965 struct drm_minor *minor = adev_to_drm(adev)->primary; 1966 struct dentry *dir; 1967 1968 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 1969 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 1970 &amdgpu_ras_debugfs_ctrl_ops); 1971 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 1972 &amdgpu_ras_debugfs_eeprom_ops); 1973 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 1974 &con->bad_page_cnt_threshold); 1975 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 1976 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 1977 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 1978 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 1979 &amdgpu_ras_debugfs_eeprom_size_ops); 1980 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 1981 S_IRUGO, dir, adev, 1982 &amdgpu_ras_debugfs_eeprom_table_ops); 1983 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 1984 1985 /* 1986 * After one uncorrectable error happens, usually GPU recovery will 1987 * be scheduled. But due to the known problem in GPU recovery failing 1988 * to bring GPU back, below interface provides one direct way to 1989 * user to reboot system automatically in such case within 1990 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 1991 * will never be called. 1992 */ 1993 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 1994 1995 /* 1996 * User could set this not to clean up hardware's error count register 1997 * of RAS IPs during ras recovery. 1998 */ 1999 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2000 &con->disable_ras_err_cnt_harvest); 2001 return dir; 2002 } 2003 2004 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2005 struct ras_fs_if *head, 2006 struct dentry *dir) 2007 { 2008 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2009 2010 if (!obj || !dir) 2011 return; 2012 2013 get_obj(obj); 2014 2015 memcpy(obj->fs_data.debugfs_name, 2016 head->debugfs_name, 2017 sizeof(obj->fs_data.debugfs_name)); 2018 2019 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2020 obj, &amdgpu_ras_debugfs_ops); 2021 } 2022 2023 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2024 { 2025 bool ret; 2026 2027 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2028 case IP_VERSION(13, 0, 6): 2029 case IP_VERSION(13, 0, 12): 2030 case IP_VERSION(13, 0, 14): 2031 ret = true; 2032 break; 2033 default: 2034 ret = false; 2035 break; 2036 } 2037 2038 return ret; 2039 } 2040 2041 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2042 { 2043 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2044 struct dentry *dir; 2045 struct ras_manager *obj; 2046 struct ras_fs_if fs_info; 2047 2048 /* 2049 * it won't be called in resume path, no need to check 2050 * suspend and gpu reset status 2051 */ 2052 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2053 return; 2054 2055 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2056 2057 list_for_each_entry(obj, &con->head, node) { 2058 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2059 (obj->attr_inuse == 1)) { 2060 sprintf(fs_info.debugfs_name, "%s_err_inject", 2061 get_ras_block_str(&obj->head)); 2062 fs_info.head = obj->head; 2063 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2064 } 2065 } 2066 2067 if (amdgpu_ras_aca_is_supported(adev)) { 2068 if (amdgpu_aca_is_enabled(adev)) 2069 amdgpu_aca_smu_debugfs_init(adev, dir); 2070 else 2071 amdgpu_mca_smu_debugfs_init(adev, dir); 2072 } 2073 } 2074 2075 /* debugfs end */ 2076 2077 /* ras fs */ 2078 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2079 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2080 static DEVICE_ATTR(features, S_IRUGO, 2081 amdgpu_ras_sysfs_features_read, NULL); 2082 static DEVICE_ATTR(version, 0444, 2083 amdgpu_ras_sysfs_version_show, NULL); 2084 static DEVICE_ATTR(schema, 0444, 2085 amdgpu_ras_sysfs_schema_show, NULL); 2086 static DEVICE_ATTR(event_state, 0444, 2087 amdgpu_ras_sysfs_event_state_show, NULL); 2088 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2089 { 2090 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2091 struct attribute_group group = { 2092 .name = RAS_FS_NAME, 2093 }; 2094 struct attribute *attrs[] = { 2095 &con->features_attr.attr, 2096 &con->version_attr.attr, 2097 &con->schema_attr.attr, 2098 &con->event_state_attr.attr, 2099 NULL 2100 }; 2101 const struct bin_attribute *bin_attrs[] = { 2102 NULL, 2103 NULL, 2104 }; 2105 int r; 2106 2107 group.attrs = attrs; 2108 2109 /* add features entry */ 2110 con->features_attr = dev_attr_features; 2111 sysfs_attr_init(attrs[0]); 2112 2113 /* add version entry */ 2114 con->version_attr = dev_attr_version; 2115 sysfs_attr_init(attrs[1]); 2116 2117 /* add schema entry */ 2118 con->schema_attr = dev_attr_schema; 2119 sysfs_attr_init(attrs[2]); 2120 2121 /* add event_state entry */ 2122 con->event_state_attr = dev_attr_event_state; 2123 sysfs_attr_init(attrs[3]); 2124 2125 if (amdgpu_bad_page_threshold != 0) { 2126 /* add bad_page_features entry */ 2127 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2128 sysfs_bin_attr_init(&con->badpages_attr); 2129 bin_attrs[0] = &con->badpages_attr; 2130 group.bin_attrs_new = bin_attrs; 2131 } 2132 2133 r = sysfs_create_group(&adev->dev->kobj, &group); 2134 if (r) 2135 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2136 2137 return 0; 2138 } 2139 2140 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2141 { 2142 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2143 struct ras_manager *con_obj, *ip_obj, *tmp; 2144 2145 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2146 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2147 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2148 if (ip_obj) 2149 put_obj(ip_obj); 2150 } 2151 } 2152 2153 amdgpu_ras_sysfs_remove_all(adev); 2154 return 0; 2155 } 2156 /* ras fs end */ 2157 2158 /* ih begin */ 2159 2160 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2161 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2162 * register to check whether the interrupt is triggered or not, and properly 2163 * ack the interrupt if it is there 2164 */ 2165 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2166 { 2167 /* Fatal error events are handled on host side */ 2168 if (amdgpu_sriov_vf(adev)) 2169 return; 2170 /* 2171 * If the current interrupt is caused by a non-fatal RAS error, skip 2172 * check for fatal error. For fatal errors, FED status of all devices 2173 * in XGMI hive gets set when the first device gets fatal error 2174 * interrupt. The error gets propagated to other devices as well, so 2175 * make sure to ack the interrupt regardless of FED status. 2176 */ 2177 if (!amdgpu_ras_get_fed_status(adev) && 2178 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2179 return; 2180 2181 if (adev->nbio.ras && 2182 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2183 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2184 2185 if (adev->nbio.ras && 2186 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2187 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2188 } 2189 2190 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2191 struct amdgpu_iv_entry *entry) 2192 { 2193 bool poison_stat = false; 2194 struct amdgpu_device *adev = obj->adev; 2195 struct amdgpu_ras_block_object *block_obj = 2196 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2197 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2198 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2199 u64 event_id; 2200 int ret; 2201 2202 if (!block_obj || !con) 2203 return; 2204 2205 ret = amdgpu_ras_mark_ras_event(adev, type); 2206 if (ret) 2207 return; 2208 2209 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2210 /* both query_poison_status and handle_poison_consumption are optional, 2211 * but at least one of them should be implemented if we need poison 2212 * consumption handler 2213 */ 2214 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2215 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2216 if (!poison_stat) { 2217 /* Not poison consumption interrupt, no need to handle it */ 2218 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2219 block_obj->ras_comm.name); 2220 2221 return; 2222 } 2223 } 2224 2225 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2226 2227 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2228 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2229 2230 /* gpu reset is fallback for failed and default cases. 2231 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2232 */ 2233 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2234 event_id = amdgpu_ras_acquire_event_id(adev, type); 2235 RAS_EVENT_LOG(adev, event_id, 2236 "GPU reset for %s RAS poison consumption is issued!\n", 2237 block_obj->ras_comm.name); 2238 amdgpu_ras_reset_gpu(adev); 2239 } 2240 2241 if (!poison_stat) 2242 amdgpu_gfx_poison_consumption_handler(adev, entry); 2243 } 2244 2245 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2246 struct amdgpu_iv_entry *entry) 2247 { 2248 struct amdgpu_device *adev = obj->adev; 2249 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2250 u64 event_id; 2251 int ret; 2252 2253 ret = amdgpu_ras_mark_ras_event(adev, type); 2254 if (ret) 2255 return; 2256 2257 event_id = amdgpu_ras_acquire_event_id(adev, type); 2258 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2259 2260 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2261 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2262 2263 atomic_inc(&con->page_retirement_req_cnt); 2264 atomic_inc(&con->poison_creation_count); 2265 2266 wake_up(&con->page_retirement_wq); 2267 } 2268 } 2269 2270 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2271 struct amdgpu_iv_entry *entry) 2272 { 2273 struct ras_ih_data *data = &obj->ih_data; 2274 struct ras_err_data err_data; 2275 int ret; 2276 2277 if (!data->cb) 2278 return; 2279 2280 ret = amdgpu_ras_error_data_init(&err_data); 2281 if (ret) 2282 return; 2283 2284 /* Let IP handle its data, maybe we need get the output 2285 * from the callback to update the error type/count, etc 2286 */ 2287 amdgpu_ras_set_fed(obj->adev, true); 2288 ret = data->cb(obj->adev, &err_data, entry); 2289 /* ue will trigger an interrupt, and in that case 2290 * we need do a reset to recovery the whole system. 2291 * But leave IP do that recovery, here we just dispatch 2292 * the error. 2293 */ 2294 if (ret == AMDGPU_RAS_SUCCESS) { 2295 /* these counts could be left as 0 if 2296 * some blocks do not count error number 2297 */ 2298 obj->err_data.ue_count += err_data.ue_count; 2299 obj->err_data.ce_count += err_data.ce_count; 2300 obj->err_data.de_count += err_data.de_count; 2301 } 2302 2303 amdgpu_ras_error_data_fini(&err_data); 2304 } 2305 2306 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2307 { 2308 struct ras_ih_data *data = &obj->ih_data; 2309 struct amdgpu_iv_entry entry; 2310 2311 while (data->rptr != data->wptr) { 2312 rmb(); 2313 memcpy(&entry, &data->ring[data->rptr], 2314 data->element_size); 2315 2316 wmb(); 2317 data->rptr = (data->aligned_element_size + 2318 data->rptr) % data->ring_size; 2319 2320 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2321 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2322 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2323 else 2324 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2325 } else { 2326 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2327 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2328 else 2329 dev_warn(obj->adev->dev, 2330 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2331 } 2332 } 2333 } 2334 2335 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2336 { 2337 struct ras_ih_data *data = 2338 container_of(work, struct ras_ih_data, ih_work); 2339 struct ras_manager *obj = 2340 container_of(data, struct ras_manager, ih_data); 2341 2342 amdgpu_ras_interrupt_handler(obj); 2343 } 2344 2345 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2346 struct ras_dispatch_if *info) 2347 { 2348 struct ras_manager *obj; 2349 struct ras_ih_data *data; 2350 2351 obj = amdgpu_ras_find_obj(adev, &info->head); 2352 if (!obj) 2353 return -EINVAL; 2354 2355 data = &obj->ih_data; 2356 2357 if (data->inuse == 0) 2358 return 0; 2359 2360 /* Might be overflow... */ 2361 memcpy(&data->ring[data->wptr], info->entry, 2362 data->element_size); 2363 2364 wmb(); 2365 data->wptr = (data->aligned_element_size + 2366 data->wptr) % data->ring_size; 2367 2368 schedule_work(&data->ih_work); 2369 2370 return 0; 2371 } 2372 2373 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2374 struct ras_common_if *head) 2375 { 2376 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2377 struct ras_ih_data *data; 2378 2379 if (!obj) 2380 return -EINVAL; 2381 2382 data = &obj->ih_data; 2383 if (data->inuse == 0) 2384 return 0; 2385 2386 cancel_work_sync(&data->ih_work); 2387 2388 kfree(data->ring); 2389 memset(data, 0, sizeof(*data)); 2390 put_obj(obj); 2391 2392 return 0; 2393 } 2394 2395 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2396 struct ras_common_if *head) 2397 { 2398 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2399 struct ras_ih_data *data; 2400 struct amdgpu_ras_block_object *ras_obj; 2401 2402 if (!obj) { 2403 /* in case we registe the IH before enable ras feature */ 2404 obj = amdgpu_ras_create_obj(adev, head); 2405 if (!obj) 2406 return -EINVAL; 2407 } else 2408 get_obj(obj); 2409 2410 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2411 2412 data = &obj->ih_data; 2413 /* add the callback.etc */ 2414 *data = (struct ras_ih_data) { 2415 .inuse = 0, 2416 .cb = ras_obj->ras_cb, 2417 .element_size = sizeof(struct amdgpu_iv_entry), 2418 .rptr = 0, 2419 .wptr = 0, 2420 }; 2421 2422 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2423 2424 data->aligned_element_size = ALIGN(data->element_size, 8); 2425 /* the ring can store 64 iv entries. */ 2426 data->ring_size = 64 * data->aligned_element_size; 2427 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2428 if (!data->ring) { 2429 put_obj(obj); 2430 return -ENOMEM; 2431 } 2432 2433 /* IH is ready */ 2434 data->inuse = 1; 2435 2436 return 0; 2437 } 2438 2439 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2440 { 2441 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2442 struct ras_manager *obj, *tmp; 2443 2444 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2445 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2446 } 2447 2448 return 0; 2449 } 2450 /* ih end */ 2451 2452 /* traversal all IPs except NBIO to query error counter */ 2453 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2454 { 2455 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2456 struct ras_manager *obj; 2457 2458 if (!adev->ras_enabled || !con) 2459 return; 2460 2461 list_for_each_entry(obj, &con->head, node) { 2462 struct ras_query_if info = { 2463 .head = obj->head, 2464 }; 2465 2466 /* 2467 * PCIE_BIF IP has one different isr by ras controller 2468 * interrupt, the specific ras counter query will be 2469 * done in that isr. So skip such block from common 2470 * sync flood interrupt isr calling. 2471 */ 2472 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2473 continue; 2474 2475 /* 2476 * this is a workaround for aldebaran, skip send msg to 2477 * smu to get ecc_info table due to smu handle get ecc 2478 * info table failed temporarily. 2479 * should be removed until smu fix handle ecc_info table. 2480 */ 2481 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2482 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2483 IP_VERSION(13, 0, 2))) 2484 continue; 2485 2486 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2487 2488 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2489 IP_VERSION(11, 0, 2) && 2490 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2491 IP_VERSION(11, 0, 4) && 2492 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2493 IP_VERSION(13, 0, 0)) { 2494 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2495 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2496 } 2497 } 2498 } 2499 2500 /* Parse RdRspStatus and WrRspStatus */ 2501 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2502 struct ras_query_if *info) 2503 { 2504 struct amdgpu_ras_block_object *block_obj; 2505 /* 2506 * Only two block need to query read/write 2507 * RspStatus at current state 2508 */ 2509 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2510 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2511 return; 2512 2513 block_obj = amdgpu_ras_get_ras_block(adev, 2514 info->head.block, 2515 info->head.sub_block_index); 2516 2517 if (!block_obj || !block_obj->hw_ops) { 2518 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2519 get_ras_block_str(&info->head)); 2520 return; 2521 } 2522 2523 if (block_obj->hw_ops->query_ras_error_status) 2524 block_obj->hw_ops->query_ras_error_status(adev); 2525 2526 } 2527 2528 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2529 { 2530 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2531 struct ras_manager *obj; 2532 2533 if (!adev->ras_enabled || !con) 2534 return; 2535 2536 list_for_each_entry(obj, &con->head, node) { 2537 struct ras_query_if info = { 2538 .head = obj->head, 2539 }; 2540 2541 amdgpu_ras_error_status_query(adev, &info); 2542 } 2543 } 2544 2545 /* recovery begin */ 2546 2547 /* return 0 on success. 2548 * caller need free bps. 2549 */ 2550 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2551 struct ras_badpage **bps, unsigned int *count) 2552 { 2553 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2554 struct ras_err_handler_data *data; 2555 int i = 0; 2556 int ret = 0, status; 2557 2558 if (!con || !con->eh_data || !bps || !count) 2559 return -EINVAL; 2560 2561 mutex_lock(&con->recovery_lock); 2562 data = con->eh_data; 2563 if (!data || data->count == 0) { 2564 *bps = NULL; 2565 ret = -EINVAL; 2566 goto out; 2567 } 2568 2569 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL); 2570 if (!*bps) { 2571 ret = -ENOMEM; 2572 goto out; 2573 } 2574 2575 for (; i < data->count; i++) { 2576 (*bps)[i] = (struct ras_badpage){ 2577 .bp = data->bps[i].retired_page, 2578 .size = AMDGPU_GPU_PAGE_SIZE, 2579 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 2580 }; 2581 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 2582 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT); 2583 if (status == -EBUSY) 2584 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 2585 else if (status == -ENOENT) 2586 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 2587 } 2588 2589 *count = data->count; 2590 out: 2591 mutex_unlock(&con->recovery_lock); 2592 return ret; 2593 } 2594 2595 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2596 struct amdgpu_hive_info *hive, bool status) 2597 { 2598 struct amdgpu_device *tmp_adev; 2599 2600 if (hive) { 2601 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2602 amdgpu_ras_set_fed(tmp_adev, status); 2603 } else { 2604 amdgpu_ras_set_fed(adev, status); 2605 } 2606 } 2607 2608 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2609 { 2610 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2611 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2612 int hive_ras_recovery = 0; 2613 2614 if (hive) { 2615 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2616 amdgpu_put_xgmi_hive(hive); 2617 } 2618 2619 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2620 return true; 2621 2622 return false; 2623 } 2624 2625 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2626 { 2627 if (amdgpu_ras_intr_triggered()) 2628 return RAS_EVENT_TYPE_FATAL; 2629 else 2630 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2631 } 2632 2633 static void amdgpu_ras_do_recovery(struct work_struct *work) 2634 { 2635 struct amdgpu_ras *ras = 2636 container_of(work, struct amdgpu_ras, recovery_work); 2637 struct amdgpu_device *remote_adev = NULL; 2638 struct amdgpu_device *adev = ras->adev; 2639 struct list_head device_list, *device_list_handle = NULL; 2640 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2641 enum ras_event_type type; 2642 2643 if (hive) { 2644 atomic_set(&hive->ras_recovery, 1); 2645 2646 /* If any device which is part of the hive received RAS fatal 2647 * error interrupt, set fatal error status on all. This 2648 * condition will need a recovery, and flag will be cleared 2649 * as part of recovery. 2650 */ 2651 list_for_each_entry(remote_adev, &hive->device_list, 2652 gmc.xgmi.head) 2653 if (amdgpu_ras_get_fed_status(remote_adev)) { 2654 amdgpu_ras_set_fed_all(adev, hive, true); 2655 break; 2656 } 2657 } 2658 if (!ras->disable_ras_err_cnt_harvest) { 2659 2660 /* Build list of devices to query RAS related errors */ 2661 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2662 device_list_handle = &hive->device_list; 2663 } else { 2664 INIT_LIST_HEAD(&device_list); 2665 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2666 device_list_handle = &device_list; 2667 } 2668 2669 type = amdgpu_ras_get_fatal_error_event(adev); 2670 list_for_each_entry(remote_adev, 2671 device_list_handle, gmc.xgmi.head) { 2672 amdgpu_ras_query_err_status(remote_adev); 2673 amdgpu_ras_log_on_err_counter(remote_adev, type); 2674 } 2675 2676 } 2677 2678 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2679 struct amdgpu_reset_context reset_context; 2680 memset(&reset_context, 0, sizeof(reset_context)); 2681 2682 reset_context.method = AMD_RESET_METHOD_NONE; 2683 reset_context.reset_req_dev = adev; 2684 reset_context.src = AMDGPU_RESET_SRC_RAS; 2685 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2686 2687 /* Perform full reset in fatal error mode */ 2688 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2689 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2690 else { 2691 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2692 2693 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2694 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2695 reset_context.method = AMD_RESET_METHOD_MODE2; 2696 } 2697 2698 /* Fatal error occurs in poison mode, mode1 reset is used to 2699 * recover gpu. 2700 */ 2701 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2702 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2703 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2704 2705 psp_fatal_error_recovery_quirk(&adev->psp); 2706 } 2707 } 2708 2709 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2710 } 2711 atomic_set(&ras->in_recovery, 0); 2712 if (hive) { 2713 atomic_set(&hive->ras_recovery, 0); 2714 amdgpu_put_xgmi_hive(hive); 2715 } 2716 } 2717 2718 /* alloc/realloc bps array */ 2719 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2720 struct ras_err_handler_data *data, int pages) 2721 { 2722 unsigned int old_space = data->count + data->space_left; 2723 unsigned int new_space = old_space + pages; 2724 unsigned int align_space = ALIGN(new_space, 512); 2725 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL); 2726 2727 if (!bps) { 2728 return -ENOMEM; 2729 } 2730 2731 if (data->bps) { 2732 memcpy(bps, data->bps, 2733 data->count * sizeof(*data->bps)); 2734 kfree(data->bps); 2735 } 2736 2737 data->bps = bps; 2738 data->space_left += align_space - old_space; 2739 return 0; 2740 } 2741 2742 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2743 struct eeprom_table_record *bps, 2744 struct ras_err_data *err_data) 2745 { 2746 struct ta_ras_query_address_input addr_in; 2747 uint32_t socket = 0; 2748 int ret = 0; 2749 2750 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2751 socket = adev->smuio.funcs->get_socket_id(adev); 2752 2753 /* reinit err_data */ 2754 err_data->err_addr_cnt = 0; 2755 err_data->err_addr_len = adev->umc.retire_unit; 2756 2757 memset(&addr_in, 0, sizeof(addr_in)); 2758 addr_in.ma.err_addr = bps->address; 2759 addr_in.ma.socket_id = socket; 2760 addr_in.ma.ch_inst = bps->mem_channel; 2761 /* tell RAS TA the node instance is not used */ 2762 addr_in.ma.node_inst = TA_RAS_INV_NODE; 2763 2764 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2765 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 2766 &addr_in, NULL, false); 2767 2768 return ret; 2769 } 2770 2771 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 2772 struct eeprom_table_record *bps, 2773 struct ras_err_data *err_data) 2774 { 2775 struct ta_ras_query_address_input addr_in; 2776 uint32_t die_id, socket = 0; 2777 2778 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2779 socket = adev->smuio.funcs->get_socket_id(adev); 2780 2781 /* although die id is gotten from PA in nps1 mode, the id is 2782 * fitable for any nps mode 2783 */ 2784 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 2785 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 2786 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 2787 else 2788 return -EINVAL; 2789 2790 /* reinit err_data */ 2791 err_data->err_addr_cnt = 0; 2792 err_data->err_addr_len = adev->umc.retire_unit; 2793 2794 memset(&addr_in, 0, sizeof(addr_in)); 2795 addr_in.ma.err_addr = bps->address; 2796 addr_in.ma.ch_inst = bps->mem_channel; 2797 addr_in.ma.umc_inst = bps->mcumc_id; 2798 addr_in.ma.node_inst = die_id; 2799 addr_in.ma.socket_id = socket; 2800 2801 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2802 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 2803 &addr_in, NULL, false); 2804 else 2805 return -EINVAL; 2806 } 2807 2808 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 2809 struct eeprom_table_record *bps, int count) 2810 { 2811 int j; 2812 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2813 struct ras_err_handler_data *data = con->eh_data; 2814 2815 for (j = 0; j < count; j++) { 2816 if (amdgpu_ras_check_bad_page_unlock(con, 2817 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2818 continue; 2819 2820 if (!data->space_left && 2821 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2822 return -ENOMEM; 2823 } 2824 2825 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 2826 2827 memcpy(&data->bps[data->count], &(bps[j]), 2828 sizeof(struct eeprom_table_record)); 2829 data->count++; 2830 data->space_left--; 2831 } 2832 2833 return 0; 2834 } 2835 2836 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 2837 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2838 enum amdgpu_memory_partition nps) 2839 { 2840 int i = 0; 2841 enum amdgpu_memory_partition save_nps; 2842 2843 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2844 2845 /*old asics just have pa in eeprom*/ 2846 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2847 memcpy(err_data->err_addr, bps, 2848 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2849 goto out; 2850 } 2851 2852 for (i = 0; i < adev->umc.retire_unit; i++) 2853 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2854 2855 if (save_nps) { 2856 if (save_nps == nps) { 2857 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2858 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2859 return -EINVAL; 2860 } else { 2861 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 2862 return -EINVAL; 2863 } 2864 } else { 2865 if (bps[0].address == 0) { 2866 /* for specific old eeprom data, mca address is not stored, 2867 * calc it from pa 2868 */ 2869 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 2870 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 2871 return -EINVAL; 2872 } 2873 2874 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 2875 if (nps == AMDGPU_NPS1_PARTITION_MODE) 2876 memcpy(err_data->err_addr, bps, 2877 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2878 else 2879 return -EOPNOTSUPP; 2880 } 2881 } 2882 2883 out: 2884 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 2885 } 2886 2887 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 2888 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2889 enum amdgpu_memory_partition nps) 2890 { 2891 enum amdgpu_memory_partition save_nps; 2892 2893 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2894 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2895 2896 if (save_nps == nps) { 2897 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2898 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2899 return -EINVAL; 2900 } else { 2901 if (bps->address) { 2902 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 2903 return -EINVAL; 2904 } else { 2905 /* for specific old eeprom data, mca address is not stored, 2906 * calc it from pa 2907 */ 2908 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 2909 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 2910 return -EINVAL; 2911 2912 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 2913 return -EOPNOTSUPP; 2914 } 2915 } 2916 2917 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 2918 adev->umc.retire_unit); 2919 } 2920 2921 /* it deal with vram only. */ 2922 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 2923 struct eeprom_table_record *bps, int pages, bool from_rom) 2924 { 2925 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2926 struct ras_err_data err_data; 2927 struct amdgpu_ras_eeprom_control *control = 2928 &adev->psp.ras_context.ras->eeprom_control; 2929 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 2930 int ret = 0; 2931 uint32_t i = 0; 2932 2933 if (!con || !con->eh_data || !bps || pages <= 0) 2934 return 0; 2935 2936 if (from_rom) { 2937 err_data.err_addr = 2938 kcalloc(adev->umc.retire_unit, 2939 sizeof(struct eeprom_table_record), GFP_KERNEL); 2940 if (!err_data.err_addr) { 2941 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 2942 return -ENOMEM; 2943 } 2944 2945 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 2946 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 2947 } 2948 2949 mutex_lock(&con->recovery_lock); 2950 2951 if (from_rom) { 2952 /* there is no pa recs in V3, so skip pa recs processing */ 2953 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 2954 for (i = 0; i < pages; i++) { 2955 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 2956 if ((bps[i].address == bps[i + 1].address) && 2957 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 2958 /* deal with retire_unit records a time */ 2959 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 2960 &bps[i], &err_data, nps); 2961 if (ret) 2962 control->ras_num_bad_pages -= adev->umc.retire_unit; 2963 i += (adev->umc.retire_unit - 1); 2964 } else { 2965 break; 2966 } 2967 } else { 2968 break; 2969 } 2970 } 2971 } 2972 for (; i < pages; i++) { 2973 ret = __amdgpu_ras_convert_rec_from_rom(adev, 2974 &bps[i], &err_data, nps); 2975 if (ret) 2976 control->ras_num_bad_pages -= adev->umc.retire_unit; 2977 } 2978 } else { 2979 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 2980 } 2981 2982 if (from_rom) 2983 kfree(err_data.err_addr); 2984 mutex_unlock(&con->recovery_lock); 2985 2986 return ret; 2987 } 2988 2989 /* 2990 * write error record array to eeprom, the function should be 2991 * protected by recovery_lock 2992 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 2993 */ 2994 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 2995 unsigned long *new_cnt) 2996 { 2997 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2998 struct ras_err_handler_data *data; 2999 struct amdgpu_ras_eeprom_control *control; 3000 int save_count, unit_num, bad_page_num, i; 3001 3002 if (!con || !con->eh_data) { 3003 if (new_cnt) 3004 *new_cnt = 0; 3005 3006 return 0; 3007 } 3008 3009 mutex_lock(&con->recovery_lock); 3010 control = &con->eeprom_control; 3011 data = con->eh_data; 3012 bad_page_num = control->ras_num_bad_pages; 3013 save_count = data->count - bad_page_num; 3014 mutex_unlock(&con->recovery_lock); 3015 3016 unit_num = save_count / adev->umc.retire_unit; 3017 if (new_cnt) 3018 *new_cnt = unit_num; 3019 3020 /* only new entries are saved */ 3021 if (save_count > 0) { 3022 /*old asics only save pa to eeprom like before*/ 3023 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3024 if (amdgpu_ras_eeprom_append(control, 3025 &data->bps[bad_page_num], save_count)) { 3026 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3027 return -EIO; 3028 } 3029 } else { 3030 for (i = 0; i < unit_num; i++) { 3031 if (amdgpu_ras_eeprom_append(control, 3032 &data->bps[bad_page_num + 3033 i * adev->umc.retire_unit], 1)) { 3034 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3035 return -EIO; 3036 } 3037 } 3038 } 3039 3040 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3041 } 3042 3043 return 0; 3044 } 3045 3046 /* 3047 * read error record array in eeprom and reserve enough space for 3048 * storing new bad pages 3049 */ 3050 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3051 { 3052 struct amdgpu_ras_eeprom_control *control = 3053 &adev->psp.ras_context.ras->eeprom_control; 3054 struct eeprom_table_record *bps; 3055 int ret, i = 0; 3056 3057 /* no bad page record, skip eeprom access */ 3058 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3059 return 0; 3060 3061 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3062 if (!bps) 3063 return -ENOMEM; 3064 3065 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3066 if (ret) { 3067 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3068 } else { 3069 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3070 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3071 as pa recs, so add verion check to avoid it. 3072 */ 3073 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3074 for (i = 0; i < control->ras_num_recs; i++) { 3075 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3076 if ((bps[i].address == bps[i + 1].address) && 3077 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3078 control->ras_num_pa_recs += adev->umc.retire_unit; 3079 i += (adev->umc.retire_unit - 1); 3080 } else { 3081 control->ras_num_mca_recs += 3082 (control->ras_num_recs - i); 3083 break; 3084 } 3085 } else { 3086 control->ras_num_mca_recs += (control->ras_num_recs - i); 3087 break; 3088 } 3089 } 3090 } else { 3091 control->ras_num_mca_recs = control->ras_num_recs; 3092 } 3093 } 3094 3095 ret = amdgpu_ras_eeprom_check(control); 3096 if (ret) 3097 goto out; 3098 3099 /* HW not usable */ 3100 if (amdgpu_ras_is_rma(adev)) { 3101 ret = -EHWPOISON; 3102 goto out; 3103 } 3104 3105 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3106 } 3107 3108 out: 3109 kfree(bps); 3110 return ret; 3111 } 3112 3113 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3114 uint64_t addr) 3115 { 3116 struct ras_err_handler_data *data = con->eh_data; 3117 int i; 3118 3119 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3120 for (i = 0; i < data->count; i++) 3121 if (addr == data->bps[i].retired_page) 3122 return true; 3123 3124 return false; 3125 } 3126 3127 /* 3128 * check if an address belongs to bad page 3129 * 3130 * Note: this check is only for umc block 3131 */ 3132 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3133 uint64_t addr) 3134 { 3135 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3136 bool ret = false; 3137 3138 if (!con || !con->eh_data) 3139 return ret; 3140 3141 mutex_lock(&con->recovery_lock); 3142 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3143 mutex_unlock(&con->recovery_lock); 3144 return ret; 3145 } 3146 3147 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3148 uint32_t max_count) 3149 { 3150 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3151 3152 /* 3153 * amdgpu_bad_page_threshold is used to config 3154 * the threshold for the number of bad pages. 3155 * -1: Threshold is set to default value 3156 * Driver will issue a warning message when threshold is reached 3157 * and continue runtime services. 3158 * 0: Disable bad page retirement 3159 * Driver will not retire bad pages 3160 * which is intended for debugging purpose. 3161 * -2: Threshold is determined by a formula 3162 * that assumes 1 bad page per 100M of local memory. 3163 * Driver will continue runtime services when threhold is reached. 3164 * 0 < threshold < max number of bad page records in EEPROM, 3165 * A user-defined threshold is set 3166 * Driver will halt runtime services when this custom threshold is reached. 3167 */ 3168 if (amdgpu_bad_page_threshold == -2) { 3169 u64 val = adev->gmc.mc_vram_size; 3170 3171 do_div(val, RAS_BAD_PAGE_COVER); 3172 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3173 max_count); 3174 } else if (amdgpu_bad_page_threshold == -1) { 3175 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3176 } else { 3177 con->bad_page_cnt_threshold = min_t(int, max_count, 3178 amdgpu_bad_page_threshold); 3179 } 3180 } 3181 3182 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3183 enum amdgpu_ras_block block, uint16_t pasid, 3184 pasid_notify pasid_fn, void *data, uint32_t reset) 3185 { 3186 int ret = 0; 3187 struct ras_poison_msg poison_msg; 3188 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3189 3190 memset(&poison_msg, 0, sizeof(poison_msg)); 3191 poison_msg.block = block; 3192 poison_msg.pasid = pasid; 3193 poison_msg.reset = reset; 3194 poison_msg.pasid_fn = pasid_fn; 3195 poison_msg.data = data; 3196 3197 ret = kfifo_put(&con->poison_fifo, poison_msg); 3198 if (!ret) { 3199 dev_err(adev->dev, "Poison message fifo is full!\n"); 3200 return -ENOSPC; 3201 } 3202 3203 return 0; 3204 } 3205 3206 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3207 struct ras_poison_msg *poison_msg) 3208 { 3209 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3210 3211 return kfifo_get(&con->poison_fifo, poison_msg); 3212 } 3213 3214 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3215 { 3216 mutex_init(&ecc_log->lock); 3217 3218 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3219 ecc_log->de_queried_count = 0; 3220 ecc_log->prev_de_queried_count = 0; 3221 } 3222 3223 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3224 { 3225 struct radix_tree_iter iter; 3226 void __rcu **slot; 3227 struct ras_ecc_err *ecc_err; 3228 3229 mutex_lock(&ecc_log->lock); 3230 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3231 ecc_err = radix_tree_deref_slot(slot); 3232 kfree(ecc_err->err_pages.pfn); 3233 kfree(ecc_err); 3234 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3235 } 3236 mutex_unlock(&ecc_log->lock); 3237 3238 mutex_destroy(&ecc_log->lock); 3239 ecc_log->de_queried_count = 0; 3240 ecc_log->prev_de_queried_count = 0; 3241 } 3242 3243 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3244 uint32_t delayed_ms) 3245 { 3246 int ret; 3247 3248 mutex_lock(&con->umc_ecc_log.lock); 3249 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3250 UMC_ECC_NEW_DETECTED_TAG); 3251 mutex_unlock(&con->umc_ecc_log.lock); 3252 3253 if (ret) 3254 schedule_delayed_work(&con->page_retirement_dwork, 3255 msecs_to_jiffies(delayed_ms)); 3256 3257 return ret ? true : false; 3258 } 3259 3260 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3261 { 3262 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3263 page_retirement_dwork.work); 3264 struct amdgpu_device *adev = con->adev; 3265 struct ras_err_data err_data; 3266 unsigned long err_cnt; 3267 3268 /* If gpu reset is ongoing, delay retiring the bad pages */ 3269 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3270 amdgpu_ras_schedule_retirement_dwork(con, 3271 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3272 return; 3273 } 3274 3275 amdgpu_ras_error_data_init(&err_data); 3276 3277 amdgpu_umc_handle_bad_pages(adev, &err_data); 3278 err_cnt = err_data.err_addr_cnt; 3279 3280 amdgpu_ras_error_data_fini(&err_data); 3281 3282 if (err_cnt && amdgpu_ras_is_rma(adev)) 3283 amdgpu_ras_reset_gpu(adev); 3284 3285 amdgpu_ras_schedule_retirement_dwork(con, 3286 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3287 } 3288 3289 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3290 uint32_t poison_creation_count) 3291 { 3292 int ret = 0; 3293 struct ras_ecc_log_info *ecc_log; 3294 struct ras_query_if info; 3295 uint32_t timeout = 0; 3296 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3297 uint64_t de_queried_count; 3298 uint32_t new_detect_count, total_detect_count; 3299 uint32_t need_query_count = poison_creation_count; 3300 bool query_data_timeout = false; 3301 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3302 3303 memset(&info, 0, sizeof(info)); 3304 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3305 3306 ecc_log = &ras->umc_ecc_log; 3307 total_detect_count = 0; 3308 do { 3309 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3310 if (ret) 3311 return ret; 3312 3313 de_queried_count = ecc_log->de_queried_count; 3314 if (de_queried_count > ecc_log->prev_de_queried_count) { 3315 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count; 3316 ecc_log->prev_de_queried_count = de_queried_count; 3317 timeout = 0; 3318 } else { 3319 new_detect_count = 0; 3320 } 3321 3322 if (new_detect_count) { 3323 total_detect_count += new_detect_count; 3324 } else { 3325 if (!timeout && need_query_count) 3326 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3327 3328 if (timeout) { 3329 if (!--timeout) { 3330 query_data_timeout = true; 3331 break; 3332 } 3333 msleep(1); 3334 } 3335 } 3336 } while (total_detect_count < need_query_count); 3337 3338 if (query_data_timeout) { 3339 dev_warn(adev->dev, "Can't find deferred error! count: %u\n", 3340 (need_query_count - total_detect_count)); 3341 return -ENOENT; 3342 } 3343 3344 if (total_detect_count) 3345 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3346 3347 return 0; 3348 } 3349 3350 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3351 { 3352 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3353 struct ras_poison_msg msg; 3354 int ret; 3355 3356 do { 3357 ret = kfifo_get(&con->poison_fifo, &msg); 3358 } while (ret); 3359 } 3360 3361 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3362 uint32_t msg_count, uint32_t *gpu_reset) 3363 { 3364 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3365 uint32_t reset_flags = 0, reset = 0; 3366 struct ras_poison_msg msg; 3367 int ret, i; 3368 3369 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3370 3371 for (i = 0; i < msg_count; i++) { 3372 ret = amdgpu_ras_get_poison_req(adev, &msg); 3373 if (!ret) 3374 continue; 3375 3376 if (msg.pasid_fn) 3377 msg.pasid_fn(adev, msg.pasid, msg.data); 3378 3379 reset_flags |= msg.reset; 3380 } 3381 3382 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3383 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3384 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3385 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3386 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3387 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3388 else 3389 reset = reset_flags; 3390 3391 flush_delayed_work(&con->page_retirement_dwork); 3392 3393 con->gpu_reset_flags |= reset; 3394 amdgpu_ras_reset_gpu(adev); 3395 3396 *gpu_reset = reset; 3397 3398 /* Wait for gpu recovery to complete */ 3399 flush_work(&con->recovery_work); 3400 } 3401 3402 return 0; 3403 } 3404 3405 static int amdgpu_ras_page_retirement_thread(void *param) 3406 { 3407 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3408 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3409 uint32_t poison_creation_count, msg_count; 3410 uint32_t gpu_reset; 3411 int ret; 3412 3413 while (!kthread_should_stop()) { 3414 3415 wait_event_interruptible(con->page_retirement_wq, 3416 kthread_should_stop() || 3417 atomic_read(&con->page_retirement_req_cnt)); 3418 3419 if (kthread_should_stop()) 3420 break; 3421 3422 gpu_reset = 0; 3423 3424 do { 3425 poison_creation_count = atomic_read(&con->poison_creation_count); 3426 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3427 if (ret == -EIO) 3428 break; 3429 3430 if (poison_creation_count) { 3431 atomic_sub(poison_creation_count, &con->poison_creation_count); 3432 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3433 } 3434 } while (atomic_read(&con->poison_creation_count)); 3435 3436 if (ret != -EIO) { 3437 msg_count = kfifo_len(&con->poison_fifo); 3438 if (msg_count) { 3439 ret = amdgpu_ras_poison_consumption_handler(adev, 3440 msg_count, &gpu_reset); 3441 if ((ret != -EIO) && 3442 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3443 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3444 } 3445 } 3446 3447 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3448 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3449 /* Clear poison creation request */ 3450 atomic_set(&con->poison_creation_count, 0); 3451 3452 /* Clear poison fifo */ 3453 amdgpu_ras_clear_poison_fifo(adev); 3454 3455 /* Clear all poison requests */ 3456 atomic_set(&con->page_retirement_req_cnt, 0); 3457 3458 if (ret == -EIO) { 3459 /* Wait for mode-1 reset to complete */ 3460 down_read(&adev->reset_domain->sem); 3461 up_read(&adev->reset_domain->sem); 3462 } 3463 3464 /* Wake up work to save bad pages to eeprom */ 3465 schedule_delayed_work(&con->page_retirement_dwork, 0); 3466 } else if (gpu_reset) { 3467 /* gpu just completed mode-2 reset or other reset */ 3468 /* Clear poison consumption messages cached in fifo */ 3469 msg_count = kfifo_len(&con->poison_fifo); 3470 if (msg_count) { 3471 amdgpu_ras_clear_poison_fifo(adev); 3472 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3473 } 3474 3475 /* Wake up work to save bad pages to eeprom */ 3476 schedule_delayed_work(&con->page_retirement_dwork, 0); 3477 } 3478 } 3479 3480 return 0; 3481 } 3482 3483 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3484 { 3485 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3486 struct amdgpu_ras_eeprom_control *control; 3487 int ret; 3488 3489 if (!con || amdgpu_sriov_vf(adev)) 3490 return 0; 3491 3492 control = &con->eeprom_control; 3493 ret = amdgpu_ras_eeprom_init(control); 3494 if (ret) 3495 return ret; 3496 3497 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3498 control->ras_num_pa_recs = control->ras_num_recs; 3499 3500 if (adev->umc.ras && 3501 adev->umc.ras->get_retire_flip_bits) 3502 adev->umc.ras->get_retire_flip_bits(adev); 3503 3504 if (control->ras_num_recs) { 3505 ret = amdgpu_ras_load_bad_pages(adev); 3506 if (ret) 3507 return ret; 3508 3509 amdgpu_dpm_send_hbm_bad_pages_num( 3510 adev, control->ras_num_bad_pages); 3511 3512 if (con->update_channel_flag == true) { 3513 amdgpu_dpm_send_hbm_bad_channel_flag( 3514 adev, control->bad_channel_bitmap); 3515 con->update_channel_flag = false; 3516 } 3517 3518 /* The format action is only applied to new ASICs */ 3519 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3520 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3521 if (!amdgpu_ras_eeprom_reset_table(control)) 3522 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3523 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3524 } 3525 3526 return ret; 3527 } 3528 3529 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3530 { 3531 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3532 struct ras_err_handler_data **data; 3533 u32 max_eeprom_records_count = 0; 3534 int ret; 3535 3536 if (!con || amdgpu_sriov_vf(adev)) 3537 return 0; 3538 3539 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3540 * supports RAS and debugfs is enabled, but when 3541 * adev->ras_enabled is unset, i.e. when "ras_enable" 3542 * module parameter is set to 0. 3543 */ 3544 con->adev = adev; 3545 3546 if (!adev->ras_enabled) 3547 return 0; 3548 3549 data = &con->eh_data; 3550 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3551 if (!*data) { 3552 ret = -ENOMEM; 3553 goto out; 3554 } 3555 3556 mutex_init(&con->recovery_lock); 3557 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3558 atomic_set(&con->in_recovery, 0); 3559 con->eeprom_control.bad_channel_bitmap = 0; 3560 3561 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3562 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3563 3564 if (init_bp_info) { 3565 ret = amdgpu_ras_init_badpage_info(adev); 3566 if (ret) 3567 goto free; 3568 } 3569 3570 mutex_init(&con->page_rsv_lock); 3571 INIT_KFIFO(con->poison_fifo); 3572 mutex_init(&con->page_retirement_lock); 3573 init_waitqueue_head(&con->page_retirement_wq); 3574 atomic_set(&con->page_retirement_req_cnt, 0); 3575 atomic_set(&con->poison_creation_count, 0); 3576 con->page_retirement_thread = 3577 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3578 if (IS_ERR(con->page_retirement_thread)) { 3579 con->page_retirement_thread = NULL; 3580 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3581 } 3582 3583 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3584 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3585 #ifdef CONFIG_X86_MCE_AMD 3586 if ((adev->asic_type == CHIP_ALDEBARAN) && 3587 (adev->gmc.xgmi.connected_to_cpu)) 3588 amdgpu_register_bad_pages_mca_notifier(adev); 3589 #endif 3590 return 0; 3591 3592 free: 3593 kfree((*data)->bps); 3594 kfree(*data); 3595 con->eh_data = NULL; 3596 out: 3597 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3598 3599 /* 3600 * Except error threshold exceeding case, other failure cases in this 3601 * function would not fail amdgpu driver init. 3602 */ 3603 if (!amdgpu_ras_is_rma(adev)) 3604 ret = 0; 3605 else 3606 ret = -EINVAL; 3607 3608 return ret; 3609 } 3610 3611 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3612 { 3613 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3614 struct ras_err_handler_data *data = con->eh_data; 3615 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3616 bool ret; 3617 3618 /* recovery_init failed to init it, fini is useless */ 3619 if (!data) 3620 return 0; 3621 3622 /* Save all cached bad pages to eeprom */ 3623 do { 3624 flush_delayed_work(&con->page_retirement_dwork); 3625 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3626 } while (ret && max_flush_timeout--); 3627 3628 if (con->page_retirement_thread) 3629 kthread_stop(con->page_retirement_thread); 3630 3631 atomic_set(&con->page_retirement_req_cnt, 0); 3632 atomic_set(&con->poison_creation_count, 0); 3633 3634 mutex_destroy(&con->page_rsv_lock); 3635 3636 cancel_work_sync(&con->recovery_work); 3637 3638 cancel_delayed_work_sync(&con->page_retirement_dwork); 3639 3640 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3641 3642 mutex_lock(&con->recovery_lock); 3643 con->eh_data = NULL; 3644 kfree(data->bps); 3645 kfree(data); 3646 mutex_unlock(&con->recovery_lock); 3647 3648 return 0; 3649 } 3650 /* recovery end */ 3651 3652 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3653 { 3654 if (amdgpu_sriov_vf(adev)) { 3655 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3656 case IP_VERSION(13, 0, 2): 3657 case IP_VERSION(13, 0, 6): 3658 case IP_VERSION(13, 0, 12): 3659 case IP_VERSION(13, 0, 14): 3660 return true; 3661 default: 3662 return false; 3663 } 3664 } 3665 3666 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3667 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3668 case IP_VERSION(13, 0, 0): 3669 case IP_VERSION(13, 0, 6): 3670 case IP_VERSION(13, 0, 10): 3671 case IP_VERSION(13, 0, 12): 3672 case IP_VERSION(13, 0, 14): 3673 case IP_VERSION(14, 0, 3): 3674 return true; 3675 default: 3676 return false; 3677 } 3678 } 3679 3680 return adev->asic_type == CHIP_VEGA10 || 3681 adev->asic_type == CHIP_VEGA20 || 3682 adev->asic_type == CHIP_ARCTURUS || 3683 adev->asic_type == CHIP_ALDEBARAN || 3684 adev->asic_type == CHIP_SIENNA_CICHLID; 3685 } 3686 3687 /* 3688 * this is workaround for vega20 workstation sku, 3689 * force enable gfx ras, ignore vbios gfx ras flag 3690 * due to GC EDC can not write 3691 */ 3692 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3693 { 3694 struct atom_context *ctx = adev->mode_info.atom_context; 3695 3696 if (!ctx) 3697 return; 3698 3699 if (strnstr(ctx->vbios_pn, "D16406", 3700 sizeof(ctx->vbios_pn)) || 3701 strnstr(ctx->vbios_pn, "D36002", 3702 sizeof(ctx->vbios_pn))) 3703 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3704 } 3705 3706 /* Query ras capablity via atomfirmware interface */ 3707 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3708 { 3709 /* mem_ecc cap */ 3710 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3711 dev_info(adev->dev, "MEM ECC is active.\n"); 3712 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3713 1 << AMDGPU_RAS_BLOCK__DF); 3714 } else { 3715 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3716 } 3717 3718 /* sram_ecc cap */ 3719 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 3720 dev_info(adev->dev, "SRAM ECC is active.\n"); 3721 if (!amdgpu_sriov_vf(adev)) 3722 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 3723 1 << AMDGPU_RAS_BLOCK__DF); 3724 else 3725 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 3726 1 << AMDGPU_RAS_BLOCK__SDMA | 3727 1 << AMDGPU_RAS_BLOCK__GFX); 3728 3729 /* 3730 * VCN/JPEG RAS can be supported on both bare metal and 3731 * SRIOV environment 3732 */ 3733 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 3734 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 3735 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 3736 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 3737 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 3738 1 << AMDGPU_RAS_BLOCK__JPEG); 3739 else 3740 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 3741 1 << AMDGPU_RAS_BLOCK__JPEG); 3742 3743 /* 3744 * XGMI RAS is not supported if xgmi num physical nodes 3745 * is zero 3746 */ 3747 if (!adev->gmc.xgmi.num_physical_nodes) 3748 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 3749 } else { 3750 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 3751 } 3752 } 3753 3754 /* Query poison mode from umc/df IP callbacks */ 3755 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 3756 { 3757 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3758 bool df_poison, umc_poison; 3759 3760 /* poison setting is useless on SRIOV guest */ 3761 if (amdgpu_sriov_vf(adev) || !con) 3762 return; 3763 3764 /* Init poison supported flag, the default value is false */ 3765 if (adev->gmc.xgmi.connected_to_cpu || 3766 adev->gmc.is_app_apu) { 3767 /* enabled by default when GPU is connected to CPU */ 3768 con->poison_supported = true; 3769 } else if (adev->df.funcs && 3770 adev->df.funcs->query_ras_poison_mode && 3771 adev->umc.ras && 3772 adev->umc.ras->query_ras_poison_mode) { 3773 df_poison = 3774 adev->df.funcs->query_ras_poison_mode(adev); 3775 umc_poison = 3776 adev->umc.ras->query_ras_poison_mode(adev); 3777 3778 /* Only poison is set in both DF and UMC, we can support it */ 3779 if (df_poison && umc_poison) 3780 con->poison_supported = true; 3781 else if (df_poison != umc_poison) 3782 dev_warn(adev->dev, 3783 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 3784 df_poison, umc_poison); 3785 } 3786 } 3787 3788 /* 3789 * check hardware's ras ability which will be saved in hw_supported. 3790 * if hardware does not support ras, we can skip some ras initializtion and 3791 * forbid some ras operations from IP. 3792 * if software itself, say boot parameter, limit the ras ability. We still 3793 * need allow IP do some limited operations, like disable. In such case, 3794 * we have to initialize ras as normal. but need check if operation is 3795 * allowed or not in each function. 3796 */ 3797 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 3798 { 3799 adev->ras_hw_enabled = adev->ras_enabled = 0; 3800 3801 if (!amdgpu_ras_asic_supported(adev)) 3802 return; 3803 3804 if (amdgpu_sriov_vf(adev)) { 3805 if (amdgpu_virt_get_ras_capability(adev)) 3806 goto init_ras_enabled_flag; 3807 } 3808 3809 /* query ras capability from psp */ 3810 if (amdgpu_psp_get_ras_capability(&adev->psp)) 3811 goto init_ras_enabled_flag; 3812 3813 /* query ras capablity from bios */ 3814 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 3815 amdgpu_ras_query_ras_capablity_from_vbios(adev); 3816 } else { 3817 /* driver only manages a few IP blocks RAS feature 3818 * when GPU is connected cpu through XGMI */ 3819 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 3820 1 << AMDGPU_RAS_BLOCK__SDMA | 3821 1 << AMDGPU_RAS_BLOCK__MMHUB); 3822 } 3823 3824 /* apply asic specific settings (vega20 only for now) */ 3825 amdgpu_ras_get_quirks(adev); 3826 3827 /* query poison mode from umc/df ip callback */ 3828 amdgpu_ras_query_poison_mode(adev); 3829 3830 init_ras_enabled_flag: 3831 /* hw_supported needs to be aligned with RAS block mask. */ 3832 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 3833 3834 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 3835 adev->ras_hw_enabled & amdgpu_ras_mask; 3836 3837 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 3838 if (!amdgpu_sriov_vf(adev)) { 3839 adev->aca.is_enabled = 3840 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 3841 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 3842 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 3843 } 3844 3845 /* bad page feature is not applicable to specific app platform */ 3846 if (adev->gmc.is_app_apu && 3847 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 3848 amdgpu_bad_page_threshold = 0; 3849 } 3850 3851 static void amdgpu_ras_counte_dw(struct work_struct *work) 3852 { 3853 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3854 ras_counte_delay_work.work); 3855 struct amdgpu_device *adev = con->adev; 3856 struct drm_device *dev = adev_to_drm(adev); 3857 unsigned long ce_count, ue_count; 3858 int res; 3859 3860 res = pm_runtime_get_sync(dev->dev); 3861 if (res < 0) 3862 goto Out; 3863 3864 /* Cache new values. 3865 */ 3866 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 3867 atomic_set(&con->ras_ce_count, ce_count); 3868 atomic_set(&con->ras_ue_count, ue_count); 3869 } 3870 3871 pm_runtime_mark_last_busy(dev->dev); 3872 Out: 3873 pm_runtime_put_autosuspend(dev->dev); 3874 } 3875 3876 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 3877 { 3878 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 3879 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 3880 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 3881 AMDGPU_RAS_ERROR__PARITY; 3882 } 3883 3884 static void ras_event_mgr_init(struct ras_event_manager *mgr) 3885 { 3886 struct ras_event_state *event_state; 3887 int i; 3888 3889 memset(mgr, 0, sizeof(*mgr)); 3890 atomic64_set(&mgr->seqno, 0); 3891 3892 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 3893 event_state = &mgr->event_state[i]; 3894 event_state->last_seqno = RAS_EVENT_INVALID_ID; 3895 atomic64_set(&event_state->count, 0); 3896 } 3897 } 3898 3899 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 3900 { 3901 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3902 struct amdgpu_hive_info *hive; 3903 3904 if (!ras) 3905 return; 3906 3907 hive = amdgpu_get_xgmi_hive(adev); 3908 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 3909 3910 /* init event manager with node 0 on xgmi system */ 3911 if (!amdgpu_reset_in_recovery(adev)) { 3912 if (!hive || adev->gmc.xgmi.node_id == 0) 3913 ras_event_mgr_init(ras->event_mgr); 3914 } 3915 3916 if (hive) 3917 amdgpu_put_xgmi_hive(hive); 3918 } 3919 3920 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 3921 { 3922 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3923 3924 if (!con || (adev->flags & AMD_IS_APU)) 3925 return; 3926 3927 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3928 case IP_VERSION(13, 0, 2): 3929 case IP_VERSION(13, 0, 6): 3930 case IP_VERSION(13, 0, 12): 3931 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 3932 break; 3933 case IP_VERSION(13, 0, 14): 3934 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 3935 break; 3936 default: 3937 break; 3938 } 3939 } 3940 3941 int amdgpu_ras_init(struct amdgpu_device *adev) 3942 { 3943 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3944 int r; 3945 3946 if (con) 3947 return 0; 3948 3949 con = kzalloc(sizeof(*con) + 3950 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 3951 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 3952 GFP_KERNEL); 3953 if (!con) 3954 return -ENOMEM; 3955 3956 con->adev = adev; 3957 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 3958 atomic_set(&con->ras_ce_count, 0); 3959 atomic_set(&con->ras_ue_count, 0); 3960 3961 con->objs = (struct ras_manager *)(con + 1); 3962 3963 amdgpu_ras_set_context(adev, con); 3964 3965 amdgpu_ras_check_supported(adev); 3966 3967 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 3968 /* set gfx block ras context feature for VEGA20 Gaming 3969 * send ras disable cmd to ras ta during ras late init. 3970 */ 3971 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 3972 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 3973 3974 return 0; 3975 } 3976 3977 r = 0; 3978 goto release_con; 3979 } 3980 3981 con->update_channel_flag = false; 3982 con->features = 0; 3983 con->schema = 0; 3984 INIT_LIST_HEAD(&con->head); 3985 /* Might need get this flag from vbios. */ 3986 con->flags = RAS_DEFAULT_FLAGS; 3987 3988 /* initialize nbio ras function ahead of any other 3989 * ras functions so hardware fatal error interrupt 3990 * can be enabled as early as possible */ 3991 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 3992 case IP_VERSION(7, 4, 0): 3993 case IP_VERSION(7, 4, 1): 3994 case IP_VERSION(7, 4, 4): 3995 if (!adev->gmc.xgmi.connected_to_cpu) 3996 adev->nbio.ras = &nbio_v7_4_ras; 3997 break; 3998 case IP_VERSION(4, 3, 0): 3999 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4000 /* unlike other generation of nbio ras, 4001 * nbio v4_3 only support fatal error interrupt 4002 * to inform software that DF is freezed due to 4003 * system fatal error event. driver should not 4004 * enable nbio ras in such case. Instead, 4005 * check DF RAS */ 4006 adev->nbio.ras = &nbio_v4_3_ras; 4007 break; 4008 case IP_VERSION(6, 3, 1): 4009 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4010 /* unlike other generation of nbio ras, 4011 * nbif v6_3_1 only support fatal error interrupt 4012 * to inform software that DF is freezed due to 4013 * system fatal error event. driver should not 4014 * enable nbio ras in such case. Instead, 4015 * check DF RAS 4016 */ 4017 adev->nbio.ras = &nbif_v6_3_1_ras; 4018 break; 4019 case IP_VERSION(7, 9, 0): 4020 case IP_VERSION(7, 9, 1): 4021 if (!adev->gmc.is_app_apu) 4022 adev->nbio.ras = &nbio_v7_9_ras; 4023 break; 4024 default: 4025 /* nbio ras is not available */ 4026 break; 4027 } 4028 4029 /* nbio ras block needs to be enabled ahead of other ras blocks 4030 * to handle fatal error */ 4031 r = amdgpu_nbio_ras_sw_init(adev); 4032 if (r) 4033 return r; 4034 4035 if (adev->nbio.ras && 4036 adev->nbio.ras->init_ras_controller_interrupt) { 4037 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4038 if (r) 4039 goto release_con; 4040 } 4041 4042 if (adev->nbio.ras && 4043 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4044 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4045 if (r) 4046 goto release_con; 4047 } 4048 4049 /* Packed socket_id to ras feature mask bits[31:29] */ 4050 if (adev->smuio.funcs && 4051 adev->smuio.funcs->get_socket_id) 4052 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4053 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4054 4055 /* Get RAS schema for particular SOC */ 4056 con->schema = amdgpu_get_ras_schema(adev); 4057 4058 amdgpu_ras_init_reserved_vram_size(adev); 4059 4060 if (amdgpu_ras_fs_init(adev)) { 4061 r = -EINVAL; 4062 goto release_con; 4063 } 4064 4065 if (amdgpu_ras_aca_is_supported(adev)) { 4066 if (amdgpu_aca_is_enabled(adev)) 4067 r = amdgpu_aca_init(adev); 4068 else 4069 r = amdgpu_mca_init(adev); 4070 if (r) 4071 goto release_con; 4072 } 4073 4074 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4075 "hardware ability[%x] ras_mask[%x]\n", 4076 adev->ras_hw_enabled, adev->ras_enabled); 4077 4078 return 0; 4079 release_con: 4080 amdgpu_ras_set_context(adev, NULL); 4081 kfree(con); 4082 4083 return r; 4084 } 4085 4086 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4087 { 4088 if (adev->gmc.xgmi.connected_to_cpu || 4089 adev->gmc.is_app_apu) 4090 return 1; 4091 return 0; 4092 } 4093 4094 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4095 struct ras_common_if *ras_block) 4096 { 4097 struct ras_query_if info = { 4098 .head = *ras_block, 4099 }; 4100 4101 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4102 return 0; 4103 4104 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4105 DRM_WARN("RAS init harvest failure"); 4106 4107 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4108 DRM_WARN("RAS init harvest reset failure"); 4109 4110 return 0; 4111 } 4112 4113 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4114 { 4115 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4116 4117 if (!con) 4118 return false; 4119 4120 return con->poison_supported; 4121 } 4122 4123 /* helper function to handle common stuff in ip late init phase */ 4124 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4125 struct ras_common_if *ras_block) 4126 { 4127 struct amdgpu_ras_block_object *ras_obj = NULL; 4128 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4129 struct ras_query_if *query_info; 4130 unsigned long ue_count, ce_count; 4131 int r; 4132 4133 /* disable RAS feature per IP block if it is not supported */ 4134 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4135 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4136 return 0; 4137 } 4138 4139 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4140 if (r) { 4141 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4142 /* in resume phase, if fail to enable ras, 4143 * clean up all ras fs nodes, and disable ras */ 4144 goto cleanup; 4145 } else 4146 return r; 4147 } 4148 4149 /* check for errors on warm reset edc persisant supported ASIC */ 4150 amdgpu_persistent_edc_harvesting(adev, ras_block); 4151 4152 /* in resume phase, no need to create ras fs node */ 4153 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4154 return 0; 4155 4156 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4157 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4158 (ras_obj->hw_ops->query_poison_status || 4159 ras_obj->hw_ops->handle_poison_consumption))) { 4160 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4161 if (r) 4162 goto cleanup; 4163 } 4164 4165 if (ras_obj->hw_ops && 4166 (ras_obj->hw_ops->query_ras_error_count || 4167 ras_obj->hw_ops->query_ras_error_status)) { 4168 r = amdgpu_ras_sysfs_create(adev, ras_block); 4169 if (r) 4170 goto interrupt; 4171 4172 /* Those are the cached values at init. 4173 */ 4174 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4175 if (!query_info) 4176 return -ENOMEM; 4177 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4178 4179 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4180 atomic_set(&con->ras_ce_count, ce_count); 4181 atomic_set(&con->ras_ue_count, ue_count); 4182 } 4183 4184 kfree(query_info); 4185 } 4186 4187 return 0; 4188 4189 interrupt: 4190 if (ras_obj->ras_cb) 4191 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4192 cleanup: 4193 amdgpu_ras_feature_enable(adev, ras_block, 0); 4194 return r; 4195 } 4196 4197 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4198 struct ras_common_if *ras_block) 4199 { 4200 return amdgpu_ras_block_late_init(adev, ras_block); 4201 } 4202 4203 /* helper function to remove ras fs node and interrupt handler */ 4204 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4205 struct ras_common_if *ras_block) 4206 { 4207 struct amdgpu_ras_block_object *ras_obj; 4208 if (!ras_block) 4209 return; 4210 4211 amdgpu_ras_sysfs_remove(adev, ras_block); 4212 4213 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4214 if (ras_obj->ras_cb) 4215 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4216 } 4217 4218 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4219 struct ras_common_if *ras_block) 4220 { 4221 return amdgpu_ras_block_late_fini(adev, ras_block); 4222 } 4223 4224 /* do some init work after IP late init as dependence. 4225 * and it runs in resume/gpu reset/booting up cases. 4226 */ 4227 void amdgpu_ras_resume(struct amdgpu_device *adev) 4228 { 4229 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4230 struct ras_manager *obj, *tmp; 4231 4232 if (!adev->ras_enabled || !con) { 4233 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4234 amdgpu_release_ras_context(adev); 4235 4236 return; 4237 } 4238 4239 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4240 /* Set up all other IPs which are not implemented. There is a 4241 * tricky thing that IP's actual ras error type should be 4242 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4243 * ERROR_NONE make sense anyway. 4244 */ 4245 amdgpu_ras_enable_all_features(adev, 1); 4246 4247 /* We enable ras on all hw_supported block, but as boot 4248 * parameter might disable some of them and one or more IP has 4249 * not implemented yet. So we disable them on behalf. 4250 */ 4251 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4252 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4253 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4254 /* there should be no any reference. */ 4255 WARN_ON(alive_obj(obj)); 4256 } 4257 } 4258 } 4259 } 4260 4261 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4262 { 4263 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4264 4265 if (!adev->ras_enabled || !con) 4266 return; 4267 4268 amdgpu_ras_disable_all_features(adev, 0); 4269 /* Make sure all ras objects are disabled. */ 4270 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4271 amdgpu_ras_disable_all_features(adev, 1); 4272 } 4273 4274 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4275 { 4276 struct amdgpu_ras_block_list *node, *tmp; 4277 struct amdgpu_ras_block_object *obj; 4278 int r; 4279 4280 amdgpu_ras_event_mgr_init(adev); 4281 4282 if (amdgpu_ras_aca_is_supported(adev)) { 4283 if (amdgpu_reset_in_recovery(adev)) { 4284 if (amdgpu_aca_is_enabled(adev)) 4285 r = amdgpu_aca_reset(adev); 4286 else 4287 r = amdgpu_mca_reset(adev); 4288 if (r) 4289 return r; 4290 } 4291 4292 if (!amdgpu_sriov_vf(adev)) { 4293 if (amdgpu_aca_is_enabled(adev)) 4294 amdgpu_ras_set_aca_debug_mode(adev, false); 4295 else 4296 amdgpu_ras_set_mca_debug_mode(adev, false); 4297 } 4298 } 4299 4300 /* Guest side doesn't need init ras feature */ 4301 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4302 return 0; 4303 4304 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4305 obj = node->ras_obj; 4306 if (!obj) { 4307 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4308 continue; 4309 } 4310 4311 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4312 continue; 4313 4314 if (obj->ras_late_init) { 4315 r = obj->ras_late_init(adev, &obj->ras_comm); 4316 if (r) { 4317 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4318 obj->ras_comm.name, r); 4319 return r; 4320 } 4321 } else 4322 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4323 } 4324 4325 return 0; 4326 } 4327 4328 /* do some fini work before IP fini as dependence */ 4329 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4330 { 4331 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4332 4333 if (!adev->ras_enabled || !con) 4334 return 0; 4335 4336 4337 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4338 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4339 amdgpu_ras_disable_all_features(adev, 0); 4340 amdgpu_ras_recovery_fini(adev); 4341 return 0; 4342 } 4343 4344 int amdgpu_ras_fini(struct amdgpu_device *adev) 4345 { 4346 struct amdgpu_ras_block_list *ras_node, *tmp; 4347 struct amdgpu_ras_block_object *obj = NULL; 4348 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4349 4350 if (!adev->ras_enabled || !con) 4351 return 0; 4352 4353 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4354 if (ras_node->ras_obj) { 4355 obj = ras_node->ras_obj; 4356 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4357 obj->ras_fini) 4358 obj->ras_fini(adev, &obj->ras_comm); 4359 else 4360 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4361 } 4362 4363 /* Clear ras blocks from ras_list and free ras block list node */ 4364 list_del(&ras_node->node); 4365 kfree(ras_node); 4366 } 4367 4368 amdgpu_ras_fs_fini(adev); 4369 amdgpu_ras_interrupt_remove_all(adev); 4370 4371 if (amdgpu_ras_aca_is_supported(adev)) { 4372 if (amdgpu_aca_is_enabled(adev)) 4373 amdgpu_aca_fini(adev); 4374 else 4375 amdgpu_mca_fini(adev); 4376 } 4377 4378 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4379 4380 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4381 amdgpu_ras_disable_all_features(adev, 0); 4382 4383 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4384 4385 amdgpu_ras_set_context(adev, NULL); 4386 kfree(con); 4387 4388 return 0; 4389 } 4390 4391 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4392 { 4393 struct amdgpu_ras *ras; 4394 4395 ras = amdgpu_ras_get_context(adev); 4396 if (!ras) 4397 return false; 4398 4399 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4400 } 4401 4402 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4403 { 4404 struct amdgpu_ras *ras; 4405 4406 ras = amdgpu_ras_get_context(adev); 4407 if (ras) { 4408 if (status) 4409 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4410 else 4411 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4412 } 4413 } 4414 4415 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4416 { 4417 struct amdgpu_ras *ras; 4418 4419 ras = amdgpu_ras_get_context(adev); 4420 if (ras) { 4421 ras->ras_err_state = 0; 4422 ras->gpu_reset_flags = 0; 4423 } 4424 } 4425 4426 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4427 enum amdgpu_ras_block block) 4428 { 4429 struct amdgpu_ras *ras; 4430 4431 ras = amdgpu_ras_get_context(adev); 4432 if (ras) 4433 set_bit(block, &ras->ras_err_state); 4434 } 4435 4436 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4437 { 4438 struct amdgpu_ras *ras; 4439 4440 ras = amdgpu_ras_get_context(adev); 4441 if (ras) { 4442 if (block == AMDGPU_RAS_BLOCK__ANY) 4443 return (ras->ras_err_state != 0); 4444 else 4445 return test_bit(block, &ras->ras_err_state) || 4446 test_bit(AMDGPU_RAS_BLOCK__LAST, 4447 &ras->ras_err_state); 4448 } 4449 4450 return false; 4451 } 4452 4453 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4454 { 4455 struct amdgpu_ras *ras; 4456 4457 ras = amdgpu_ras_get_context(adev); 4458 if (!ras) 4459 return NULL; 4460 4461 return ras->event_mgr; 4462 } 4463 4464 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4465 const void *caller) 4466 { 4467 struct ras_event_manager *event_mgr; 4468 struct ras_event_state *event_state; 4469 int ret = 0; 4470 4471 if (type >= RAS_EVENT_TYPE_COUNT) { 4472 ret = -EINVAL; 4473 goto out; 4474 } 4475 4476 event_mgr = __get_ras_event_mgr(adev); 4477 if (!event_mgr) { 4478 ret = -EINVAL; 4479 goto out; 4480 } 4481 4482 event_state = &event_mgr->event_state[type]; 4483 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4484 atomic64_inc(&event_state->count); 4485 4486 out: 4487 if (ret && caller) 4488 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4489 (int)type, caller, ret); 4490 4491 return ret; 4492 } 4493 4494 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4495 { 4496 struct ras_event_manager *event_mgr; 4497 u64 id; 4498 4499 if (type >= RAS_EVENT_TYPE_COUNT) 4500 return RAS_EVENT_INVALID_ID; 4501 4502 switch (type) { 4503 case RAS_EVENT_TYPE_FATAL: 4504 case RAS_EVENT_TYPE_POISON_CREATION: 4505 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4506 event_mgr = __get_ras_event_mgr(adev); 4507 if (!event_mgr) 4508 return RAS_EVENT_INVALID_ID; 4509 4510 id = event_mgr->event_state[type].last_seqno; 4511 break; 4512 case RAS_EVENT_TYPE_INVALID: 4513 default: 4514 id = RAS_EVENT_INVALID_ID; 4515 break; 4516 } 4517 4518 return id; 4519 } 4520 4521 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4522 { 4523 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4524 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4525 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4526 u64 event_id; 4527 4528 if (amdgpu_ras_mark_ras_event(adev, type)) { 4529 dev_err(adev->dev, 4530 "uncorrectable hardware error (ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4531 return; 4532 } 4533 4534 event_id = amdgpu_ras_acquire_event_id(adev, type); 4535 4536 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4537 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4538 4539 amdgpu_ras_set_fed(adev, true); 4540 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4541 amdgpu_ras_reset_gpu(adev); 4542 } 4543 } 4544 4545 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4546 { 4547 if (adev->asic_type == CHIP_VEGA20 && 4548 adev->pm.fw_version <= 0x283400) { 4549 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4550 amdgpu_ras_intr_triggered(); 4551 } 4552 4553 return false; 4554 } 4555 4556 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4557 { 4558 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4559 4560 if (!con) 4561 return; 4562 4563 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4564 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4565 amdgpu_ras_set_context(adev, NULL); 4566 kfree(con); 4567 } 4568 } 4569 4570 #ifdef CONFIG_X86_MCE_AMD 4571 static struct amdgpu_device *find_adev(uint32_t node_id) 4572 { 4573 int i; 4574 struct amdgpu_device *adev = NULL; 4575 4576 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4577 adev = mce_adev_list.devs[i]; 4578 4579 if (adev && adev->gmc.xgmi.connected_to_cpu && 4580 adev->gmc.xgmi.physical_node_id == node_id) 4581 break; 4582 adev = NULL; 4583 } 4584 4585 return adev; 4586 } 4587 4588 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4589 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4590 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4591 #define GPU_ID_OFFSET 8 4592 4593 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4594 unsigned long val, void *data) 4595 { 4596 struct mce *m = (struct mce *)data; 4597 struct amdgpu_device *adev = NULL; 4598 uint32_t gpu_id = 0; 4599 uint32_t umc_inst = 0, ch_inst = 0; 4600 4601 /* 4602 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4603 * and error occurred in DramECC (Extended error code = 0) then only 4604 * process the error, else bail out. 4605 */ 4606 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4607 (XEC(m->status, 0x3f) == 0x0))) 4608 return NOTIFY_DONE; 4609 4610 /* 4611 * If it is correctable error, return. 4612 */ 4613 if (mce_is_correctable(m)) 4614 return NOTIFY_OK; 4615 4616 /* 4617 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4618 */ 4619 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4620 4621 adev = find_adev(gpu_id); 4622 if (!adev) { 4623 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4624 gpu_id); 4625 return NOTIFY_DONE; 4626 } 4627 4628 /* 4629 * If it is uncorrectable error, then find out UMC instance and 4630 * channel index. 4631 */ 4632 umc_inst = GET_UMC_INST(m->ipid); 4633 ch_inst = GET_CHAN_INDEX(m->ipid); 4634 4635 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4636 umc_inst, ch_inst); 4637 4638 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4639 return NOTIFY_OK; 4640 else 4641 return NOTIFY_DONE; 4642 } 4643 4644 static struct notifier_block amdgpu_bad_page_nb = { 4645 .notifier_call = amdgpu_bad_page_notifier, 4646 .priority = MCE_PRIO_UC, 4647 }; 4648 4649 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4650 { 4651 /* 4652 * Add the adev to the mce_adev_list. 4653 * During mode2 reset, amdgpu device is temporarily 4654 * removed from the mgpu_info list which can cause 4655 * page retirement to fail. 4656 * Use this list instead of mgpu_info to find the amdgpu 4657 * device on which the UMC error was reported. 4658 */ 4659 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4660 4661 /* 4662 * Register the x86 notifier only once 4663 * with MCE subsystem. 4664 */ 4665 if (notifier_registered == false) { 4666 mce_register_decode_chain(&amdgpu_bad_page_nb); 4667 notifier_registered = true; 4668 } 4669 } 4670 #endif 4671 4672 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4673 { 4674 if (!adev) 4675 return NULL; 4676 4677 return adev->psp.ras_context.ras; 4678 } 4679 4680 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4681 { 4682 if (!adev) 4683 return -EINVAL; 4684 4685 adev->psp.ras_context.ras = ras_con; 4686 return 0; 4687 } 4688 4689 /* check if ras is supported on block, say, sdma, gfx */ 4690 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4691 unsigned int block) 4692 { 4693 int ret = 0; 4694 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4695 4696 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4697 return 0; 4698 4699 ret = ras && (adev->ras_enabled & (1 << block)); 4700 4701 /* For the special asic with mem ecc enabled but sram ecc 4702 * not enabled, even if the ras block is not supported on 4703 * .ras_enabled, if the asic supports poison mode and the 4704 * ras block has ras configuration, it can be considered 4705 * that the ras block supports ras function. 4706 */ 4707 if (!ret && 4708 (block == AMDGPU_RAS_BLOCK__GFX || 4709 block == AMDGPU_RAS_BLOCK__SDMA || 4710 block == AMDGPU_RAS_BLOCK__VCN || 4711 block == AMDGPU_RAS_BLOCK__JPEG) && 4712 (amdgpu_ras_mask & (1 << block)) && 4713 amdgpu_ras_is_poison_mode_supported(adev) && 4714 amdgpu_ras_get_ras_block(adev, block, 0)) 4715 ret = 1; 4716 4717 return ret; 4718 } 4719 4720 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 4721 { 4722 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4723 4724 /* mode1 is the only selection for RMA status */ 4725 if (amdgpu_ras_is_rma(adev)) { 4726 ras->gpu_reset_flags = 0; 4727 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4728 } 4729 4730 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 4731 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4732 int hive_ras_recovery = 0; 4733 4734 if (hive) { 4735 hive_ras_recovery = atomic_read(&hive->ras_recovery); 4736 amdgpu_put_xgmi_hive(hive); 4737 } 4738 /* In the case of multiple GPUs, after a GPU has started 4739 * resetting all GPUs on hive, other GPUs do not need to 4740 * trigger GPU reset again. 4741 */ 4742 if (!hive_ras_recovery) 4743 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4744 else 4745 atomic_set(&ras->in_recovery, 0); 4746 } else { 4747 flush_work(&ras->recovery_work); 4748 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4749 } 4750 4751 return 0; 4752 } 4753 4754 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 4755 { 4756 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4757 int ret = 0; 4758 4759 if (con) { 4760 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4761 if (!ret) 4762 con->is_aca_debug_mode = enable; 4763 } 4764 4765 return ret; 4766 } 4767 4768 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 4769 { 4770 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4771 int ret = 0; 4772 4773 if (con) { 4774 if (amdgpu_aca_is_enabled(adev)) 4775 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 4776 else 4777 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4778 if (!ret) 4779 con->is_aca_debug_mode = enable; 4780 } 4781 4782 return ret; 4783 } 4784 4785 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 4786 { 4787 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4788 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4789 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4790 4791 if (!con) 4792 return false; 4793 4794 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 4795 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 4796 return con->is_aca_debug_mode; 4797 else 4798 return true; 4799 } 4800 4801 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 4802 unsigned int *error_query_mode) 4803 { 4804 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4805 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4806 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4807 4808 if (!con) { 4809 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 4810 return false; 4811 } 4812 4813 if (amdgpu_sriov_vf(adev)) { 4814 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 4815 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 4816 *error_query_mode = 4817 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 4818 } else { 4819 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 4820 } 4821 4822 return true; 4823 } 4824 4825 /* Register each ip ras block into amdgpu ras */ 4826 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 4827 struct amdgpu_ras_block_object *ras_block_obj) 4828 { 4829 struct amdgpu_ras_block_list *ras_node; 4830 if (!adev || !ras_block_obj) 4831 return -EINVAL; 4832 4833 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 4834 if (!ras_node) 4835 return -ENOMEM; 4836 4837 INIT_LIST_HEAD(&ras_node->node); 4838 ras_node->ras_obj = ras_block_obj; 4839 list_add_tail(&ras_node->node, &adev->ras_list); 4840 4841 return 0; 4842 } 4843 4844 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 4845 { 4846 if (!err_type_name) 4847 return; 4848 4849 switch (err_type) { 4850 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 4851 sprintf(err_type_name, "correctable"); 4852 break; 4853 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 4854 sprintf(err_type_name, "uncorrectable"); 4855 break; 4856 default: 4857 sprintf(err_type_name, "unknown"); 4858 break; 4859 } 4860 } 4861 4862 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 4863 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4864 uint32_t instance, 4865 uint32_t *memory_id) 4866 { 4867 uint32_t err_status_lo_data, err_status_lo_offset; 4868 4869 if (!reg_entry) 4870 return false; 4871 4872 err_status_lo_offset = 4873 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4874 reg_entry->seg_lo, reg_entry->reg_lo); 4875 err_status_lo_data = RREG32(err_status_lo_offset); 4876 4877 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 4878 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 4879 return false; 4880 4881 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 4882 4883 return true; 4884 } 4885 4886 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 4887 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4888 uint32_t instance, 4889 unsigned long *err_cnt) 4890 { 4891 uint32_t err_status_hi_data, err_status_hi_offset; 4892 4893 if (!reg_entry) 4894 return false; 4895 4896 err_status_hi_offset = 4897 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4898 reg_entry->seg_hi, reg_entry->reg_hi); 4899 err_status_hi_data = RREG32(err_status_hi_offset); 4900 4901 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 4902 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 4903 /* keep the check here in case we need to refer to the result later */ 4904 dev_dbg(adev->dev, "Invalid err_info field\n"); 4905 4906 /* read err count */ 4907 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 4908 4909 return true; 4910 } 4911 4912 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 4913 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4914 uint32_t reg_list_size, 4915 const struct amdgpu_ras_memory_id_entry *mem_list, 4916 uint32_t mem_list_size, 4917 uint32_t instance, 4918 uint32_t err_type, 4919 unsigned long *err_count) 4920 { 4921 uint32_t memory_id; 4922 unsigned long err_cnt; 4923 char err_type_name[16]; 4924 uint32_t i, j; 4925 4926 for (i = 0; i < reg_list_size; i++) { 4927 /* query memory_id from err_status_lo */ 4928 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 4929 instance, &memory_id)) 4930 continue; 4931 4932 /* query err_cnt from err_status_hi */ 4933 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 4934 instance, &err_cnt) || 4935 !err_cnt) 4936 continue; 4937 4938 *err_count += err_cnt; 4939 4940 /* log the errors */ 4941 amdgpu_ras_get_error_type_name(err_type, err_type_name); 4942 if (!mem_list) { 4943 /* memory_list is not supported */ 4944 dev_info(adev->dev, 4945 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 4946 err_cnt, err_type_name, 4947 reg_list[i].block_name, 4948 instance, memory_id); 4949 } else { 4950 for (j = 0; j < mem_list_size; j++) { 4951 if (memory_id == mem_list[j].memory_id) { 4952 dev_info(adev->dev, 4953 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 4954 err_cnt, err_type_name, 4955 reg_list[i].block_name, 4956 instance, mem_list[j].name); 4957 break; 4958 } 4959 } 4960 } 4961 } 4962 } 4963 4964 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 4965 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4966 uint32_t reg_list_size, 4967 uint32_t instance) 4968 { 4969 uint32_t err_status_lo_offset, err_status_hi_offset; 4970 uint32_t i; 4971 4972 for (i = 0; i < reg_list_size; i++) { 4973 err_status_lo_offset = 4974 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4975 reg_list[i].seg_lo, reg_list[i].reg_lo); 4976 err_status_hi_offset = 4977 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4978 reg_list[i].seg_hi, reg_list[i].reg_hi); 4979 WREG32(err_status_lo_offset, 0); 4980 WREG32(err_status_hi_offset, 0); 4981 } 4982 } 4983 4984 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 4985 { 4986 memset(err_data, 0, sizeof(*err_data)); 4987 4988 INIT_LIST_HEAD(&err_data->err_node_list); 4989 4990 return 0; 4991 } 4992 4993 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 4994 { 4995 if (!err_node) 4996 return; 4997 4998 list_del(&err_node->node); 4999 kvfree(err_node); 5000 } 5001 5002 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5003 { 5004 struct ras_err_node *err_node, *tmp; 5005 5006 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5007 amdgpu_ras_error_node_release(err_node); 5008 } 5009 5010 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5011 struct amdgpu_smuio_mcm_config_info *mcm_info) 5012 { 5013 struct ras_err_node *err_node; 5014 struct amdgpu_smuio_mcm_config_info *ref_id; 5015 5016 if (!err_data || !mcm_info) 5017 return NULL; 5018 5019 for_each_ras_error(err_node, err_data) { 5020 ref_id = &err_node->err_info.mcm_info; 5021 5022 if (mcm_info->socket_id == ref_id->socket_id && 5023 mcm_info->die_id == ref_id->die_id) 5024 return err_node; 5025 } 5026 5027 return NULL; 5028 } 5029 5030 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5031 { 5032 struct ras_err_node *err_node; 5033 5034 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5035 if (!err_node) 5036 return NULL; 5037 5038 INIT_LIST_HEAD(&err_node->node); 5039 5040 return err_node; 5041 } 5042 5043 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5044 { 5045 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5046 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5047 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5048 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5049 5050 if (unlikely(infoa->socket_id != infob->socket_id)) 5051 return infoa->socket_id - infob->socket_id; 5052 else 5053 return infoa->die_id - infob->die_id; 5054 5055 return 0; 5056 } 5057 5058 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5059 struct amdgpu_smuio_mcm_config_info *mcm_info) 5060 { 5061 struct ras_err_node *err_node; 5062 5063 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5064 if (err_node) 5065 return &err_node->err_info; 5066 5067 err_node = amdgpu_ras_error_node_new(); 5068 if (!err_node) 5069 return NULL; 5070 5071 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5072 5073 err_data->err_list_count++; 5074 list_add_tail(&err_node->node, &err_data->err_node_list); 5075 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5076 5077 return &err_node->err_info; 5078 } 5079 5080 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5081 struct amdgpu_smuio_mcm_config_info *mcm_info, 5082 u64 count) 5083 { 5084 struct ras_err_info *err_info; 5085 5086 if (!err_data || !mcm_info) 5087 return -EINVAL; 5088 5089 if (!count) 5090 return 0; 5091 5092 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5093 if (!err_info) 5094 return -EINVAL; 5095 5096 err_info->ue_count += count; 5097 err_data->ue_count += count; 5098 5099 return 0; 5100 } 5101 5102 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5103 struct amdgpu_smuio_mcm_config_info *mcm_info, 5104 u64 count) 5105 { 5106 struct ras_err_info *err_info; 5107 5108 if (!err_data || !mcm_info) 5109 return -EINVAL; 5110 5111 if (!count) 5112 return 0; 5113 5114 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5115 if (!err_info) 5116 return -EINVAL; 5117 5118 err_info->ce_count += count; 5119 err_data->ce_count += count; 5120 5121 return 0; 5122 } 5123 5124 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5125 struct amdgpu_smuio_mcm_config_info *mcm_info, 5126 u64 count) 5127 { 5128 struct ras_err_info *err_info; 5129 5130 if (!err_data || !mcm_info) 5131 return -EINVAL; 5132 5133 if (!count) 5134 return 0; 5135 5136 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5137 if (!err_info) 5138 return -EINVAL; 5139 5140 err_info->de_count += count; 5141 err_data->de_count += count; 5142 5143 return 0; 5144 } 5145 5146 #define mmMP0_SMN_C2PMSG_92 0x1609C 5147 #define mmMP0_SMN_C2PMSG_126 0x160BE 5148 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5149 u32 instance) 5150 { 5151 u32 socket_id, aid_id, hbm_id; 5152 u32 fw_status; 5153 u32 boot_error; 5154 u64 reg_addr; 5155 5156 /* The pattern for smn addressing in other SOC could be different from 5157 * the one for aqua_vanjaram. We should revisit the code if the pattern 5158 * is changed. In such case, replace the aqua_vanjaram implementation 5159 * with more common helper */ 5160 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5161 aqua_vanjaram_encode_ext_smn_addressing(instance); 5162 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5163 5164 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5165 aqua_vanjaram_encode_ext_smn_addressing(instance); 5166 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5167 5168 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5169 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5170 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5171 5172 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5173 dev_info(adev->dev, 5174 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5175 socket_id, aid_id, hbm_id, fw_status); 5176 5177 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5178 dev_info(adev->dev, 5179 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5180 socket_id, aid_id, fw_status); 5181 5182 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5183 dev_info(adev->dev, 5184 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5185 socket_id, aid_id, fw_status); 5186 5187 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5188 dev_info(adev->dev, 5189 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5190 socket_id, aid_id, fw_status); 5191 5192 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5193 dev_info(adev->dev, 5194 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5195 socket_id, aid_id, fw_status); 5196 5197 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5198 dev_info(adev->dev, 5199 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5200 socket_id, aid_id, fw_status); 5201 5202 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5203 dev_info(adev->dev, 5204 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5205 socket_id, aid_id, hbm_id, fw_status); 5206 5207 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5208 dev_info(adev->dev, 5209 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5210 socket_id, aid_id, hbm_id, fw_status); 5211 5212 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5213 dev_info(adev->dev, 5214 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5215 socket_id, aid_id, fw_status); 5216 5217 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5218 dev_info(adev->dev, 5219 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5220 socket_id, aid_id, fw_status); 5221 } 5222 5223 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5224 u32 instance) 5225 { 5226 u64 reg_addr; 5227 u32 reg_data; 5228 int retry_loop; 5229 5230 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5231 aqua_vanjaram_encode_ext_smn_addressing(instance); 5232 5233 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5234 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5235 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5236 return false; 5237 else 5238 msleep(1); 5239 } 5240 5241 return true; 5242 } 5243 5244 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5245 { 5246 u32 i; 5247 5248 for (i = 0; i < num_instances; i++) { 5249 if (amdgpu_ras_boot_error_detected(adev, i)) 5250 amdgpu_ras_boot_time_error_reporting(adev, i); 5251 } 5252 } 5253 5254 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5255 { 5256 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5257 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5258 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5259 int ret = 0; 5260 5261 mutex_lock(&con->page_rsv_lock); 5262 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5263 if (ret == -ENOENT) 5264 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5265 mutex_unlock(&con->page_rsv_lock); 5266 5267 return ret; 5268 } 5269 5270 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5271 const char *fmt, ...) 5272 { 5273 struct va_format vaf; 5274 va_list args; 5275 5276 va_start(args, fmt); 5277 vaf.fmt = fmt; 5278 vaf.va = &args; 5279 5280 if (RAS_EVENT_ID_IS_VALID(event_id)) 5281 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5282 else 5283 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5284 5285 va_end(args); 5286 } 5287 5288 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5289 { 5290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5291 5292 if (!con) 5293 return false; 5294 5295 return con->is_rma; 5296 } 5297