1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 /** 616 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 617 * 618 * Some boards contain an EEPROM which is used to persistently store a list of 619 * bad pages which experiences ECC errors in vram. This interface provides 620 * a way to reset the EEPROM, e.g., after testing error injection. 621 * 622 * Usage: 623 * 624 * .. code-block:: bash 625 * 626 * echo 1 > ../ras/ras_eeprom_reset 627 * 628 * will reset EEPROM table to 0 entries. 629 * 630 */ 631 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 632 const char __user *buf, 633 size_t size, loff_t *pos) 634 { 635 struct amdgpu_device *adev = 636 (struct amdgpu_device *)file_inode(f)->i_private; 637 int ret; 638 639 ret = amdgpu_ras_eeprom_reset_table( 640 &(amdgpu_ras_get_context(adev)->eeprom_control)); 641 642 if (!ret) { 643 /* Something was written to EEPROM. 644 */ 645 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 646 return size; 647 } else { 648 return ret; 649 } 650 } 651 652 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 653 .owner = THIS_MODULE, 654 .read = NULL, 655 .write = amdgpu_ras_debugfs_ctrl_write, 656 .llseek = default_llseek 657 }; 658 659 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_eeprom_write, 663 .llseek = default_llseek 664 }; 665 666 /** 667 * DOC: AMDGPU RAS sysfs Error Count Interface 668 * 669 * It allows the user to read the error count for each IP block on the gpu through 670 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 671 * 672 * It outputs the multiple lines which report the uncorrected (ue) and corrected 673 * (ce) error counts. 674 * 675 * The format of one line is below, 676 * 677 * [ce|ue]: count 678 * 679 * Example: 680 * 681 * .. code-block:: bash 682 * 683 * ue: 0 684 * ce: 1 685 * 686 */ 687 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 688 struct device_attribute *attr, char *buf) 689 { 690 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 691 struct ras_query_if info = { 692 .head = obj->head, 693 }; 694 695 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 696 return sysfs_emit(buf, "Query currently inaccessible\n"); 697 698 if (amdgpu_ras_query_error_status(obj->adev, &info)) 699 return -EINVAL; 700 701 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 702 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 703 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 704 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 705 } 706 707 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 708 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 709 "ce", info.ce_count, "de", info.de_count); 710 else 711 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 712 "ce", info.ce_count); 713 } 714 715 /* obj begin */ 716 717 #define get_obj(obj) do { (obj)->use++; } while (0) 718 #define alive_obj(obj) ((obj)->use) 719 720 static inline void put_obj(struct ras_manager *obj) 721 { 722 if (obj && (--obj->use == 0)) { 723 list_del(&obj->node); 724 amdgpu_ras_error_data_fini(&obj->err_data); 725 } 726 727 if (obj && (obj->use < 0)) 728 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 729 } 730 731 /* make one obj and return it. */ 732 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 733 struct ras_common_if *head) 734 { 735 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 736 struct ras_manager *obj; 737 738 if (!adev->ras_enabled || !con) 739 return NULL; 740 741 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 742 return NULL; 743 744 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 745 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 746 return NULL; 747 748 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 749 } else 750 obj = &con->objs[head->block]; 751 752 /* already exist. return obj? */ 753 if (alive_obj(obj)) 754 return NULL; 755 756 if (amdgpu_ras_error_data_init(&obj->err_data)) 757 return NULL; 758 759 obj->head = *head; 760 obj->adev = adev; 761 list_add(&obj->node, &con->head); 762 get_obj(obj); 763 764 return obj; 765 } 766 767 /* return an obj equal to head, or the first when head is NULL */ 768 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 769 struct ras_common_if *head) 770 { 771 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 772 struct ras_manager *obj; 773 int i; 774 775 if (!adev->ras_enabled || !con) 776 return NULL; 777 778 if (head) { 779 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 780 return NULL; 781 782 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 783 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 784 return NULL; 785 786 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 787 } else 788 obj = &con->objs[head->block]; 789 790 if (alive_obj(obj)) 791 return obj; 792 } else { 793 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 794 obj = &con->objs[i]; 795 if (alive_obj(obj)) 796 return obj; 797 } 798 } 799 800 return NULL; 801 } 802 /* obj end */ 803 804 /* feature ctl begin */ 805 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 806 struct ras_common_if *head) 807 { 808 return adev->ras_hw_enabled & BIT(head->block); 809 } 810 811 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 812 struct ras_common_if *head) 813 { 814 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 815 816 return con->features & BIT(head->block); 817 } 818 819 /* 820 * if obj is not created, then create one. 821 * set feature enable flag. 822 */ 823 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 824 struct ras_common_if *head, int enable) 825 { 826 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 827 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 828 829 /* If hardware does not support ras, then do not create obj. 830 * But if hardware support ras, we can create the obj. 831 * Ras framework checks con->hw_supported to see if it need do 832 * corresponding initialization. 833 * IP checks con->support to see if it need disable ras. 834 */ 835 if (!amdgpu_ras_is_feature_allowed(adev, head)) 836 return 0; 837 838 if (enable) { 839 if (!obj) { 840 obj = amdgpu_ras_create_obj(adev, head); 841 if (!obj) 842 return -EINVAL; 843 } else { 844 /* In case we create obj somewhere else */ 845 get_obj(obj); 846 } 847 con->features |= BIT(head->block); 848 } else { 849 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 850 con->features &= ~BIT(head->block); 851 put_obj(obj); 852 } 853 } 854 855 return 0; 856 } 857 858 /* wrapper of psp_ras_enable_features */ 859 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 860 struct ras_common_if *head, bool enable) 861 { 862 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 863 union ta_ras_cmd_input *info; 864 int ret; 865 866 if (!con) 867 return -EINVAL; 868 869 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 870 /* For gfx ip, regardless of feature support status, */ 871 /* Force issue enable or disable ras feature commands */ 872 if (head->block != AMDGPU_RAS_BLOCK__GFX && 873 !amdgpu_ras_is_feature_allowed(adev, head)) 874 return 0; 875 876 /* Only enable gfx ras feature from host side */ 877 if (head->block == AMDGPU_RAS_BLOCK__GFX && 878 !amdgpu_sriov_vf(adev) && 879 !amdgpu_ras_intr_triggered()) { 880 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 881 if (!info) 882 return -ENOMEM; 883 884 if (!enable) { 885 info->disable_features = (struct ta_ras_disable_features_input) { 886 .block_id = amdgpu_ras_block_to_ta(head->block), 887 .error_type = amdgpu_ras_error_to_ta(head->type), 888 }; 889 } else { 890 info->enable_features = (struct ta_ras_enable_features_input) { 891 .block_id = amdgpu_ras_block_to_ta(head->block), 892 .error_type = amdgpu_ras_error_to_ta(head->type), 893 }; 894 } 895 896 ret = psp_ras_enable_features(&adev->psp, info, enable); 897 if (ret) { 898 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 899 enable ? "enable":"disable", 900 get_ras_block_str(head), 901 amdgpu_ras_is_poison_mode_supported(adev), ret); 902 kfree(info); 903 return ret; 904 } 905 906 kfree(info); 907 } 908 909 /* setup the obj */ 910 __amdgpu_ras_feature_enable(adev, head, enable); 911 912 return 0; 913 } 914 915 /* Only used in device probe stage and called only once. */ 916 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 917 struct ras_common_if *head, bool enable) 918 { 919 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 920 int ret; 921 922 if (!con) 923 return -EINVAL; 924 925 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 926 if (enable) { 927 /* There is no harm to issue a ras TA cmd regardless of 928 * the currecnt ras state. 929 * If current state == target state, it will do nothing 930 * But sometimes it requests driver to reset and repost 931 * with error code -EAGAIN. 932 */ 933 ret = amdgpu_ras_feature_enable(adev, head, 1); 934 /* With old ras TA, we might fail to enable ras. 935 * Log it and just setup the object. 936 * TODO need remove this WA in the future. 937 */ 938 if (ret == -EINVAL) { 939 ret = __amdgpu_ras_feature_enable(adev, head, 1); 940 if (!ret) 941 dev_info(adev->dev, 942 "RAS INFO: %s setup object\n", 943 get_ras_block_str(head)); 944 } 945 } else { 946 /* setup the object then issue a ras TA disable cmd.*/ 947 ret = __amdgpu_ras_feature_enable(adev, head, 1); 948 if (ret) 949 return ret; 950 951 /* gfx block ras disable cmd must send to ras-ta */ 952 if (head->block == AMDGPU_RAS_BLOCK__GFX) 953 con->features |= BIT(head->block); 954 955 ret = amdgpu_ras_feature_enable(adev, head, 0); 956 957 /* clean gfx block ras features flag */ 958 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 959 con->features &= ~BIT(head->block); 960 } 961 } else 962 ret = amdgpu_ras_feature_enable(adev, head, enable); 963 964 return ret; 965 } 966 967 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 968 bool bypass) 969 { 970 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 971 struct ras_manager *obj, *tmp; 972 973 list_for_each_entry_safe(obj, tmp, &con->head, node) { 974 /* bypass psp. 975 * aka just release the obj and corresponding flags 976 */ 977 if (bypass) { 978 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 979 break; 980 } else { 981 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 982 break; 983 } 984 } 985 986 return con->features; 987 } 988 989 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 990 bool bypass) 991 { 992 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 993 int i; 994 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 995 996 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 997 struct ras_common_if head = { 998 .block = i, 999 .type = default_ras_type, 1000 .sub_block_index = 0, 1001 }; 1002 1003 if (i == AMDGPU_RAS_BLOCK__MCA) 1004 continue; 1005 1006 if (bypass) { 1007 /* 1008 * bypass psp. vbios enable ras for us. 1009 * so just create the obj 1010 */ 1011 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1012 break; 1013 } else { 1014 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1015 break; 1016 } 1017 } 1018 1019 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1020 struct ras_common_if head = { 1021 .block = AMDGPU_RAS_BLOCK__MCA, 1022 .type = default_ras_type, 1023 .sub_block_index = i, 1024 }; 1025 1026 if (bypass) { 1027 /* 1028 * bypass psp. vbios enable ras for us. 1029 * so just create the obj 1030 */ 1031 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1032 break; 1033 } else { 1034 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1035 break; 1036 } 1037 } 1038 1039 return con->features; 1040 } 1041 /* feature ctl end */ 1042 1043 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1044 enum amdgpu_ras_block block) 1045 { 1046 if (!block_obj) 1047 return -EINVAL; 1048 1049 if (block_obj->ras_comm.block == block) 1050 return 0; 1051 1052 return -EINVAL; 1053 } 1054 1055 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1056 enum amdgpu_ras_block block, uint32_t sub_block_index) 1057 { 1058 struct amdgpu_ras_block_list *node, *tmp; 1059 struct amdgpu_ras_block_object *obj; 1060 1061 if (block >= AMDGPU_RAS_BLOCK__LAST) 1062 return NULL; 1063 1064 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1065 if (!node->ras_obj) { 1066 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1067 continue; 1068 } 1069 1070 obj = node->ras_obj; 1071 if (obj->ras_block_match) { 1072 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1073 return obj; 1074 } else { 1075 if (amdgpu_ras_block_match_default(obj, block) == 0) 1076 return obj; 1077 } 1078 } 1079 1080 return NULL; 1081 } 1082 1083 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1084 { 1085 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1086 int ret = 0; 1087 1088 /* 1089 * choosing right query method according to 1090 * whether smu support query error information 1091 */ 1092 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1093 if (ret == -EOPNOTSUPP) { 1094 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1095 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1096 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1097 1098 /* umc query_ras_error_address is also responsible for clearing 1099 * error status 1100 */ 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1104 } else if (!ret) { 1105 if (adev->umc.ras && 1106 adev->umc.ras->ecc_info_query_ras_error_count) 1107 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1108 1109 if (adev->umc.ras && 1110 adev->umc.ras->ecc_info_query_ras_error_address) 1111 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1112 } 1113 } 1114 1115 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1116 struct ras_manager *ras_mgr, 1117 struct ras_err_data *err_data, 1118 struct ras_query_context *qctx, 1119 const char *blk_name, 1120 bool is_ue, 1121 bool is_de) 1122 { 1123 struct amdgpu_smuio_mcm_config_info *mcm_info; 1124 struct ras_err_node *err_node; 1125 struct ras_err_info *err_info; 1126 u64 event_id = qctx->evid.event_id; 1127 1128 if (is_ue) { 1129 for_each_ras_error(err_node, err_data) { 1130 err_info = &err_node->err_info; 1131 mcm_info = &err_info->mcm_info; 1132 if (err_info->ue_count) { 1133 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1134 "%lld new uncorrectable hardware errors detected in %s block\n", 1135 mcm_info->socket_id, 1136 mcm_info->die_id, 1137 err_info->ue_count, 1138 blk_name); 1139 } 1140 } 1141 1142 for_each_ras_error(err_node, &ras_mgr->err_data) { 1143 err_info = &err_node->err_info; 1144 mcm_info = &err_info->mcm_info; 1145 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1146 "%lld uncorrectable hardware errors detected in total in %s block\n", 1147 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1148 } 1149 1150 } else { 1151 if (is_de) { 1152 for_each_ras_error(err_node, err_data) { 1153 err_info = &err_node->err_info; 1154 mcm_info = &err_info->mcm_info; 1155 if (err_info->de_count) { 1156 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1157 "%lld new deferred hardware errors detected in %s block\n", 1158 mcm_info->socket_id, 1159 mcm_info->die_id, 1160 err_info->de_count, 1161 blk_name); 1162 } 1163 } 1164 1165 for_each_ras_error(err_node, &ras_mgr->err_data) { 1166 err_info = &err_node->err_info; 1167 mcm_info = &err_info->mcm_info; 1168 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1169 "%lld deferred hardware errors detected in total in %s block\n", 1170 mcm_info->socket_id, mcm_info->die_id, 1171 err_info->de_count, blk_name); 1172 } 1173 } else { 1174 if (adev->debug_disable_ce_logs) 1175 return; 1176 1177 for_each_ras_error(err_node, err_data) { 1178 err_info = &err_node->err_info; 1179 mcm_info = &err_info->mcm_info; 1180 if (err_info->ce_count) { 1181 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1182 "%lld new correctable hardware errors detected in %s block\n", 1183 mcm_info->socket_id, 1184 mcm_info->die_id, 1185 err_info->ce_count, 1186 blk_name); 1187 } 1188 } 1189 1190 for_each_ras_error(err_node, &ras_mgr->err_data) { 1191 err_info = &err_node->err_info; 1192 mcm_info = &err_info->mcm_info; 1193 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1194 "%lld correctable hardware errors detected in total in %s block\n", 1195 mcm_info->socket_id, mcm_info->die_id, 1196 err_info->ce_count, blk_name); 1197 } 1198 } 1199 } 1200 } 1201 1202 static inline bool err_data_has_source_info(struct ras_err_data *data) 1203 { 1204 return !list_empty(&data->err_node_list); 1205 } 1206 1207 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1208 struct ras_query_if *query_if, 1209 struct ras_err_data *err_data, 1210 struct ras_query_context *qctx) 1211 { 1212 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1213 const char *blk_name = get_ras_block_str(&query_if->head); 1214 u64 event_id = qctx->evid.event_id; 1215 1216 if (err_data->ce_count) { 1217 if (err_data_has_source_info(err_data)) { 1218 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1219 blk_name, false, false); 1220 } else if (!adev->aid_mask && 1221 adev->smuio.funcs && 1222 adev->smuio.funcs->get_socket_id && 1223 adev->smuio.funcs->get_die_id) { 1224 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1225 "%ld correctable hardware errors " 1226 "detected in %s block\n", 1227 adev->smuio.funcs->get_socket_id(adev), 1228 adev->smuio.funcs->get_die_id(adev), 1229 ras_mgr->err_data.ce_count, 1230 blk_name); 1231 } else { 1232 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 ras_mgr->err_data.ce_count, 1235 blk_name); 1236 } 1237 } 1238 1239 if (err_data->ue_count) { 1240 if (err_data_has_source_info(err_data)) { 1241 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1242 blk_name, true, false); 1243 } else if (!adev->aid_mask && 1244 adev->smuio.funcs && 1245 adev->smuio.funcs->get_socket_id && 1246 adev->smuio.funcs->get_die_id) { 1247 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1248 "%ld uncorrectable hardware errors " 1249 "detected in %s block\n", 1250 adev->smuio.funcs->get_socket_id(adev), 1251 adev->smuio.funcs->get_die_id(adev), 1252 ras_mgr->err_data.ue_count, 1253 blk_name); 1254 } else { 1255 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 ras_mgr->err_data.ue_count, 1258 blk_name); 1259 } 1260 } 1261 1262 if (err_data->de_count) { 1263 if (err_data_has_source_info(err_data)) { 1264 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1265 blk_name, false, true); 1266 } else if (!adev->aid_mask && 1267 adev->smuio.funcs && 1268 adev->smuio.funcs->get_socket_id && 1269 adev->smuio.funcs->get_die_id) { 1270 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1271 "%ld deferred hardware errors " 1272 "detected in %s block\n", 1273 adev->smuio.funcs->get_socket_id(adev), 1274 adev->smuio.funcs->get_die_id(adev), 1275 ras_mgr->err_data.de_count, 1276 blk_name); 1277 } else { 1278 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 ras_mgr->err_data.de_count, 1281 blk_name); 1282 } 1283 } 1284 } 1285 1286 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1287 struct ras_query_if *query_if, 1288 struct ras_err_data *err_data, 1289 struct ras_query_context *qctx) 1290 { 1291 unsigned long new_ue, new_ce, new_de; 1292 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1293 const char *blk_name = get_ras_block_str(&query_if->head); 1294 u64 event_id = qctx->evid.event_id; 1295 1296 new_ce = err_data->ce_count - obj->err_data.ce_count; 1297 new_ue = err_data->ue_count - obj->err_data.ue_count; 1298 new_de = err_data->de_count - obj->err_data.de_count; 1299 1300 if (new_ce) { 1301 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1302 "detected in %s block\n", 1303 new_ce, 1304 blk_name); 1305 } 1306 1307 if (new_ue) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1309 "detected in %s block\n", 1310 new_ue, 1311 blk_name); 1312 } 1313 1314 if (new_de) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1316 "detected in %s block\n", 1317 new_de, 1318 blk_name); 1319 } 1320 } 1321 1322 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1323 { 1324 struct ras_err_node *err_node; 1325 struct ras_err_info *err_info; 1326 1327 if (err_data_has_source_info(err_data)) { 1328 for_each_ras_error(err_node, err_data) { 1329 err_info = &err_node->err_info; 1330 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1331 &err_info->mcm_info, err_info->de_count); 1332 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1333 &err_info->mcm_info, err_info->ce_count); 1334 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1335 &err_info->mcm_info, err_info->ue_count); 1336 } 1337 } else { 1338 /* for legacy asic path which doesn't has error source info */ 1339 obj->err_data.ue_count += err_data->ue_count; 1340 obj->err_data.ce_count += err_data->ce_count; 1341 obj->err_data.de_count += err_data->de_count; 1342 } 1343 } 1344 1345 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1346 struct ras_err_data *err_data) 1347 { 1348 /* Host reports absolute counts */ 1349 obj->err_data.ue_count = err_data->ue_count; 1350 obj->err_data.ce_count = err_data->ce_count; 1351 obj->err_data.de_count = err_data->de_count; 1352 } 1353 1354 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1355 { 1356 struct ras_common_if head; 1357 1358 memset(&head, 0, sizeof(head)); 1359 head.block = blk; 1360 1361 return amdgpu_ras_find_obj(adev, &head); 1362 } 1363 1364 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1365 const struct aca_info *aca_info, void *data) 1366 { 1367 struct ras_manager *obj; 1368 1369 /* in resume phase, no need to create aca fs node */ 1370 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1371 return 0; 1372 1373 obj = get_ras_manager(adev, blk); 1374 if (!obj) 1375 return -EINVAL; 1376 1377 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1378 } 1379 1380 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1381 { 1382 struct ras_manager *obj; 1383 1384 obj = get_ras_manager(adev, blk); 1385 if (!obj) 1386 return -EINVAL; 1387 1388 amdgpu_aca_remove_handle(&obj->aca_handle); 1389 1390 return 0; 1391 } 1392 1393 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1394 enum aca_error_type type, struct ras_err_data *err_data, 1395 struct ras_query_context *qctx) 1396 { 1397 struct ras_manager *obj; 1398 1399 obj = get_ras_manager(adev, blk); 1400 if (!obj) 1401 return -EINVAL; 1402 1403 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1404 } 1405 1406 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1407 struct aca_handle *handle, char *buf, void *data) 1408 { 1409 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1410 struct ras_query_if info = { 1411 .head = obj->head, 1412 }; 1413 1414 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1415 return sysfs_emit(buf, "Query currently inaccessible\n"); 1416 1417 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1418 return -EINVAL; 1419 1420 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1421 "ce", info.ce_count, "de", info.de_count); 1422 } 1423 1424 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1425 struct ras_query_if *info, 1426 struct ras_err_data *err_data, 1427 struct ras_query_context *qctx, 1428 unsigned int error_query_mode) 1429 { 1430 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1431 struct amdgpu_ras_block_object *block_obj = NULL; 1432 int ret; 1433 1434 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1435 return -EINVAL; 1436 1437 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1438 return -EINVAL; 1439 1440 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1441 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1442 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1443 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1444 amdgpu_ras_get_ecc_info(adev, err_data); 1445 } else { 1446 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1447 if (!block_obj || !block_obj->hw_ops) { 1448 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1449 get_ras_block_str(&info->head)); 1450 return -EINVAL; 1451 } 1452 1453 if (block_obj->hw_ops->query_ras_error_count) 1454 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1455 1456 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1457 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1458 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1459 if (block_obj->hw_ops->query_ras_error_status) 1460 block_obj->hw_ops->query_ras_error_status(adev); 1461 } 1462 } 1463 } else { 1464 if (amdgpu_aca_is_enabled(adev)) { 1465 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1466 if (ret) 1467 return ret; 1468 1469 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1470 if (ret) 1471 return ret; 1472 1473 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1474 if (ret) 1475 return ret; 1476 } else { 1477 /* FIXME: add code to check return value later */ 1478 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1479 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1480 } 1481 } 1482 1483 return 0; 1484 } 1485 1486 /* query/inject/cure begin */ 1487 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1488 struct ras_query_if *info, 1489 enum ras_event_type type) 1490 { 1491 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1492 struct ras_err_data err_data; 1493 struct ras_query_context qctx; 1494 unsigned int error_query_mode; 1495 int ret; 1496 1497 if (!obj) 1498 return -EINVAL; 1499 1500 ret = amdgpu_ras_error_data_init(&err_data); 1501 if (ret) 1502 return ret; 1503 1504 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1505 return -EINVAL; 1506 1507 memset(&qctx, 0, sizeof(qctx)); 1508 qctx.evid.type = type; 1509 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1510 1511 if (!down_read_trylock(&adev->reset_domain->sem)) { 1512 ret = -EIO; 1513 goto out_fini_err_data; 1514 } 1515 1516 ret = amdgpu_ras_query_error_status_helper(adev, info, 1517 &err_data, 1518 &qctx, 1519 error_query_mode); 1520 up_read(&adev->reset_domain->sem); 1521 if (ret) 1522 goto out_fini_err_data; 1523 1524 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1525 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1526 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1527 } else { 1528 /* Host provides absolute error counts. First generate the report 1529 * using the previous VF internal count against new host count. 1530 * Then Update VF internal count. 1531 */ 1532 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1533 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1534 } 1535 1536 info->ue_count = obj->err_data.ue_count; 1537 info->ce_count = obj->err_data.ce_count; 1538 info->de_count = obj->err_data.de_count; 1539 1540 out_fini_err_data: 1541 amdgpu_ras_error_data_fini(&err_data); 1542 1543 return ret; 1544 } 1545 1546 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1547 { 1548 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1549 } 1550 1551 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1552 enum amdgpu_ras_block block) 1553 { 1554 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1555 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1556 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1557 1558 if (!block_obj || !block_obj->hw_ops) { 1559 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1560 ras_block_str(block)); 1561 return -EOPNOTSUPP; 1562 } 1563 1564 if (!amdgpu_ras_is_supported(adev, block) || 1565 !amdgpu_ras_get_aca_debug_mode(adev)) 1566 return -EOPNOTSUPP; 1567 1568 if (amdgpu_sriov_vf(adev)) 1569 return -EOPNOTSUPP; 1570 1571 /* skip ras error reset in gpu reset */ 1572 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1573 ((smu_funcs && smu_funcs->set_debug_mode) || 1574 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1575 return -EOPNOTSUPP; 1576 1577 if (block_obj->hw_ops->reset_ras_error_count) 1578 block_obj->hw_ops->reset_ras_error_count(adev); 1579 1580 return 0; 1581 } 1582 1583 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1584 enum amdgpu_ras_block block) 1585 { 1586 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1587 1588 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1589 return 0; 1590 1591 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1592 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1593 if (block_obj->hw_ops->reset_ras_error_status) 1594 block_obj->hw_ops->reset_ras_error_status(adev); 1595 } 1596 1597 return 0; 1598 } 1599 1600 /* wrapper of psp_ras_trigger_error */ 1601 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1602 struct ras_inject_if *info) 1603 { 1604 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1605 struct ta_ras_trigger_error_input block_info = { 1606 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1607 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1608 .sub_block_index = info->head.sub_block_index, 1609 .address = info->address, 1610 .value = info->value, 1611 }; 1612 int ret = -EINVAL; 1613 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1614 info->head.block, 1615 info->head.sub_block_index); 1616 1617 /* inject on guest isn't allowed, return success directly */ 1618 if (amdgpu_sriov_vf(adev)) 1619 return 0; 1620 1621 if (!obj) 1622 return -EINVAL; 1623 1624 if (!block_obj || !block_obj->hw_ops) { 1625 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1626 get_ras_block_str(&info->head)); 1627 return -EINVAL; 1628 } 1629 1630 /* Calculate XGMI relative offset */ 1631 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1632 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1633 block_info.address = 1634 amdgpu_xgmi_get_relative_phy_addr(adev, 1635 block_info.address); 1636 } 1637 1638 if (block_obj->hw_ops->ras_error_inject) { 1639 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1640 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1641 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1642 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1643 info->instance_mask); 1644 } else { 1645 /* default path */ 1646 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1647 } 1648 1649 if (ret) 1650 dev_err(adev->dev, "ras inject %s failed %d\n", 1651 get_ras_block_str(&info->head), ret); 1652 1653 return ret; 1654 } 1655 1656 /** 1657 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1658 * @adev: pointer to AMD GPU device 1659 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1660 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1661 * @query_info: pointer to ras_query_if 1662 * 1663 * Return 0 for query success or do nothing, otherwise return an error 1664 * on failures 1665 */ 1666 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1667 unsigned long *ce_count, 1668 unsigned long *ue_count, 1669 struct ras_query_if *query_info) 1670 { 1671 int ret; 1672 1673 if (!query_info) 1674 /* do nothing if query_info is not specified */ 1675 return 0; 1676 1677 ret = amdgpu_ras_query_error_status(adev, query_info); 1678 if (ret) 1679 return ret; 1680 1681 *ce_count += query_info->ce_count; 1682 *ue_count += query_info->ue_count; 1683 1684 /* some hardware/IP supports read to clear 1685 * no need to explictly reset the err status after the query call */ 1686 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1687 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1688 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1689 dev_warn(adev->dev, 1690 "Failed to reset error counter and error status\n"); 1691 } 1692 1693 return 0; 1694 } 1695 1696 /** 1697 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1698 * @adev: pointer to AMD GPU device 1699 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1700 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1701 * errors. 1702 * @query_info: pointer to ras_query_if if the query request is only for 1703 * specific ip block; if info is NULL, then the qurey request is for 1704 * all the ip blocks that support query ras error counters/status 1705 * 1706 * If set, @ce_count or @ue_count, count and return the corresponding 1707 * error counts in those integer pointers. Return 0 if the device 1708 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1709 */ 1710 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1711 unsigned long *ce_count, 1712 unsigned long *ue_count, 1713 struct ras_query_if *query_info) 1714 { 1715 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1716 struct ras_manager *obj; 1717 unsigned long ce, ue; 1718 int ret; 1719 1720 if (!adev->ras_enabled || !con) 1721 return -EOPNOTSUPP; 1722 1723 /* Don't count since no reporting. 1724 */ 1725 if (!ce_count && !ue_count) 1726 return 0; 1727 1728 ce = 0; 1729 ue = 0; 1730 if (!query_info) { 1731 /* query all the ip blocks that support ras query interface */ 1732 list_for_each_entry(obj, &con->head, node) { 1733 struct ras_query_if info = { 1734 .head = obj->head, 1735 }; 1736 1737 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1738 } 1739 } else { 1740 /* query specific ip block */ 1741 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1742 } 1743 1744 if (ret) 1745 return ret; 1746 1747 if (ce_count) 1748 *ce_count = ce; 1749 1750 if (ue_count) 1751 *ue_count = ue; 1752 1753 return 0; 1754 } 1755 /* query/inject/cure end */ 1756 1757 1758 /* sysfs begin */ 1759 1760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1761 struct ras_badpage **bps, unsigned int *count); 1762 1763 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1764 { 1765 switch (flags) { 1766 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1767 return "R"; 1768 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1769 return "P"; 1770 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1771 default: 1772 return "F"; 1773 } 1774 } 1775 1776 /** 1777 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1778 * 1779 * It allows user to read the bad pages of vram on the gpu through 1780 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1781 * 1782 * It outputs multiple lines, and each line stands for one gpu page. 1783 * 1784 * The format of one line is below, 1785 * gpu pfn : gpu page size : flags 1786 * 1787 * gpu pfn and gpu page size are printed in hex format. 1788 * flags can be one of below character, 1789 * 1790 * R: reserved, this gpu page is reserved and not able to use. 1791 * 1792 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1793 * in next window of page_reserve. 1794 * 1795 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1796 * 1797 * Examples: 1798 * 1799 * .. code-block:: bash 1800 * 1801 * 0x00000001 : 0x00001000 : R 1802 * 0x00000002 : 0x00001000 : P 1803 * 1804 */ 1805 1806 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1807 struct kobject *kobj, const struct bin_attribute *attr, 1808 char *buf, loff_t ppos, size_t count) 1809 { 1810 struct amdgpu_ras *con = 1811 container_of(attr, struct amdgpu_ras, badpages_attr); 1812 struct amdgpu_device *adev = con->adev; 1813 const unsigned int element_size = 1814 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1815 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1816 unsigned int end = div64_ul(ppos + count - 1, element_size); 1817 ssize_t s = 0; 1818 struct ras_badpage *bps = NULL; 1819 unsigned int bps_count = 0; 1820 1821 memset(buf, 0, count); 1822 1823 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1824 return 0; 1825 1826 for (; start < end && start < bps_count; start++) 1827 s += scnprintf(&buf[s], element_size + 1, 1828 "0x%08x : 0x%08x : %1s\n", 1829 bps[start].bp, 1830 bps[start].size, 1831 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1832 1833 kfree(bps); 1834 1835 return s; 1836 } 1837 1838 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1839 struct device_attribute *attr, char *buf) 1840 { 1841 struct amdgpu_ras *con = 1842 container_of(attr, struct amdgpu_ras, features_attr); 1843 1844 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1845 } 1846 1847 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1848 struct device_attribute *attr, char *buf) 1849 { 1850 struct amdgpu_ras *con = 1851 container_of(attr, struct amdgpu_ras, version_attr); 1852 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version); 1853 } 1854 1855 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1856 struct device_attribute *attr, char *buf) 1857 { 1858 struct amdgpu_ras *con = 1859 container_of(attr, struct amdgpu_ras, schema_attr); 1860 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1861 } 1862 1863 static struct { 1864 enum ras_event_type type; 1865 const char *name; 1866 } dump_event[] = { 1867 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 1868 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 1869 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 1870 }; 1871 1872 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 1873 struct device_attribute *attr, char *buf) 1874 { 1875 struct amdgpu_ras *con = 1876 container_of(attr, struct amdgpu_ras, event_state_attr); 1877 struct ras_event_manager *event_mgr = con->event_mgr; 1878 struct ras_event_state *event_state; 1879 int i, size = 0; 1880 1881 if (!event_mgr) 1882 return -EINVAL; 1883 1884 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 1885 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 1886 event_state = &event_mgr->event_state[dump_event[i].type]; 1887 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 1888 dump_event[i].name, 1889 atomic64_read(&event_state->count), 1890 event_state->last_seqno); 1891 } 1892 1893 return (ssize_t)size; 1894 } 1895 1896 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1897 { 1898 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1899 1900 if (adev->dev->kobj.sd) 1901 sysfs_remove_file_from_group(&adev->dev->kobj, 1902 &con->badpages_attr.attr, 1903 RAS_FS_NAME); 1904 } 1905 1906 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 1907 { 1908 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1909 struct attribute *attrs[] = { 1910 &con->features_attr.attr, 1911 &con->version_attr.attr, 1912 &con->schema_attr.attr, 1913 &con->event_state_attr.attr, 1914 NULL 1915 }; 1916 struct attribute_group group = { 1917 .name = RAS_FS_NAME, 1918 .attrs = attrs, 1919 }; 1920 1921 if (adev->dev->kobj.sd) 1922 sysfs_remove_group(&adev->dev->kobj, &group); 1923 1924 return 0; 1925 } 1926 1927 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1928 struct ras_common_if *head) 1929 { 1930 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1931 1932 if (amdgpu_aca_is_enabled(adev)) 1933 return 0; 1934 1935 if (!obj || obj->attr_inuse) 1936 return -EINVAL; 1937 1938 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 1939 return 0; 1940 1941 get_obj(obj); 1942 1943 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1944 "%s_err_count", head->name); 1945 1946 obj->sysfs_attr = (struct device_attribute){ 1947 .attr = { 1948 .name = obj->fs_data.sysfs_name, 1949 .mode = S_IRUGO, 1950 }, 1951 .show = amdgpu_ras_sysfs_read, 1952 }; 1953 sysfs_attr_init(&obj->sysfs_attr.attr); 1954 1955 if (sysfs_add_file_to_group(&adev->dev->kobj, 1956 &obj->sysfs_attr.attr, 1957 RAS_FS_NAME)) { 1958 put_obj(obj); 1959 return -EINVAL; 1960 } 1961 1962 obj->attr_inuse = 1; 1963 1964 return 0; 1965 } 1966 1967 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1968 struct ras_common_if *head) 1969 { 1970 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1971 1972 if (amdgpu_aca_is_enabled(adev)) 1973 return 0; 1974 1975 if (!obj || !obj->attr_inuse) 1976 return -EINVAL; 1977 1978 if (adev->dev->kobj.sd) 1979 sysfs_remove_file_from_group(&adev->dev->kobj, 1980 &obj->sysfs_attr.attr, 1981 RAS_FS_NAME); 1982 obj->attr_inuse = 0; 1983 put_obj(obj); 1984 1985 return 0; 1986 } 1987 1988 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1989 { 1990 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1991 struct ras_manager *obj, *tmp; 1992 1993 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1994 amdgpu_ras_sysfs_remove(adev, &obj->head); 1995 } 1996 1997 if (amdgpu_bad_page_threshold != 0) 1998 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1999 2000 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2001 2002 return 0; 2003 } 2004 /* sysfs end */ 2005 2006 /** 2007 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2008 * 2009 * Normally when there is an uncorrectable error, the driver will reset 2010 * the GPU to recover. However, in the event of an unrecoverable error, 2011 * the driver provides an interface to reboot the system automatically 2012 * in that event. 2013 * 2014 * The following file in debugfs provides that interface: 2015 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2016 * 2017 * Usage: 2018 * 2019 * .. code-block:: bash 2020 * 2021 * echo true > .../ras/auto_reboot 2022 * 2023 */ 2024 /* debugfs begin */ 2025 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2026 { 2027 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2028 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2029 struct drm_minor *minor = adev_to_drm(adev)->primary; 2030 struct dentry *dir; 2031 2032 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2033 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2034 &amdgpu_ras_debugfs_ctrl_ops); 2035 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2036 &amdgpu_ras_debugfs_eeprom_ops); 2037 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2038 &con->bad_page_cnt_threshold); 2039 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2040 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2041 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2042 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2043 &amdgpu_ras_debugfs_eeprom_size_ops); 2044 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2045 S_IRUGO, dir, adev, 2046 &amdgpu_ras_debugfs_eeprom_table_ops); 2047 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2048 2049 /* 2050 * After one uncorrectable error happens, usually GPU recovery will 2051 * be scheduled. But due to the known problem in GPU recovery failing 2052 * to bring GPU back, below interface provides one direct way to 2053 * user to reboot system automatically in such case within 2054 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2055 * will never be called. 2056 */ 2057 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2058 2059 /* 2060 * User could set this not to clean up hardware's error count register 2061 * of RAS IPs during ras recovery. 2062 */ 2063 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2064 &con->disable_ras_err_cnt_harvest); 2065 return dir; 2066 } 2067 2068 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2069 struct ras_fs_if *head, 2070 struct dentry *dir) 2071 { 2072 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2073 2074 if (!obj || !dir) 2075 return; 2076 2077 get_obj(obj); 2078 2079 memcpy(obj->fs_data.debugfs_name, 2080 head->debugfs_name, 2081 sizeof(obj->fs_data.debugfs_name)); 2082 2083 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2084 obj, &amdgpu_ras_debugfs_ops); 2085 } 2086 2087 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2088 { 2089 bool ret; 2090 2091 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2092 case IP_VERSION(13, 0, 6): 2093 case IP_VERSION(13, 0, 12): 2094 case IP_VERSION(13, 0, 14): 2095 ret = true; 2096 break; 2097 default: 2098 ret = false; 2099 break; 2100 } 2101 2102 return ret; 2103 } 2104 2105 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2106 { 2107 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2108 struct dentry *dir; 2109 struct ras_manager *obj; 2110 struct ras_fs_if fs_info; 2111 2112 /* 2113 * it won't be called in resume path, no need to check 2114 * suspend and gpu reset status 2115 */ 2116 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2117 return; 2118 2119 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2120 2121 list_for_each_entry(obj, &con->head, node) { 2122 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2123 (obj->attr_inuse == 1)) { 2124 sprintf(fs_info.debugfs_name, "%s_err_inject", 2125 get_ras_block_str(&obj->head)); 2126 fs_info.head = obj->head; 2127 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2128 } 2129 } 2130 2131 if (amdgpu_ras_aca_is_supported(adev)) { 2132 if (amdgpu_aca_is_enabled(adev)) 2133 amdgpu_aca_smu_debugfs_init(adev, dir); 2134 else 2135 amdgpu_mca_smu_debugfs_init(adev, dir); 2136 } 2137 } 2138 2139 /* debugfs end */ 2140 2141 /* ras fs */ 2142 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2143 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2144 static DEVICE_ATTR(features, S_IRUGO, 2145 amdgpu_ras_sysfs_features_read, NULL); 2146 static DEVICE_ATTR(version, 0444, 2147 amdgpu_ras_sysfs_version_show, NULL); 2148 static DEVICE_ATTR(schema, 0444, 2149 amdgpu_ras_sysfs_schema_show, NULL); 2150 static DEVICE_ATTR(event_state, 0444, 2151 amdgpu_ras_sysfs_event_state_show, NULL); 2152 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2153 { 2154 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2155 struct attribute_group group = { 2156 .name = RAS_FS_NAME, 2157 }; 2158 struct attribute *attrs[] = { 2159 &con->features_attr.attr, 2160 &con->version_attr.attr, 2161 &con->schema_attr.attr, 2162 &con->event_state_attr.attr, 2163 NULL 2164 }; 2165 const struct bin_attribute *bin_attrs[] = { 2166 NULL, 2167 NULL, 2168 }; 2169 int r; 2170 2171 group.attrs = attrs; 2172 2173 /* add features entry */ 2174 con->features_attr = dev_attr_features; 2175 sysfs_attr_init(attrs[0]); 2176 2177 /* add version entry */ 2178 con->version_attr = dev_attr_version; 2179 sysfs_attr_init(attrs[1]); 2180 2181 /* add schema entry */ 2182 con->schema_attr = dev_attr_schema; 2183 sysfs_attr_init(attrs[2]); 2184 2185 /* add event_state entry */ 2186 con->event_state_attr = dev_attr_event_state; 2187 sysfs_attr_init(attrs[3]); 2188 2189 if (amdgpu_bad_page_threshold != 0) { 2190 /* add bad_page_features entry */ 2191 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2192 sysfs_bin_attr_init(&con->badpages_attr); 2193 bin_attrs[0] = &con->badpages_attr; 2194 group.bin_attrs = bin_attrs; 2195 } 2196 2197 r = sysfs_create_group(&adev->dev->kobj, &group); 2198 if (r) 2199 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2200 2201 return 0; 2202 } 2203 2204 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2205 { 2206 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2207 struct ras_manager *con_obj, *ip_obj, *tmp; 2208 2209 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2210 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2211 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2212 if (ip_obj) 2213 put_obj(ip_obj); 2214 } 2215 } 2216 2217 amdgpu_ras_sysfs_remove_all(adev); 2218 return 0; 2219 } 2220 /* ras fs end */ 2221 2222 /* ih begin */ 2223 2224 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2225 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2226 * register to check whether the interrupt is triggered or not, and properly 2227 * ack the interrupt if it is there 2228 */ 2229 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2230 { 2231 /* Fatal error events are handled on host side */ 2232 if (amdgpu_sriov_vf(adev)) 2233 return; 2234 /* 2235 * If the current interrupt is caused by a non-fatal RAS error, skip 2236 * check for fatal error. For fatal errors, FED status of all devices 2237 * in XGMI hive gets set when the first device gets fatal error 2238 * interrupt. The error gets propagated to other devices as well, so 2239 * make sure to ack the interrupt regardless of FED status. 2240 */ 2241 if (!amdgpu_ras_get_fed_status(adev) && 2242 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2243 return; 2244 2245 if (amdgpu_uniras_enabled(adev)) { 2246 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2247 return; 2248 } 2249 2250 if (adev->nbio.ras && 2251 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2252 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2253 2254 if (adev->nbio.ras && 2255 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2256 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2257 } 2258 2259 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2260 struct amdgpu_iv_entry *entry) 2261 { 2262 bool poison_stat = false; 2263 struct amdgpu_device *adev = obj->adev; 2264 struct amdgpu_ras_block_object *block_obj = 2265 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2266 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2267 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2268 u64 event_id; 2269 int ret; 2270 2271 if (!block_obj || !con) 2272 return; 2273 2274 ret = amdgpu_ras_mark_ras_event(adev, type); 2275 if (ret) 2276 return; 2277 2278 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2279 /* both query_poison_status and handle_poison_consumption are optional, 2280 * but at least one of them should be implemented if we need poison 2281 * consumption handler 2282 */ 2283 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2284 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2285 if (!poison_stat) { 2286 /* Not poison consumption interrupt, no need to handle it */ 2287 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2288 block_obj->ras_comm.name); 2289 2290 return; 2291 } 2292 } 2293 2294 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2295 2296 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2297 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2298 2299 /* gpu reset is fallback for failed and default cases. 2300 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2301 */ 2302 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2303 event_id = amdgpu_ras_acquire_event_id(adev, type); 2304 RAS_EVENT_LOG(adev, event_id, 2305 "GPU reset for %s RAS poison consumption is issued!\n", 2306 block_obj->ras_comm.name); 2307 amdgpu_ras_reset_gpu(adev); 2308 } 2309 2310 if (!poison_stat) 2311 amdgpu_gfx_poison_consumption_handler(adev, entry); 2312 } 2313 2314 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2315 struct amdgpu_iv_entry *entry) 2316 { 2317 struct amdgpu_device *adev = obj->adev; 2318 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2319 u64 event_id; 2320 int ret; 2321 2322 ret = amdgpu_ras_mark_ras_event(adev, type); 2323 if (ret) 2324 return; 2325 2326 event_id = amdgpu_ras_acquire_event_id(adev, type); 2327 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2328 2329 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2330 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2331 2332 atomic_inc(&con->page_retirement_req_cnt); 2333 atomic_inc(&con->poison_creation_count); 2334 2335 wake_up(&con->page_retirement_wq); 2336 } 2337 } 2338 2339 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2340 struct amdgpu_iv_entry *entry) 2341 { 2342 struct ras_ih_data *data = &obj->ih_data; 2343 struct ras_err_data err_data; 2344 int ret; 2345 2346 if (!data->cb) 2347 return; 2348 2349 ret = amdgpu_ras_error_data_init(&err_data); 2350 if (ret) 2351 return; 2352 2353 /* Let IP handle its data, maybe we need get the output 2354 * from the callback to update the error type/count, etc 2355 */ 2356 amdgpu_ras_set_fed(obj->adev, true); 2357 ret = data->cb(obj->adev, &err_data, entry); 2358 /* ue will trigger an interrupt, and in that case 2359 * we need do a reset to recovery the whole system. 2360 * But leave IP do that recovery, here we just dispatch 2361 * the error. 2362 */ 2363 if (ret == AMDGPU_RAS_SUCCESS) { 2364 /* these counts could be left as 0 if 2365 * some blocks do not count error number 2366 */ 2367 obj->err_data.ue_count += err_data.ue_count; 2368 obj->err_data.ce_count += err_data.ce_count; 2369 obj->err_data.de_count += err_data.de_count; 2370 } 2371 2372 amdgpu_ras_error_data_fini(&err_data); 2373 } 2374 2375 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2376 { 2377 struct ras_ih_data *data = &obj->ih_data; 2378 struct amdgpu_iv_entry entry; 2379 2380 while (data->rptr != data->wptr) { 2381 rmb(); 2382 memcpy(&entry, &data->ring[data->rptr], 2383 data->element_size); 2384 2385 wmb(); 2386 data->rptr = (data->aligned_element_size + 2387 data->rptr) % data->ring_size; 2388 2389 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2390 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2391 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2392 else 2393 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2394 } else { 2395 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2396 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2397 else 2398 dev_warn(obj->adev->dev, 2399 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2400 } 2401 } 2402 } 2403 2404 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2405 { 2406 struct ras_ih_data *data = 2407 container_of(work, struct ras_ih_data, ih_work); 2408 struct ras_manager *obj = 2409 container_of(data, struct ras_manager, ih_data); 2410 2411 amdgpu_ras_interrupt_handler(obj); 2412 } 2413 2414 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2415 struct ras_dispatch_if *info) 2416 { 2417 struct ras_manager *obj; 2418 struct ras_ih_data *data; 2419 2420 if (amdgpu_uniras_enabled(adev)) { 2421 struct ras_ih_info ih_info; 2422 2423 memset(&ih_info, 0, sizeof(ih_info)); 2424 ih_info.block = info->head.block; 2425 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2426 2427 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2428 } 2429 2430 obj = amdgpu_ras_find_obj(adev, &info->head); 2431 if (!obj) 2432 return -EINVAL; 2433 2434 data = &obj->ih_data; 2435 2436 if (data->inuse == 0) 2437 return 0; 2438 2439 /* Might be overflow... */ 2440 memcpy(&data->ring[data->wptr], info->entry, 2441 data->element_size); 2442 2443 wmb(); 2444 data->wptr = (data->aligned_element_size + 2445 data->wptr) % data->ring_size; 2446 2447 schedule_work(&data->ih_work); 2448 2449 return 0; 2450 } 2451 2452 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2453 struct ras_common_if *head) 2454 { 2455 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2456 struct ras_ih_data *data; 2457 2458 if (!obj) 2459 return -EINVAL; 2460 2461 data = &obj->ih_data; 2462 if (data->inuse == 0) 2463 return 0; 2464 2465 cancel_work_sync(&data->ih_work); 2466 2467 kfree(data->ring); 2468 memset(data, 0, sizeof(*data)); 2469 put_obj(obj); 2470 2471 return 0; 2472 } 2473 2474 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2475 struct ras_common_if *head) 2476 { 2477 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2478 struct ras_ih_data *data; 2479 struct amdgpu_ras_block_object *ras_obj; 2480 2481 if (!obj) { 2482 /* in case we registe the IH before enable ras feature */ 2483 obj = amdgpu_ras_create_obj(adev, head); 2484 if (!obj) 2485 return -EINVAL; 2486 } else 2487 get_obj(obj); 2488 2489 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2490 2491 data = &obj->ih_data; 2492 /* add the callback.etc */ 2493 *data = (struct ras_ih_data) { 2494 .inuse = 0, 2495 .cb = ras_obj->ras_cb, 2496 .element_size = sizeof(struct amdgpu_iv_entry), 2497 .rptr = 0, 2498 .wptr = 0, 2499 }; 2500 2501 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2502 2503 data->aligned_element_size = ALIGN(data->element_size, 8); 2504 /* the ring can store 64 iv entries. */ 2505 data->ring_size = 64 * data->aligned_element_size; 2506 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2507 if (!data->ring) { 2508 put_obj(obj); 2509 return -ENOMEM; 2510 } 2511 2512 /* IH is ready */ 2513 data->inuse = 1; 2514 2515 return 0; 2516 } 2517 2518 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2519 { 2520 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2521 struct ras_manager *obj, *tmp; 2522 2523 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2524 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2525 } 2526 2527 return 0; 2528 } 2529 /* ih end */ 2530 2531 /* traversal all IPs except NBIO to query error counter */ 2532 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2533 { 2534 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2535 struct ras_manager *obj; 2536 2537 if (!adev->ras_enabled || !con) 2538 return; 2539 2540 list_for_each_entry(obj, &con->head, node) { 2541 struct ras_query_if info = { 2542 .head = obj->head, 2543 }; 2544 2545 /* 2546 * PCIE_BIF IP has one different isr by ras controller 2547 * interrupt, the specific ras counter query will be 2548 * done in that isr. So skip such block from common 2549 * sync flood interrupt isr calling. 2550 */ 2551 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2552 continue; 2553 2554 /* 2555 * this is a workaround for aldebaran, skip send msg to 2556 * smu to get ecc_info table due to smu handle get ecc 2557 * info table failed temporarily. 2558 * should be removed until smu fix handle ecc_info table. 2559 */ 2560 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2561 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2562 IP_VERSION(13, 0, 2))) 2563 continue; 2564 2565 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2566 2567 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2568 IP_VERSION(11, 0, 2) && 2569 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2570 IP_VERSION(11, 0, 4) && 2571 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2572 IP_VERSION(13, 0, 0)) { 2573 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2574 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2575 } 2576 } 2577 } 2578 2579 /* Parse RdRspStatus and WrRspStatus */ 2580 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2581 struct ras_query_if *info) 2582 { 2583 struct amdgpu_ras_block_object *block_obj; 2584 /* 2585 * Only two block need to query read/write 2586 * RspStatus at current state 2587 */ 2588 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2589 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2590 return; 2591 2592 block_obj = amdgpu_ras_get_ras_block(adev, 2593 info->head.block, 2594 info->head.sub_block_index); 2595 2596 if (!block_obj || !block_obj->hw_ops) { 2597 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2598 get_ras_block_str(&info->head)); 2599 return; 2600 } 2601 2602 if (block_obj->hw_ops->query_ras_error_status) 2603 block_obj->hw_ops->query_ras_error_status(adev); 2604 2605 } 2606 2607 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2608 { 2609 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2610 struct ras_manager *obj; 2611 2612 if (!adev->ras_enabled || !con) 2613 return; 2614 2615 list_for_each_entry(obj, &con->head, node) { 2616 struct ras_query_if info = { 2617 .head = obj->head, 2618 }; 2619 2620 amdgpu_ras_error_status_query(adev, &info); 2621 } 2622 } 2623 2624 /* recovery begin */ 2625 2626 /* return 0 on success. 2627 * caller need free bps. 2628 */ 2629 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2630 struct ras_badpage **bps, unsigned int *count) 2631 { 2632 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2633 struct ras_err_handler_data *data; 2634 int i = 0; 2635 int ret = 0, status; 2636 2637 if (!con || !con->eh_data || !bps || !count) 2638 return -EINVAL; 2639 2640 mutex_lock(&con->recovery_lock); 2641 data = con->eh_data; 2642 if (!data || data->count == 0) { 2643 *bps = NULL; 2644 ret = -EINVAL; 2645 goto out; 2646 } 2647 2648 *bps = kmalloc_array(data->count, sizeof(struct ras_badpage), GFP_KERNEL); 2649 if (!*bps) { 2650 ret = -ENOMEM; 2651 goto out; 2652 } 2653 2654 for (; i < data->count; i++) { 2655 if (!data->bps[i].ts) 2656 continue; 2657 2658 (*bps)[i] = (struct ras_badpage){ 2659 .bp = data->bps[i].retired_page, 2660 .size = AMDGPU_GPU_PAGE_SIZE, 2661 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 2662 }; 2663 2664 if (amdgpu_ras_check_critical_address(adev, 2665 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2666 continue; 2667 2668 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 2669 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT); 2670 if (status == -EBUSY) 2671 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 2672 else if (status == -ENOENT) 2673 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 2674 } 2675 2676 *count = con->bad_page_num; 2677 out: 2678 mutex_unlock(&con->recovery_lock); 2679 return ret; 2680 } 2681 2682 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2683 struct amdgpu_hive_info *hive, bool status) 2684 { 2685 struct amdgpu_device *tmp_adev; 2686 2687 if (hive) { 2688 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2689 amdgpu_ras_set_fed(tmp_adev, status); 2690 } else { 2691 amdgpu_ras_set_fed(adev, status); 2692 } 2693 } 2694 2695 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2696 { 2697 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2698 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2699 int hive_ras_recovery = 0; 2700 2701 if (hive) { 2702 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2703 amdgpu_put_xgmi_hive(hive); 2704 } 2705 2706 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2707 return true; 2708 2709 return false; 2710 } 2711 2712 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2713 { 2714 if (amdgpu_ras_intr_triggered()) 2715 return RAS_EVENT_TYPE_FATAL; 2716 else 2717 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2718 } 2719 2720 static void amdgpu_ras_do_recovery(struct work_struct *work) 2721 { 2722 struct amdgpu_ras *ras = 2723 container_of(work, struct amdgpu_ras, recovery_work); 2724 struct amdgpu_device *remote_adev = NULL; 2725 struct amdgpu_device *adev = ras->adev; 2726 struct list_head device_list, *device_list_handle = NULL; 2727 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2728 unsigned int error_query_mode; 2729 enum ras_event_type type; 2730 2731 if (hive) { 2732 atomic_set(&hive->ras_recovery, 1); 2733 2734 /* If any device which is part of the hive received RAS fatal 2735 * error interrupt, set fatal error status on all. This 2736 * condition will need a recovery, and flag will be cleared 2737 * as part of recovery. 2738 */ 2739 list_for_each_entry(remote_adev, &hive->device_list, 2740 gmc.xgmi.head) 2741 if (amdgpu_ras_get_fed_status(remote_adev)) { 2742 amdgpu_ras_set_fed_all(adev, hive, true); 2743 break; 2744 } 2745 } 2746 if (!ras->disable_ras_err_cnt_harvest) { 2747 2748 /* Build list of devices to query RAS related errors */ 2749 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2750 device_list_handle = &hive->device_list; 2751 } else { 2752 INIT_LIST_HEAD(&device_list); 2753 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2754 device_list_handle = &device_list; 2755 } 2756 2757 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2758 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2759 /* wait 500ms to ensure pmfw polling mca bank info done */ 2760 msleep(500); 2761 } 2762 } 2763 2764 type = amdgpu_ras_get_fatal_error_event(adev); 2765 list_for_each_entry(remote_adev, 2766 device_list_handle, gmc.xgmi.head) { 2767 amdgpu_ras_query_err_status(remote_adev); 2768 amdgpu_ras_log_on_err_counter(remote_adev, type); 2769 } 2770 2771 } 2772 2773 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2774 struct amdgpu_reset_context reset_context; 2775 memset(&reset_context, 0, sizeof(reset_context)); 2776 2777 reset_context.method = AMD_RESET_METHOD_NONE; 2778 reset_context.reset_req_dev = adev; 2779 reset_context.src = AMDGPU_RESET_SRC_RAS; 2780 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2781 2782 /* Perform full reset in fatal error mode */ 2783 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2784 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2785 else { 2786 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2787 2788 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2789 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2790 reset_context.method = AMD_RESET_METHOD_MODE2; 2791 } 2792 2793 /* Fatal error occurs in poison mode, mode1 reset is used to 2794 * recover gpu. 2795 */ 2796 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2797 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2798 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2799 2800 psp_fatal_error_recovery_quirk(&adev->psp); 2801 } 2802 } 2803 2804 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2805 } 2806 atomic_set(&ras->in_recovery, 0); 2807 if (hive) { 2808 atomic_set(&hive->ras_recovery, 0); 2809 amdgpu_put_xgmi_hive(hive); 2810 } 2811 } 2812 2813 /* alloc/realloc bps array */ 2814 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2815 struct ras_err_handler_data *data, int pages) 2816 { 2817 unsigned int old_space = data->count + data->space_left; 2818 unsigned int new_space = old_space + pages; 2819 unsigned int align_space = ALIGN(new_space, 512); 2820 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2821 2822 if (!bps) { 2823 return -ENOMEM; 2824 } 2825 2826 if (data->bps) { 2827 memcpy(bps, data->bps, 2828 data->count * sizeof(*data->bps)); 2829 kfree(data->bps); 2830 } 2831 2832 data->bps = bps; 2833 data->space_left += align_space - old_space; 2834 return 0; 2835 } 2836 2837 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2838 struct eeprom_table_record *bps, 2839 struct ras_err_data *err_data) 2840 { 2841 struct ta_ras_query_address_input addr_in; 2842 uint32_t socket = 0; 2843 int ret = 0; 2844 2845 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2846 socket = adev->smuio.funcs->get_socket_id(adev); 2847 2848 /* reinit err_data */ 2849 err_data->err_addr_cnt = 0; 2850 err_data->err_addr_len = adev->umc.retire_unit; 2851 2852 memset(&addr_in, 0, sizeof(addr_in)); 2853 addr_in.ma.err_addr = bps->address; 2854 addr_in.ma.socket_id = socket; 2855 addr_in.ma.ch_inst = bps->mem_channel; 2856 /* tell RAS TA the node instance is not used */ 2857 addr_in.ma.node_inst = TA_RAS_INV_NODE; 2858 2859 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2860 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 2861 &addr_in, NULL, false); 2862 2863 return ret; 2864 } 2865 2866 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 2867 struct eeprom_table_record *bps, 2868 struct ras_err_data *err_data) 2869 { 2870 struct ta_ras_query_address_input addr_in; 2871 uint32_t die_id, socket = 0; 2872 2873 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2874 socket = adev->smuio.funcs->get_socket_id(adev); 2875 2876 /* although die id is gotten from PA in nps1 mode, the id is 2877 * fitable for any nps mode 2878 */ 2879 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 2880 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 2881 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 2882 else 2883 return -EINVAL; 2884 2885 /* reinit err_data */ 2886 err_data->err_addr_cnt = 0; 2887 err_data->err_addr_len = adev->umc.retire_unit; 2888 2889 memset(&addr_in, 0, sizeof(addr_in)); 2890 addr_in.ma.err_addr = bps->address; 2891 addr_in.ma.ch_inst = bps->mem_channel; 2892 addr_in.ma.umc_inst = bps->mcumc_id; 2893 addr_in.ma.node_inst = die_id; 2894 addr_in.ma.socket_id = socket; 2895 2896 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2897 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 2898 &addr_in, NULL, false); 2899 else 2900 return -EINVAL; 2901 } 2902 2903 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 2904 struct eeprom_table_record *bps, int count) 2905 { 2906 int j; 2907 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2908 struct ras_err_handler_data *data = con->eh_data; 2909 2910 for (j = 0; j < count; j++) { 2911 if (amdgpu_ras_check_bad_page_unlock(con, 2912 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 2913 data->count++; 2914 data->space_left--; 2915 continue; 2916 } 2917 2918 if (!data->space_left && 2919 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2920 return -ENOMEM; 2921 } 2922 2923 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 2924 2925 memcpy(&data->bps[data->count], &(bps[j]), 2926 sizeof(struct eeprom_table_record)); 2927 data->count++; 2928 data->space_left--; 2929 con->bad_page_num++; 2930 } 2931 2932 return 0; 2933 } 2934 2935 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 2936 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2937 enum amdgpu_memory_partition nps) 2938 { 2939 int i = 0; 2940 enum amdgpu_memory_partition save_nps; 2941 2942 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2943 2944 /*old asics just have pa in eeprom*/ 2945 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2946 memcpy(err_data->err_addr, bps, 2947 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2948 goto out; 2949 } 2950 2951 for (i = 0; i < adev->umc.retire_unit; i++) 2952 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2953 2954 if (save_nps) { 2955 if (save_nps == nps) { 2956 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2957 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2958 return -EINVAL; 2959 for (i = 0; i < adev->umc.retire_unit; i++) { 2960 err_data->err_addr[i].address = bps[0].address; 2961 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 2962 err_data->err_addr[i].bank = bps[0].bank; 2963 err_data->err_addr[i].err_type = bps[0].err_type; 2964 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 2965 } 2966 } else { 2967 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 2968 return -EINVAL; 2969 } 2970 } else { 2971 if (bps[0].address == 0) { 2972 /* for specific old eeprom data, mca address is not stored, 2973 * calc it from pa 2974 */ 2975 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 2976 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 2977 return -EINVAL; 2978 } 2979 2980 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 2981 if (nps == AMDGPU_NPS1_PARTITION_MODE) 2982 memcpy(err_data->err_addr, bps, 2983 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2984 else 2985 return -EOPNOTSUPP; 2986 } 2987 } 2988 2989 out: 2990 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 2991 } 2992 2993 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 2994 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2995 enum amdgpu_memory_partition nps) 2996 { 2997 int i = 0; 2998 enum amdgpu_memory_partition save_nps; 2999 3000 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3001 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3002 3003 if (save_nps == nps) { 3004 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3005 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3006 return -EINVAL; 3007 for (i = 0; i < adev->umc.retire_unit; i++) { 3008 err_data->err_addr[i].address = bps->address; 3009 err_data->err_addr[i].mem_channel = bps->mem_channel; 3010 err_data->err_addr[i].bank = bps->bank; 3011 err_data->err_addr[i].err_type = bps->err_type; 3012 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3013 } 3014 } else { 3015 if (bps->address) { 3016 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3017 return -EINVAL; 3018 } else { 3019 /* for specific old eeprom data, mca address is not stored, 3020 * calc it from pa 3021 */ 3022 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3023 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3024 return -EINVAL; 3025 3026 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3027 return -EOPNOTSUPP; 3028 } 3029 } 3030 3031 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3032 adev->umc.retire_unit); 3033 } 3034 3035 /* it deal with vram only. */ 3036 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3037 struct eeprom_table_record *bps, int pages, bool from_rom) 3038 { 3039 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3040 struct ras_err_data err_data; 3041 struct amdgpu_ras_eeprom_control *control = 3042 &adev->psp.ras_context.ras->eeprom_control; 3043 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3044 int ret = 0; 3045 uint32_t i = 0; 3046 3047 if (!con || !con->eh_data || !bps || pages <= 0) 3048 return 0; 3049 3050 if (from_rom) { 3051 err_data.err_addr = 3052 kcalloc(adev->umc.retire_unit, 3053 sizeof(struct eeprom_table_record), GFP_KERNEL); 3054 if (!err_data.err_addr) { 3055 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3056 return -ENOMEM; 3057 } 3058 3059 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3060 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3061 } 3062 3063 mutex_lock(&con->recovery_lock); 3064 3065 if (from_rom) { 3066 /* there is no pa recs in V3, so skip pa recs processing */ 3067 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3068 for (i = 0; i < pages; i++) { 3069 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3070 if ((bps[i].address == bps[i + 1].address) && 3071 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3072 /* deal with retire_unit records a time */ 3073 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3074 &bps[i], &err_data, nps); 3075 if (ret) 3076 con->bad_page_num -= adev->umc.retire_unit; 3077 i += (adev->umc.retire_unit - 1); 3078 } else { 3079 break; 3080 } 3081 } else { 3082 break; 3083 } 3084 } 3085 } 3086 for (; i < pages; i++) { 3087 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3088 &bps[i], &err_data, nps); 3089 if (ret) 3090 con->bad_page_num -= adev->umc.retire_unit; 3091 } 3092 3093 con->eh_data->count_saved = con->eh_data->count; 3094 } else { 3095 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3096 } 3097 3098 if (from_rom) 3099 kfree(err_data.err_addr); 3100 mutex_unlock(&con->recovery_lock); 3101 3102 return ret; 3103 } 3104 3105 /* 3106 * write error record array to eeprom, the function should be 3107 * protected by recovery_lock 3108 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3109 */ 3110 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3111 unsigned long *new_cnt) 3112 { 3113 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3114 struct ras_err_handler_data *data; 3115 struct amdgpu_ras_eeprom_control *control; 3116 int save_count, unit_num, i; 3117 3118 if (!con || !con->eh_data) { 3119 if (new_cnt) 3120 *new_cnt = 0; 3121 3122 return 0; 3123 } 3124 3125 if (!con->eeprom_control.is_eeprom_valid) { 3126 dev_warn(adev->dev, 3127 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3128 if (new_cnt) 3129 *new_cnt = 0; 3130 3131 return 0; 3132 } 3133 3134 mutex_lock(&con->recovery_lock); 3135 control = &con->eeprom_control; 3136 data = con->eh_data; 3137 unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; 3138 save_count = con->bad_page_num - control->ras_num_bad_pages; 3139 mutex_unlock(&con->recovery_lock); 3140 3141 if (new_cnt) 3142 *new_cnt = unit_num; 3143 3144 /* only new entries are saved */ 3145 if (unit_num > 0) { 3146 /*old asics only save pa to eeprom like before*/ 3147 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3148 if (amdgpu_ras_eeprom_append(control, 3149 &data->bps[data->count_saved], unit_num)) { 3150 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3151 return -EIO; 3152 } 3153 } else { 3154 for (i = 0; i < unit_num; i++) { 3155 if (amdgpu_ras_eeprom_append(control, 3156 &data->bps[data->count_saved + 3157 i * adev->umc.retire_unit], 1)) { 3158 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3159 return -EIO; 3160 } 3161 } 3162 } 3163 3164 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3165 data->count_saved = data->count; 3166 } 3167 3168 return 0; 3169 } 3170 3171 /* 3172 * read error record array in eeprom and reserve enough space for 3173 * storing new bad pages 3174 */ 3175 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3176 { 3177 struct amdgpu_ras_eeprom_control *control = 3178 &adev->psp.ras_context.ras->eeprom_control; 3179 struct eeprom_table_record *bps; 3180 int ret, i = 0; 3181 3182 /* no bad page record, skip eeprom access */ 3183 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3184 return 0; 3185 3186 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3187 if (!bps) 3188 return -ENOMEM; 3189 3190 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3191 if (ret) { 3192 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3193 } else { 3194 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3195 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3196 as pa recs, so add verion check to avoid it. 3197 */ 3198 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3199 for (i = 0; i < control->ras_num_recs; i++) { 3200 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3201 if ((bps[i].address == bps[i + 1].address) && 3202 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3203 control->ras_num_pa_recs += adev->umc.retire_unit; 3204 i += (adev->umc.retire_unit - 1); 3205 } else { 3206 control->ras_num_mca_recs += 3207 (control->ras_num_recs - i); 3208 break; 3209 } 3210 } else { 3211 control->ras_num_mca_recs += (control->ras_num_recs - i); 3212 break; 3213 } 3214 } 3215 } else { 3216 control->ras_num_mca_recs = control->ras_num_recs; 3217 } 3218 } 3219 3220 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3221 if (ret) 3222 goto out; 3223 3224 ret = amdgpu_ras_eeprom_check(control); 3225 if (ret) 3226 goto out; 3227 3228 /* HW not usable */ 3229 if (amdgpu_ras_is_rma(adev)) 3230 ret = -EHWPOISON; 3231 } 3232 3233 out: 3234 kfree(bps); 3235 return ret; 3236 } 3237 3238 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3239 uint64_t addr) 3240 { 3241 struct ras_err_handler_data *data = con->eh_data; 3242 struct amdgpu_device *adev = con->adev; 3243 int i; 3244 3245 if ((addr >= adev->gmc.mc_vram_size && 3246 adev->gmc.mc_vram_size) || 3247 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3248 return -EINVAL; 3249 3250 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3251 for (i = 0; i < data->count; i++) 3252 if (addr == data->bps[i].retired_page) 3253 return 1; 3254 3255 return 0; 3256 } 3257 3258 /* 3259 * check if an address belongs to bad page 3260 * 3261 * Note: this check is only for umc block 3262 */ 3263 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3264 uint64_t addr) 3265 { 3266 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3267 int ret = 0; 3268 3269 if (!con || !con->eh_data) 3270 return ret; 3271 3272 mutex_lock(&con->recovery_lock); 3273 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3274 mutex_unlock(&con->recovery_lock); 3275 return ret; 3276 } 3277 3278 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3279 uint32_t max_count) 3280 { 3281 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3282 3283 /* 3284 * amdgpu_bad_page_threshold is used to config 3285 * the threshold for the number of bad pages. 3286 * -1: Threshold is set to default value 3287 * Driver will issue a warning message when threshold is reached 3288 * and continue runtime services. 3289 * 0: Disable bad page retirement 3290 * Driver will not retire bad pages 3291 * which is intended for debugging purpose. 3292 * -2: Threshold is determined by a formula 3293 * that assumes 1 bad page per 100M of local memory. 3294 * Driver will continue runtime services when threhold is reached. 3295 * 0 < threshold < max number of bad page records in EEPROM, 3296 * A user-defined threshold is set 3297 * Driver will halt runtime services when this custom threshold is reached. 3298 */ 3299 if (amdgpu_bad_page_threshold == -2) { 3300 u64 val = adev->gmc.mc_vram_size; 3301 3302 do_div(val, RAS_BAD_PAGE_COVER); 3303 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3304 max_count); 3305 } else if (amdgpu_bad_page_threshold == -1) { 3306 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3307 } else { 3308 con->bad_page_cnt_threshold = min_t(int, max_count, 3309 amdgpu_bad_page_threshold); 3310 } 3311 } 3312 3313 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3314 enum amdgpu_ras_block block, uint16_t pasid, 3315 pasid_notify pasid_fn, void *data, uint32_t reset) 3316 { 3317 int ret = 0; 3318 struct ras_poison_msg poison_msg; 3319 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3320 3321 memset(&poison_msg, 0, sizeof(poison_msg)); 3322 poison_msg.block = block; 3323 poison_msg.pasid = pasid; 3324 poison_msg.reset = reset; 3325 poison_msg.pasid_fn = pasid_fn; 3326 poison_msg.data = data; 3327 3328 ret = kfifo_put(&con->poison_fifo, poison_msg); 3329 if (!ret) { 3330 dev_err(adev->dev, "Poison message fifo is full!\n"); 3331 return -ENOSPC; 3332 } 3333 3334 return 0; 3335 } 3336 3337 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3338 struct ras_poison_msg *poison_msg) 3339 { 3340 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3341 3342 return kfifo_get(&con->poison_fifo, poison_msg); 3343 } 3344 3345 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3346 { 3347 mutex_init(&ecc_log->lock); 3348 3349 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3350 ecc_log->de_queried_count = 0; 3351 ecc_log->consumption_q_count = 0; 3352 } 3353 3354 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3355 { 3356 struct radix_tree_iter iter; 3357 void __rcu **slot; 3358 struct ras_ecc_err *ecc_err; 3359 3360 mutex_lock(&ecc_log->lock); 3361 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3362 ecc_err = radix_tree_deref_slot(slot); 3363 kfree(ecc_err->err_pages.pfn); 3364 kfree(ecc_err); 3365 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3366 } 3367 mutex_unlock(&ecc_log->lock); 3368 3369 mutex_destroy(&ecc_log->lock); 3370 ecc_log->de_queried_count = 0; 3371 ecc_log->consumption_q_count = 0; 3372 } 3373 3374 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3375 uint32_t delayed_ms) 3376 { 3377 int ret; 3378 3379 mutex_lock(&con->umc_ecc_log.lock); 3380 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3381 UMC_ECC_NEW_DETECTED_TAG); 3382 mutex_unlock(&con->umc_ecc_log.lock); 3383 3384 if (ret) 3385 schedule_delayed_work(&con->page_retirement_dwork, 3386 msecs_to_jiffies(delayed_ms)); 3387 3388 return ret ? true : false; 3389 } 3390 3391 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3392 { 3393 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3394 page_retirement_dwork.work); 3395 struct amdgpu_device *adev = con->adev; 3396 struct ras_err_data err_data; 3397 3398 /* If gpu reset is ongoing, delay retiring the bad pages */ 3399 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3400 amdgpu_ras_schedule_retirement_dwork(con, 3401 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3402 return; 3403 } 3404 3405 amdgpu_ras_error_data_init(&err_data); 3406 3407 amdgpu_umc_handle_bad_pages(adev, &err_data); 3408 3409 amdgpu_ras_error_data_fini(&err_data); 3410 3411 amdgpu_ras_schedule_retirement_dwork(con, 3412 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3413 } 3414 3415 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3416 uint32_t poison_creation_count) 3417 { 3418 int ret = 0; 3419 struct ras_ecc_log_info *ecc_log; 3420 struct ras_query_if info; 3421 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3422 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3423 u64 de_queried_count; 3424 u64 consumption_q_count; 3425 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3426 3427 memset(&info, 0, sizeof(info)); 3428 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3429 3430 ecc_log = &ras->umc_ecc_log; 3431 ecc_log->de_queried_count = 0; 3432 ecc_log->consumption_q_count = 0; 3433 3434 do { 3435 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3436 if (ret) 3437 return ret; 3438 3439 de_queried_count = ecc_log->de_queried_count; 3440 consumption_q_count = ecc_log->consumption_q_count; 3441 3442 if (de_queried_count && consumption_q_count) 3443 break; 3444 3445 msleep(100); 3446 } while (--timeout); 3447 3448 if (de_queried_count) 3449 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3450 3451 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3452 amdgpu_ras_reset_gpu(adev); 3453 3454 return 0; 3455 } 3456 3457 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3458 { 3459 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3460 struct ras_poison_msg msg; 3461 int ret; 3462 3463 do { 3464 ret = kfifo_get(&con->poison_fifo, &msg); 3465 } while (ret); 3466 } 3467 3468 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3469 uint32_t msg_count, uint32_t *gpu_reset) 3470 { 3471 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3472 uint32_t reset_flags = 0, reset = 0; 3473 struct ras_poison_msg msg; 3474 int ret, i; 3475 3476 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3477 3478 for (i = 0; i < msg_count; i++) { 3479 ret = amdgpu_ras_get_poison_req(adev, &msg); 3480 if (!ret) 3481 continue; 3482 3483 if (msg.pasid_fn) 3484 msg.pasid_fn(adev, msg.pasid, msg.data); 3485 3486 reset_flags |= msg.reset; 3487 } 3488 3489 /* 3490 * Try to ensure poison creation handler is completed first 3491 * to set rma if bad page exceed threshold. 3492 */ 3493 flush_delayed_work(&con->page_retirement_dwork); 3494 3495 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3496 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3497 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3498 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3499 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3500 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3501 else 3502 reset = reset_flags; 3503 3504 con->gpu_reset_flags |= reset; 3505 amdgpu_ras_reset_gpu(adev); 3506 3507 *gpu_reset = reset; 3508 3509 /* Wait for gpu recovery to complete */ 3510 flush_work(&con->recovery_work); 3511 } 3512 3513 return 0; 3514 } 3515 3516 static int amdgpu_ras_page_retirement_thread(void *param) 3517 { 3518 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3519 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3520 uint32_t poison_creation_count, msg_count; 3521 uint32_t gpu_reset; 3522 int ret; 3523 3524 while (!kthread_should_stop()) { 3525 3526 wait_event_interruptible(con->page_retirement_wq, 3527 kthread_should_stop() || 3528 atomic_read(&con->page_retirement_req_cnt)); 3529 3530 if (kthread_should_stop()) 3531 break; 3532 3533 mutex_lock(&con->poison_lock); 3534 gpu_reset = 0; 3535 3536 do { 3537 poison_creation_count = atomic_read(&con->poison_creation_count); 3538 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3539 if (ret == -EIO) 3540 break; 3541 3542 if (poison_creation_count) { 3543 atomic_sub(poison_creation_count, &con->poison_creation_count); 3544 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3545 } 3546 } while (atomic_read(&con->poison_creation_count) && 3547 !atomic_read(&con->poison_consumption_count)); 3548 3549 if (ret != -EIO) { 3550 msg_count = kfifo_len(&con->poison_fifo); 3551 if (msg_count) { 3552 ret = amdgpu_ras_poison_consumption_handler(adev, 3553 msg_count, &gpu_reset); 3554 if ((ret != -EIO) && 3555 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3556 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3557 } 3558 } 3559 3560 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3561 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3562 /* Clear poison creation request */ 3563 atomic_set(&con->poison_creation_count, 0); 3564 atomic_set(&con->poison_consumption_count, 0); 3565 3566 /* Clear poison fifo */ 3567 amdgpu_ras_clear_poison_fifo(adev); 3568 3569 /* Clear all poison requests */ 3570 atomic_set(&con->page_retirement_req_cnt, 0); 3571 3572 if (ret == -EIO) { 3573 /* Wait for mode-1 reset to complete */ 3574 down_read(&adev->reset_domain->sem); 3575 up_read(&adev->reset_domain->sem); 3576 } 3577 3578 /* Wake up work to save bad pages to eeprom */ 3579 schedule_delayed_work(&con->page_retirement_dwork, 0); 3580 } else if (gpu_reset) { 3581 /* gpu just completed mode-2 reset or other reset */ 3582 /* Clear poison consumption messages cached in fifo */ 3583 msg_count = kfifo_len(&con->poison_fifo); 3584 if (msg_count) { 3585 amdgpu_ras_clear_poison_fifo(adev); 3586 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3587 } 3588 3589 atomic_set(&con->poison_consumption_count, 0); 3590 3591 /* Wake up work to save bad pages to eeprom */ 3592 schedule_delayed_work(&con->page_retirement_dwork, 0); 3593 } 3594 mutex_unlock(&con->poison_lock); 3595 } 3596 3597 return 0; 3598 } 3599 3600 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3601 { 3602 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3603 struct amdgpu_ras_eeprom_control *control; 3604 int ret; 3605 3606 if (!con || amdgpu_sriov_vf(adev)) 3607 return 0; 3608 3609 if (amdgpu_uniras_enabled(adev)) 3610 return 0; 3611 3612 control = &con->eeprom_control; 3613 ret = amdgpu_ras_eeprom_init(control); 3614 control->is_eeprom_valid = !ret; 3615 3616 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3617 control->ras_num_pa_recs = control->ras_num_recs; 3618 3619 if (adev->umc.ras && 3620 adev->umc.ras->get_retire_flip_bits) 3621 adev->umc.ras->get_retire_flip_bits(adev); 3622 3623 if (control->ras_num_recs && control->is_eeprom_valid) { 3624 ret = amdgpu_ras_load_bad_pages(adev); 3625 if (ret) { 3626 control->is_eeprom_valid = false; 3627 return 0; 3628 } 3629 3630 amdgpu_dpm_send_hbm_bad_pages_num( 3631 adev, control->ras_num_bad_pages); 3632 3633 if (con->update_channel_flag == true) { 3634 amdgpu_dpm_send_hbm_bad_channel_flag( 3635 adev, control->bad_channel_bitmap); 3636 con->update_channel_flag = false; 3637 } 3638 3639 /* The format action is only applied to new ASICs */ 3640 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3641 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3642 if (!amdgpu_ras_eeprom_reset_table(control)) 3643 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3644 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3645 } 3646 3647 return 0; 3648 } 3649 3650 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3651 { 3652 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3653 struct ras_err_handler_data **data; 3654 u32 max_eeprom_records_count = 0; 3655 int ret; 3656 3657 if (!con || amdgpu_sriov_vf(adev)) 3658 return 0; 3659 3660 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3661 * supports RAS and debugfs is enabled, but when 3662 * adev->ras_enabled is unset, i.e. when "ras_enable" 3663 * module parameter is set to 0. 3664 */ 3665 con->adev = adev; 3666 3667 if (!adev->ras_enabled) 3668 return 0; 3669 3670 data = &con->eh_data; 3671 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3672 if (!*data) { 3673 ret = -ENOMEM; 3674 goto out; 3675 } 3676 3677 mutex_init(&con->recovery_lock); 3678 mutex_init(&con->poison_lock); 3679 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3680 atomic_set(&con->in_recovery, 0); 3681 atomic_set(&con->rma_in_recovery, 0); 3682 con->eeprom_control.bad_channel_bitmap = 0; 3683 3684 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3685 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3686 3687 if (init_bp_info) { 3688 ret = amdgpu_ras_init_badpage_info(adev); 3689 if (ret) 3690 goto free; 3691 } 3692 3693 mutex_init(&con->page_rsv_lock); 3694 INIT_KFIFO(con->poison_fifo); 3695 mutex_init(&con->page_retirement_lock); 3696 init_waitqueue_head(&con->page_retirement_wq); 3697 atomic_set(&con->page_retirement_req_cnt, 0); 3698 atomic_set(&con->poison_creation_count, 0); 3699 atomic_set(&con->poison_consumption_count, 0); 3700 con->page_retirement_thread = 3701 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3702 if (IS_ERR(con->page_retirement_thread)) { 3703 con->page_retirement_thread = NULL; 3704 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3705 } 3706 3707 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3708 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3709 #ifdef CONFIG_X86_MCE_AMD 3710 if ((adev->asic_type == CHIP_ALDEBARAN) && 3711 (adev->gmc.xgmi.connected_to_cpu)) 3712 amdgpu_register_bad_pages_mca_notifier(adev); 3713 #endif 3714 return 0; 3715 3716 free: 3717 kfree((*data)->bps); 3718 kfree(*data); 3719 con->eh_data = NULL; 3720 out: 3721 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3722 3723 /* 3724 * Except error threshold exceeding case, other failure cases in this 3725 * function would not fail amdgpu driver init. 3726 */ 3727 if (!amdgpu_ras_is_rma(adev)) 3728 ret = 0; 3729 else 3730 ret = -EINVAL; 3731 3732 return ret; 3733 } 3734 3735 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3736 { 3737 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3738 struct ras_err_handler_data *data = con->eh_data; 3739 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3740 bool ret; 3741 3742 /* recovery_init failed to init it, fini is useless */ 3743 if (!data) 3744 return 0; 3745 3746 /* Save all cached bad pages to eeprom */ 3747 do { 3748 flush_delayed_work(&con->page_retirement_dwork); 3749 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3750 } while (ret && max_flush_timeout--); 3751 3752 if (con->page_retirement_thread) 3753 kthread_stop(con->page_retirement_thread); 3754 3755 atomic_set(&con->page_retirement_req_cnt, 0); 3756 atomic_set(&con->poison_creation_count, 0); 3757 3758 mutex_destroy(&con->page_rsv_lock); 3759 3760 cancel_work_sync(&con->recovery_work); 3761 3762 cancel_delayed_work_sync(&con->page_retirement_dwork); 3763 3764 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3765 3766 mutex_lock(&con->recovery_lock); 3767 con->eh_data = NULL; 3768 kfree(data->bps); 3769 kfree(data); 3770 mutex_unlock(&con->recovery_lock); 3771 3772 amdgpu_ras_critical_region_init(adev); 3773 3774 return 0; 3775 } 3776 /* recovery end */ 3777 3778 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3779 { 3780 if (amdgpu_sriov_vf(adev)) { 3781 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3782 case IP_VERSION(13, 0, 2): 3783 case IP_VERSION(13, 0, 6): 3784 case IP_VERSION(13, 0, 12): 3785 case IP_VERSION(13, 0, 14): 3786 return true; 3787 default: 3788 return false; 3789 } 3790 } 3791 3792 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3793 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3794 case IP_VERSION(13, 0, 0): 3795 case IP_VERSION(13, 0, 6): 3796 case IP_VERSION(13, 0, 10): 3797 case IP_VERSION(13, 0, 12): 3798 case IP_VERSION(13, 0, 14): 3799 case IP_VERSION(14, 0, 3): 3800 return true; 3801 default: 3802 return false; 3803 } 3804 } 3805 3806 return adev->asic_type == CHIP_VEGA10 || 3807 adev->asic_type == CHIP_VEGA20 || 3808 adev->asic_type == CHIP_ARCTURUS || 3809 adev->asic_type == CHIP_ALDEBARAN || 3810 adev->asic_type == CHIP_SIENNA_CICHLID; 3811 } 3812 3813 /* 3814 * this is workaround for vega20 workstation sku, 3815 * force enable gfx ras, ignore vbios gfx ras flag 3816 * due to GC EDC can not write 3817 */ 3818 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3819 { 3820 struct atom_context *ctx = adev->mode_info.atom_context; 3821 3822 if (!ctx) 3823 return; 3824 3825 if (strnstr(ctx->vbios_pn, "D16406", 3826 sizeof(ctx->vbios_pn)) || 3827 strnstr(ctx->vbios_pn, "D36002", 3828 sizeof(ctx->vbios_pn))) 3829 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3830 } 3831 3832 /* Query ras capablity via atomfirmware interface */ 3833 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3834 { 3835 /* mem_ecc cap */ 3836 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3837 dev_info(adev->dev, "MEM ECC is active.\n"); 3838 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3839 1 << AMDGPU_RAS_BLOCK__DF); 3840 } else { 3841 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3842 } 3843 3844 /* sram_ecc cap */ 3845 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 3846 dev_info(adev->dev, "SRAM ECC is active.\n"); 3847 if (!amdgpu_sriov_vf(adev)) 3848 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 3849 1 << AMDGPU_RAS_BLOCK__DF); 3850 else 3851 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 3852 1 << AMDGPU_RAS_BLOCK__SDMA | 3853 1 << AMDGPU_RAS_BLOCK__GFX); 3854 3855 /* 3856 * VCN/JPEG RAS can be supported on both bare metal and 3857 * SRIOV environment 3858 */ 3859 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 3860 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 3861 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 3862 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 3863 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 3864 1 << AMDGPU_RAS_BLOCK__JPEG); 3865 else 3866 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 3867 1 << AMDGPU_RAS_BLOCK__JPEG); 3868 3869 /* 3870 * XGMI RAS is not supported if xgmi num physical nodes 3871 * is zero 3872 */ 3873 if (!adev->gmc.xgmi.num_physical_nodes) 3874 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 3875 } else { 3876 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 3877 } 3878 } 3879 3880 /* Query poison mode from umc/df IP callbacks */ 3881 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 3882 { 3883 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3884 bool df_poison, umc_poison; 3885 3886 /* poison setting is useless on SRIOV guest */ 3887 if (amdgpu_sriov_vf(adev) || !con) 3888 return; 3889 3890 /* Init poison supported flag, the default value is false */ 3891 if (adev->gmc.xgmi.connected_to_cpu || 3892 adev->gmc.is_app_apu) { 3893 /* enabled by default when GPU is connected to CPU */ 3894 con->poison_supported = true; 3895 } else if (adev->df.funcs && 3896 adev->df.funcs->query_ras_poison_mode && 3897 adev->umc.ras && 3898 adev->umc.ras->query_ras_poison_mode) { 3899 df_poison = 3900 adev->df.funcs->query_ras_poison_mode(adev); 3901 umc_poison = 3902 adev->umc.ras->query_ras_poison_mode(adev); 3903 3904 /* Only poison is set in both DF and UMC, we can support it */ 3905 if (df_poison && umc_poison) 3906 con->poison_supported = true; 3907 else if (df_poison != umc_poison) 3908 dev_warn(adev->dev, 3909 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 3910 df_poison, umc_poison); 3911 } 3912 } 3913 3914 /* 3915 * check hardware's ras ability which will be saved in hw_supported. 3916 * if hardware does not support ras, we can skip some ras initializtion and 3917 * forbid some ras operations from IP. 3918 * if software itself, say boot parameter, limit the ras ability. We still 3919 * need allow IP do some limited operations, like disable. In such case, 3920 * we have to initialize ras as normal. but need check if operation is 3921 * allowed or not in each function. 3922 */ 3923 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 3924 { 3925 adev->ras_hw_enabled = adev->ras_enabled = 0; 3926 3927 if (!amdgpu_ras_asic_supported(adev)) 3928 return; 3929 3930 if (amdgpu_sriov_vf(adev)) { 3931 if (amdgpu_virt_get_ras_capability(adev)) 3932 goto init_ras_enabled_flag; 3933 } 3934 3935 /* query ras capability from psp */ 3936 if (amdgpu_psp_get_ras_capability(&adev->psp)) 3937 goto init_ras_enabled_flag; 3938 3939 /* query ras capablity from bios */ 3940 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 3941 amdgpu_ras_query_ras_capablity_from_vbios(adev); 3942 } else { 3943 /* driver only manages a few IP blocks RAS feature 3944 * when GPU is connected cpu through XGMI */ 3945 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 3946 1 << AMDGPU_RAS_BLOCK__SDMA | 3947 1 << AMDGPU_RAS_BLOCK__MMHUB); 3948 } 3949 3950 /* apply asic specific settings (vega20 only for now) */ 3951 amdgpu_ras_get_quirks(adev); 3952 3953 /* query poison mode from umc/df ip callback */ 3954 amdgpu_ras_query_poison_mode(adev); 3955 3956 init_ras_enabled_flag: 3957 /* hw_supported needs to be aligned with RAS block mask. */ 3958 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 3959 3960 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 3961 adev->ras_hw_enabled & amdgpu_ras_mask; 3962 3963 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 3964 if (!amdgpu_sriov_vf(adev)) { 3965 adev->aca.is_enabled = 3966 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 3967 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 3968 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 3969 } 3970 3971 /* bad page feature is not applicable to specific app platform */ 3972 if (adev->gmc.is_app_apu && 3973 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 3974 amdgpu_bad_page_threshold = 0; 3975 } 3976 3977 static void amdgpu_ras_counte_dw(struct work_struct *work) 3978 { 3979 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3980 ras_counte_delay_work.work); 3981 struct amdgpu_device *adev = con->adev; 3982 struct drm_device *dev = adev_to_drm(adev); 3983 unsigned long ce_count, ue_count; 3984 int res; 3985 3986 res = pm_runtime_get_sync(dev->dev); 3987 if (res < 0) 3988 goto Out; 3989 3990 /* Cache new values. 3991 */ 3992 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 3993 atomic_set(&con->ras_ce_count, ce_count); 3994 atomic_set(&con->ras_ue_count, ue_count); 3995 } 3996 3997 pm_runtime_mark_last_busy(dev->dev); 3998 Out: 3999 pm_runtime_put_autosuspend(dev->dev); 4000 } 4001 4002 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4003 { 4004 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4005 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4006 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4007 AMDGPU_RAS_ERROR__PARITY; 4008 } 4009 4010 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4011 { 4012 struct ras_event_state *event_state; 4013 int i; 4014 4015 memset(mgr, 0, sizeof(*mgr)); 4016 atomic64_set(&mgr->seqno, 0); 4017 4018 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4019 event_state = &mgr->event_state[i]; 4020 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4021 atomic64_set(&event_state->count, 0); 4022 } 4023 } 4024 4025 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4026 { 4027 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4028 struct amdgpu_hive_info *hive; 4029 4030 if (!ras) 4031 return; 4032 4033 hive = amdgpu_get_xgmi_hive(adev); 4034 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4035 4036 /* init event manager with node 0 on xgmi system */ 4037 if (!amdgpu_reset_in_recovery(adev)) { 4038 if (!hive || adev->gmc.xgmi.node_id == 0) 4039 ras_event_mgr_init(ras->event_mgr); 4040 } 4041 4042 if (hive) 4043 amdgpu_put_xgmi_hive(hive); 4044 } 4045 4046 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4047 { 4048 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4049 4050 if (!con || (adev->flags & AMD_IS_APU)) 4051 return; 4052 4053 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4054 case IP_VERSION(13, 0, 2): 4055 case IP_VERSION(13, 0, 6): 4056 case IP_VERSION(13, 0, 12): 4057 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4058 break; 4059 case IP_VERSION(13, 0, 14): 4060 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4061 break; 4062 default: 4063 break; 4064 } 4065 } 4066 4067 int amdgpu_ras_init(struct amdgpu_device *adev) 4068 { 4069 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4070 int r; 4071 4072 if (con) 4073 return 0; 4074 4075 con = kzalloc(sizeof(*con) + 4076 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4077 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4078 GFP_KERNEL); 4079 if (!con) 4080 return -ENOMEM; 4081 4082 con->adev = adev; 4083 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4084 atomic_set(&con->ras_ce_count, 0); 4085 atomic_set(&con->ras_ue_count, 0); 4086 4087 con->objs = (struct ras_manager *)(con + 1); 4088 4089 amdgpu_ras_set_context(adev, con); 4090 4091 amdgpu_ras_check_supported(adev); 4092 4093 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4094 /* set gfx block ras context feature for VEGA20 Gaming 4095 * send ras disable cmd to ras ta during ras late init. 4096 */ 4097 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4098 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4099 4100 return 0; 4101 } 4102 4103 r = 0; 4104 goto release_con; 4105 } 4106 4107 con->update_channel_flag = false; 4108 con->features = 0; 4109 con->schema = 0; 4110 INIT_LIST_HEAD(&con->head); 4111 /* Might need get this flag from vbios. */ 4112 con->flags = RAS_DEFAULT_FLAGS; 4113 4114 /* initialize nbio ras function ahead of any other 4115 * ras functions so hardware fatal error interrupt 4116 * can be enabled as early as possible */ 4117 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4118 case IP_VERSION(7, 4, 0): 4119 case IP_VERSION(7, 4, 1): 4120 case IP_VERSION(7, 4, 4): 4121 if (!adev->gmc.xgmi.connected_to_cpu) 4122 adev->nbio.ras = &nbio_v7_4_ras; 4123 break; 4124 case IP_VERSION(4, 3, 0): 4125 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4126 /* unlike other generation of nbio ras, 4127 * nbio v4_3 only support fatal error interrupt 4128 * to inform software that DF is freezed due to 4129 * system fatal error event. driver should not 4130 * enable nbio ras in such case. Instead, 4131 * check DF RAS */ 4132 adev->nbio.ras = &nbio_v4_3_ras; 4133 break; 4134 case IP_VERSION(6, 3, 1): 4135 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4136 /* unlike other generation of nbio ras, 4137 * nbif v6_3_1 only support fatal error interrupt 4138 * to inform software that DF is freezed due to 4139 * system fatal error event. driver should not 4140 * enable nbio ras in such case. Instead, 4141 * check DF RAS 4142 */ 4143 adev->nbio.ras = &nbif_v6_3_1_ras; 4144 break; 4145 case IP_VERSION(7, 9, 0): 4146 case IP_VERSION(7, 9, 1): 4147 if (!adev->gmc.is_app_apu) 4148 adev->nbio.ras = &nbio_v7_9_ras; 4149 break; 4150 default: 4151 /* nbio ras is not available */ 4152 break; 4153 } 4154 4155 /* nbio ras block needs to be enabled ahead of other ras blocks 4156 * to handle fatal error */ 4157 r = amdgpu_nbio_ras_sw_init(adev); 4158 if (r) 4159 return r; 4160 4161 if (adev->nbio.ras && 4162 adev->nbio.ras->init_ras_controller_interrupt) { 4163 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4164 if (r) 4165 goto release_con; 4166 } 4167 4168 if (adev->nbio.ras && 4169 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4170 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4171 if (r) 4172 goto release_con; 4173 } 4174 4175 /* Packed socket_id to ras feature mask bits[31:29] */ 4176 if (adev->smuio.funcs && 4177 adev->smuio.funcs->get_socket_id) 4178 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4179 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4180 4181 /* Get RAS schema for particular SOC */ 4182 con->schema = amdgpu_get_ras_schema(adev); 4183 4184 amdgpu_ras_init_reserved_vram_size(adev); 4185 4186 if (amdgpu_ras_fs_init(adev)) { 4187 r = -EINVAL; 4188 goto release_con; 4189 } 4190 4191 if (amdgpu_ras_aca_is_supported(adev)) { 4192 if (amdgpu_aca_is_enabled(adev)) 4193 r = amdgpu_aca_init(adev); 4194 else 4195 r = amdgpu_mca_init(adev); 4196 if (r) 4197 goto release_con; 4198 } 4199 4200 con->init_task_pid = task_pid_nr(current); 4201 get_task_comm(con->init_task_comm, current); 4202 4203 mutex_init(&con->critical_region_lock); 4204 INIT_LIST_HEAD(&con->critical_region_head); 4205 4206 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4207 "hardware ability[%x] ras_mask[%x]\n", 4208 adev->ras_hw_enabled, adev->ras_enabled); 4209 4210 return 0; 4211 release_con: 4212 amdgpu_ras_set_context(adev, NULL); 4213 kfree(con); 4214 4215 return r; 4216 } 4217 4218 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4219 { 4220 if (adev->gmc.xgmi.connected_to_cpu || 4221 adev->gmc.is_app_apu) 4222 return 1; 4223 return 0; 4224 } 4225 4226 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4227 struct ras_common_if *ras_block) 4228 { 4229 struct ras_query_if info = { 4230 .head = *ras_block, 4231 }; 4232 4233 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4234 return 0; 4235 4236 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4237 DRM_WARN("RAS init harvest failure"); 4238 4239 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4240 DRM_WARN("RAS init harvest reset failure"); 4241 4242 return 0; 4243 } 4244 4245 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4246 { 4247 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4248 4249 if (!con) 4250 return false; 4251 4252 return con->poison_supported; 4253 } 4254 4255 /* helper function to handle common stuff in ip late init phase */ 4256 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4257 struct ras_common_if *ras_block) 4258 { 4259 struct amdgpu_ras_block_object *ras_obj = NULL; 4260 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4261 struct ras_query_if *query_info; 4262 unsigned long ue_count, ce_count; 4263 int r; 4264 4265 /* disable RAS feature per IP block if it is not supported */ 4266 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4267 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4268 return 0; 4269 } 4270 4271 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4272 if (r) { 4273 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4274 /* in resume phase, if fail to enable ras, 4275 * clean up all ras fs nodes, and disable ras */ 4276 goto cleanup; 4277 } else 4278 return r; 4279 } 4280 4281 /* check for errors on warm reset edc persisant supported ASIC */ 4282 amdgpu_persistent_edc_harvesting(adev, ras_block); 4283 4284 /* in resume phase, no need to create ras fs node */ 4285 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4286 return 0; 4287 4288 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4289 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4290 (ras_obj->hw_ops->query_poison_status || 4291 ras_obj->hw_ops->handle_poison_consumption))) { 4292 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4293 if (r) 4294 goto cleanup; 4295 } 4296 4297 if (ras_obj->hw_ops && 4298 (ras_obj->hw_ops->query_ras_error_count || 4299 ras_obj->hw_ops->query_ras_error_status)) { 4300 r = amdgpu_ras_sysfs_create(adev, ras_block); 4301 if (r) 4302 goto interrupt; 4303 4304 /* Those are the cached values at init. 4305 */ 4306 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4307 if (!query_info) 4308 return -ENOMEM; 4309 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4310 4311 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4312 atomic_set(&con->ras_ce_count, ce_count); 4313 atomic_set(&con->ras_ue_count, ue_count); 4314 } 4315 4316 kfree(query_info); 4317 } 4318 4319 return 0; 4320 4321 interrupt: 4322 if (ras_obj->ras_cb) 4323 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4324 cleanup: 4325 amdgpu_ras_feature_enable(adev, ras_block, 0); 4326 return r; 4327 } 4328 4329 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4330 struct ras_common_if *ras_block) 4331 { 4332 return amdgpu_ras_block_late_init(adev, ras_block); 4333 } 4334 4335 /* helper function to remove ras fs node and interrupt handler */ 4336 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4337 struct ras_common_if *ras_block) 4338 { 4339 struct amdgpu_ras_block_object *ras_obj; 4340 if (!ras_block) 4341 return; 4342 4343 amdgpu_ras_sysfs_remove(adev, ras_block); 4344 4345 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4346 if (ras_obj->ras_cb) 4347 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4348 } 4349 4350 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4351 struct ras_common_if *ras_block) 4352 { 4353 return amdgpu_ras_block_late_fini(adev, ras_block); 4354 } 4355 4356 /* do some init work after IP late init as dependence. 4357 * and it runs in resume/gpu reset/booting up cases. 4358 */ 4359 void amdgpu_ras_resume(struct amdgpu_device *adev) 4360 { 4361 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4362 struct ras_manager *obj, *tmp; 4363 4364 if (!adev->ras_enabled || !con) { 4365 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4366 amdgpu_release_ras_context(adev); 4367 4368 return; 4369 } 4370 4371 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4372 /* Set up all other IPs which are not implemented. There is a 4373 * tricky thing that IP's actual ras error type should be 4374 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4375 * ERROR_NONE make sense anyway. 4376 */ 4377 amdgpu_ras_enable_all_features(adev, 1); 4378 4379 /* We enable ras on all hw_supported block, but as boot 4380 * parameter might disable some of them and one or more IP has 4381 * not implemented yet. So we disable them on behalf. 4382 */ 4383 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4384 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4385 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4386 /* there should be no any reference. */ 4387 WARN_ON(alive_obj(obj)); 4388 } 4389 } 4390 } 4391 } 4392 4393 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4394 { 4395 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4396 4397 if (!adev->ras_enabled || !con) 4398 return; 4399 4400 amdgpu_ras_disable_all_features(adev, 0); 4401 /* Make sure all ras objects are disabled. */ 4402 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4403 amdgpu_ras_disable_all_features(adev, 1); 4404 } 4405 4406 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4407 { 4408 struct amdgpu_ras_block_list *node, *tmp; 4409 struct amdgpu_ras_block_object *obj; 4410 int r; 4411 4412 amdgpu_ras_event_mgr_init(adev); 4413 4414 if (amdgpu_ras_aca_is_supported(adev)) { 4415 if (amdgpu_reset_in_recovery(adev)) { 4416 if (amdgpu_aca_is_enabled(adev)) 4417 r = amdgpu_aca_reset(adev); 4418 else 4419 r = amdgpu_mca_reset(adev); 4420 if (r) 4421 return r; 4422 } 4423 4424 if (!amdgpu_sriov_vf(adev)) { 4425 if (amdgpu_aca_is_enabled(adev)) 4426 amdgpu_ras_set_aca_debug_mode(adev, false); 4427 else 4428 amdgpu_ras_set_mca_debug_mode(adev, false); 4429 } 4430 } 4431 4432 /* Guest side doesn't need init ras feature */ 4433 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4434 return 0; 4435 4436 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4437 obj = node->ras_obj; 4438 if (!obj) { 4439 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4440 continue; 4441 } 4442 4443 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4444 continue; 4445 4446 if (obj->ras_late_init) { 4447 r = obj->ras_late_init(adev, &obj->ras_comm); 4448 if (r) { 4449 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4450 obj->ras_comm.name, r); 4451 return r; 4452 } 4453 } else 4454 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4455 } 4456 4457 return 0; 4458 } 4459 4460 /* do some fini work before IP fini as dependence */ 4461 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4462 { 4463 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4464 4465 if (!adev->ras_enabled || !con) 4466 return 0; 4467 4468 4469 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4470 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4471 amdgpu_ras_disable_all_features(adev, 0); 4472 amdgpu_ras_recovery_fini(adev); 4473 return 0; 4474 } 4475 4476 int amdgpu_ras_fini(struct amdgpu_device *adev) 4477 { 4478 struct amdgpu_ras_block_list *ras_node, *tmp; 4479 struct amdgpu_ras_block_object *obj = NULL; 4480 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4481 4482 if (!adev->ras_enabled || !con) 4483 return 0; 4484 4485 amdgpu_ras_critical_region_fini(adev); 4486 mutex_destroy(&con->critical_region_lock); 4487 4488 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4489 if (ras_node->ras_obj) { 4490 obj = ras_node->ras_obj; 4491 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4492 obj->ras_fini) 4493 obj->ras_fini(adev, &obj->ras_comm); 4494 else 4495 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4496 } 4497 4498 /* Clear ras blocks from ras_list and free ras block list node */ 4499 list_del(&ras_node->node); 4500 kfree(ras_node); 4501 } 4502 4503 amdgpu_ras_fs_fini(adev); 4504 amdgpu_ras_interrupt_remove_all(adev); 4505 4506 if (amdgpu_ras_aca_is_supported(adev)) { 4507 if (amdgpu_aca_is_enabled(adev)) 4508 amdgpu_aca_fini(adev); 4509 else 4510 amdgpu_mca_fini(adev); 4511 } 4512 4513 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4514 4515 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4516 amdgpu_ras_disable_all_features(adev, 0); 4517 4518 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4519 4520 amdgpu_ras_set_context(adev, NULL); 4521 kfree(con); 4522 4523 return 0; 4524 } 4525 4526 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4527 { 4528 struct amdgpu_ras *ras; 4529 4530 ras = amdgpu_ras_get_context(adev); 4531 if (!ras) 4532 return false; 4533 4534 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4535 } 4536 4537 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4538 { 4539 struct amdgpu_ras *ras; 4540 4541 ras = amdgpu_ras_get_context(adev); 4542 if (ras) { 4543 if (status) 4544 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4545 else 4546 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4547 } 4548 } 4549 4550 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4551 { 4552 struct amdgpu_ras *ras; 4553 4554 ras = amdgpu_ras_get_context(adev); 4555 if (ras) { 4556 ras->ras_err_state = 0; 4557 ras->gpu_reset_flags = 0; 4558 } 4559 } 4560 4561 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4562 enum amdgpu_ras_block block) 4563 { 4564 struct amdgpu_ras *ras; 4565 4566 ras = amdgpu_ras_get_context(adev); 4567 if (ras) 4568 set_bit(block, &ras->ras_err_state); 4569 } 4570 4571 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4572 { 4573 struct amdgpu_ras *ras; 4574 4575 ras = amdgpu_ras_get_context(adev); 4576 if (ras) { 4577 if (block == AMDGPU_RAS_BLOCK__ANY) 4578 return (ras->ras_err_state != 0); 4579 else 4580 return test_bit(block, &ras->ras_err_state) || 4581 test_bit(AMDGPU_RAS_BLOCK__LAST, 4582 &ras->ras_err_state); 4583 } 4584 4585 return false; 4586 } 4587 4588 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4589 { 4590 struct amdgpu_ras *ras; 4591 4592 ras = amdgpu_ras_get_context(adev); 4593 if (!ras) 4594 return NULL; 4595 4596 return ras->event_mgr; 4597 } 4598 4599 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4600 const void *caller) 4601 { 4602 struct ras_event_manager *event_mgr; 4603 struct ras_event_state *event_state; 4604 int ret = 0; 4605 4606 if (amdgpu_uniras_enabled(adev)) 4607 return 0; 4608 4609 if (type >= RAS_EVENT_TYPE_COUNT) { 4610 ret = -EINVAL; 4611 goto out; 4612 } 4613 4614 event_mgr = __get_ras_event_mgr(adev); 4615 if (!event_mgr) { 4616 ret = -EINVAL; 4617 goto out; 4618 } 4619 4620 event_state = &event_mgr->event_state[type]; 4621 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4622 atomic64_inc(&event_state->count); 4623 4624 out: 4625 if (ret && caller) 4626 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4627 (int)type, caller, ret); 4628 4629 return ret; 4630 } 4631 4632 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4633 { 4634 struct ras_event_manager *event_mgr; 4635 u64 id; 4636 4637 if (type >= RAS_EVENT_TYPE_COUNT) 4638 return RAS_EVENT_INVALID_ID; 4639 4640 switch (type) { 4641 case RAS_EVENT_TYPE_FATAL: 4642 case RAS_EVENT_TYPE_POISON_CREATION: 4643 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4644 event_mgr = __get_ras_event_mgr(adev); 4645 if (!event_mgr) 4646 return RAS_EVENT_INVALID_ID; 4647 4648 id = event_mgr->event_state[type].last_seqno; 4649 break; 4650 case RAS_EVENT_TYPE_INVALID: 4651 default: 4652 id = RAS_EVENT_INVALID_ID; 4653 break; 4654 } 4655 4656 return id; 4657 } 4658 4659 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4660 { 4661 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4662 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4663 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4664 u64 event_id = RAS_EVENT_INVALID_ID; 4665 4666 if (amdgpu_uniras_enabled(adev)) 4667 return 0; 4668 4669 if (!amdgpu_ras_mark_ras_event(adev, type)) 4670 event_id = amdgpu_ras_acquire_event_id(adev, type); 4671 4672 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4673 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4674 4675 amdgpu_ras_set_fed(adev, true); 4676 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4677 amdgpu_ras_reset_gpu(adev); 4678 } 4679 4680 return -EBUSY; 4681 } 4682 4683 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4684 { 4685 if (adev->asic_type == CHIP_VEGA20 && 4686 adev->pm.fw_version <= 0x283400) { 4687 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4688 amdgpu_ras_intr_triggered(); 4689 } 4690 4691 return false; 4692 } 4693 4694 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4695 { 4696 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4697 4698 if (!con) 4699 return; 4700 4701 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4702 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4703 amdgpu_ras_set_context(adev, NULL); 4704 kfree(con); 4705 } 4706 } 4707 4708 #ifdef CONFIG_X86_MCE_AMD 4709 static struct amdgpu_device *find_adev(uint32_t node_id) 4710 { 4711 int i; 4712 struct amdgpu_device *adev = NULL; 4713 4714 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4715 adev = mce_adev_list.devs[i]; 4716 4717 if (adev && adev->gmc.xgmi.connected_to_cpu && 4718 adev->gmc.xgmi.physical_node_id == node_id) 4719 break; 4720 adev = NULL; 4721 } 4722 4723 return adev; 4724 } 4725 4726 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4727 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4728 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4729 #define GPU_ID_OFFSET 8 4730 4731 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4732 unsigned long val, void *data) 4733 { 4734 struct mce *m = (struct mce *)data; 4735 struct amdgpu_device *adev = NULL; 4736 uint32_t gpu_id = 0; 4737 uint32_t umc_inst = 0, ch_inst = 0; 4738 4739 /* 4740 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4741 * and error occurred in DramECC (Extended error code = 0) then only 4742 * process the error, else bail out. 4743 */ 4744 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4745 (XEC(m->status, 0x3f) == 0x0))) 4746 return NOTIFY_DONE; 4747 4748 /* 4749 * If it is correctable error, return. 4750 */ 4751 if (mce_is_correctable(m)) 4752 return NOTIFY_OK; 4753 4754 /* 4755 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4756 */ 4757 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4758 4759 adev = find_adev(gpu_id); 4760 if (!adev) { 4761 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4762 gpu_id); 4763 return NOTIFY_DONE; 4764 } 4765 4766 /* 4767 * If it is uncorrectable error, then find out UMC instance and 4768 * channel index. 4769 */ 4770 umc_inst = GET_UMC_INST(m->ipid); 4771 ch_inst = GET_CHAN_INDEX(m->ipid); 4772 4773 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4774 umc_inst, ch_inst); 4775 4776 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4777 return NOTIFY_OK; 4778 else 4779 return NOTIFY_DONE; 4780 } 4781 4782 static struct notifier_block amdgpu_bad_page_nb = { 4783 .notifier_call = amdgpu_bad_page_notifier, 4784 .priority = MCE_PRIO_UC, 4785 }; 4786 4787 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4788 { 4789 /* 4790 * Add the adev to the mce_adev_list. 4791 * During mode2 reset, amdgpu device is temporarily 4792 * removed from the mgpu_info list which can cause 4793 * page retirement to fail. 4794 * Use this list instead of mgpu_info to find the amdgpu 4795 * device on which the UMC error was reported. 4796 */ 4797 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4798 4799 /* 4800 * Register the x86 notifier only once 4801 * with MCE subsystem. 4802 */ 4803 if (notifier_registered == false) { 4804 mce_register_decode_chain(&amdgpu_bad_page_nb); 4805 notifier_registered = true; 4806 } 4807 } 4808 #endif 4809 4810 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4811 { 4812 if (!adev) 4813 return NULL; 4814 4815 return adev->psp.ras_context.ras; 4816 } 4817 4818 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4819 { 4820 if (!adev) 4821 return -EINVAL; 4822 4823 adev->psp.ras_context.ras = ras_con; 4824 return 0; 4825 } 4826 4827 /* check if ras is supported on block, say, sdma, gfx */ 4828 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4829 unsigned int block) 4830 { 4831 int ret = 0; 4832 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4833 4834 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4835 return 0; 4836 4837 ret = ras && (adev->ras_enabled & (1 << block)); 4838 4839 /* For the special asic with mem ecc enabled but sram ecc 4840 * not enabled, even if the ras block is not supported on 4841 * .ras_enabled, if the asic supports poison mode and the 4842 * ras block has ras configuration, it can be considered 4843 * that the ras block supports ras function. 4844 */ 4845 if (!ret && 4846 (block == AMDGPU_RAS_BLOCK__GFX || 4847 block == AMDGPU_RAS_BLOCK__SDMA || 4848 block == AMDGPU_RAS_BLOCK__VCN || 4849 block == AMDGPU_RAS_BLOCK__JPEG) && 4850 (amdgpu_ras_mask & (1 << block)) && 4851 amdgpu_ras_is_poison_mode_supported(adev) && 4852 amdgpu_ras_get_ras_block(adev, block, 0)) 4853 ret = 1; 4854 4855 return ret; 4856 } 4857 4858 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 4859 { 4860 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4861 4862 /* mode1 is the only selection for RMA status */ 4863 if (amdgpu_ras_is_rma(adev)) { 4864 ras->gpu_reset_flags = 0; 4865 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4866 } 4867 4868 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 4869 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4870 int hive_ras_recovery = 0; 4871 4872 if (hive) { 4873 hive_ras_recovery = atomic_read(&hive->ras_recovery); 4874 amdgpu_put_xgmi_hive(hive); 4875 } 4876 /* In the case of multiple GPUs, after a GPU has started 4877 * resetting all GPUs on hive, other GPUs do not need to 4878 * trigger GPU reset again. 4879 */ 4880 if (!hive_ras_recovery) 4881 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4882 else 4883 atomic_set(&ras->in_recovery, 0); 4884 } else { 4885 flush_work(&ras->recovery_work); 4886 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4887 } 4888 4889 return 0; 4890 } 4891 4892 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 4893 { 4894 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4895 int ret = 0; 4896 4897 if (con) { 4898 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4899 if (!ret) 4900 con->is_aca_debug_mode = enable; 4901 } 4902 4903 return ret; 4904 } 4905 4906 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 4907 { 4908 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4909 int ret = 0; 4910 4911 if (con) { 4912 if (amdgpu_aca_is_enabled(adev)) 4913 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 4914 else 4915 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4916 if (!ret) 4917 con->is_aca_debug_mode = enable; 4918 } 4919 4920 return ret; 4921 } 4922 4923 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 4924 { 4925 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4926 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4927 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4928 4929 if (!con) 4930 return false; 4931 4932 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 4933 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 4934 return con->is_aca_debug_mode; 4935 else 4936 return true; 4937 } 4938 4939 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 4940 unsigned int *error_query_mode) 4941 { 4942 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4943 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4944 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4945 4946 if (!con) { 4947 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 4948 return false; 4949 } 4950 4951 if (amdgpu_sriov_vf(adev)) { 4952 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 4953 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 4954 *error_query_mode = 4955 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 4956 } else { 4957 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 4958 } 4959 4960 return true; 4961 } 4962 4963 /* Register each ip ras block into amdgpu ras */ 4964 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 4965 struct amdgpu_ras_block_object *ras_block_obj) 4966 { 4967 struct amdgpu_ras_block_list *ras_node; 4968 if (!adev || !ras_block_obj) 4969 return -EINVAL; 4970 4971 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 4972 if (!ras_node) 4973 return -ENOMEM; 4974 4975 INIT_LIST_HEAD(&ras_node->node); 4976 ras_node->ras_obj = ras_block_obj; 4977 list_add_tail(&ras_node->node, &adev->ras_list); 4978 4979 return 0; 4980 } 4981 4982 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 4983 { 4984 if (!err_type_name) 4985 return; 4986 4987 switch (err_type) { 4988 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 4989 sprintf(err_type_name, "correctable"); 4990 break; 4991 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 4992 sprintf(err_type_name, "uncorrectable"); 4993 break; 4994 default: 4995 sprintf(err_type_name, "unknown"); 4996 break; 4997 } 4998 } 4999 5000 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5001 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5002 uint32_t instance, 5003 uint32_t *memory_id) 5004 { 5005 uint32_t err_status_lo_data, err_status_lo_offset; 5006 5007 if (!reg_entry) 5008 return false; 5009 5010 err_status_lo_offset = 5011 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5012 reg_entry->seg_lo, reg_entry->reg_lo); 5013 err_status_lo_data = RREG32(err_status_lo_offset); 5014 5015 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5016 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5017 return false; 5018 5019 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5020 5021 return true; 5022 } 5023 5024 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5025 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5026 uint32_t instance, 5027 unsigned long *err_cnt) 5028 { 5029 uint32_t err_status_hi_data, err_status_hi_offset; 5030 5031 if (!reg_entry) 5032 return false; 5033 5034 err_status_hi_offset = 5035 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5036 reg_entry->seg_hi, reg_entry->reg_hi); 5037 err_status_hi_data = RREG32(err_status_hi_offset); 5038 5039 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5040 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5041 /* keep the check here in case we need to refer to the result later */ 5042 dev_dbg(adev->dev, "Invalid err_info field\n"); 5043 5044 /* read err count */ 5045 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5046 5047 return true; 5048 } 5049 5050 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5051 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5052 uint32_t reg_list_size, 5053 const struct amdgpu_ras_memory_id_entry *mem_list, 5054 uint32_t mem_list_size, 5055 uint32_t instance, 5056 uint32_t err_type, 5057 unsigned long *err_count) 5058 { 5059 uint32_t memory_id; 5060 unsigned long err_cnt; 5061 char err_type_name[16]; 5062 uint32_t i, j; 5063 5064 for (i = 0; i < reg_list_size; i++) { 5065 /* query memory_id from err_status_lo */ 5066 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5067 instance, &memory_id)) 5068 continue; 5069 5070 /* query err_cnt from err_status_hi */ 5071 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5072 instance, &err_cnt) || 5073 !err_cnt) 5074 continue; 5075 5076 *err_count += err_cnt; 5077 5078 /* log the errors */ 5079 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5080 if (!mem_list) { 5081 /* memory_list is not supported */ 5082 dev_info(adev->dev, 5083 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5084 err_cnt, err_type_name, 5085 reg_list[i].block_name, 5086 instance, memory_id); 5087 } else { 5088 for (j = 0; j < mem_list_size; j++) { 5089 if (memory_id == mem_list[j].memory_id) { 5090 dev_info(adev->dev, 5091 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5092 err_cnt, err_type_name, 5093 reg_list[i].block_name, 5094 instance, mem_list[j].name); 5095 break; 5096 } 5097 } 5098 } 5099 } 5100 } 5101 5102 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5103 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5104 uint32_t reg_list_size, 5105 uint32_t instance) 5106 { 5107 uint32_t err_status_lo_offset, err_status_hi_offset; 5108 uint32_t i; 5109 5110 for (i = 0; i < reg_list_size; i++) { 5111 err_status_lo_offset = 5112 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5113 reg_list[i].seg_lo, reg_list[i].reg_lo); 5114 err_status_hi_offset = 5115 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5116 reg_list[i].seg_hi, reg_list[i].reg_hi); 5117 WREG32(err_status_lo_offset, 0); 5118 WREG32(err_status_hi_offset, 0); 5119 } 5120 } 5121 5122 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5123 { 5124 memset(err_data, 0, sizeof(*err_data)); 5125 5126 INIT_LIST_HEAD(&err_data->err_node_list); 5127 5128 return 0; 5129 } 5130 5131 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5132 { 5133 if (!err_node) 5134 return; 5135 5136 list_del(&err_node->node); 5137 kvfree(err_node); 5138 } 5139 5140 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5141 { 5142 struct ras_err_node *err_node, *tmp; 5143 5144 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5145 amdgpu_ras_error_node_release(err_node); 5146 } 5147 5148 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5149 struct amdgpu_smuio_mcm_config_info *mcm_info) 5150 { 5151 struct ras_err_node *err_node; 5152 struct amdgpu_smuio_mcm_config_info *ref_id; 5153 5154 if (!err_data || !mcm_info) 5155 return NULL; 5156 5157 for_each_ras_error(err_node, err_data) { 5158 ref_id = &err_node->err_info.mcm_info; 5159 5160 if (mcm_info->socket_id == ref_id->socket_id && 5161 mcm_info->die_id == ref_id->die_id) 5162 return err_node; 5163 } 5164 5165 return NULL; 5166 } 5167 5168 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5169 { 5170 struct ras_err_node *err_node; 5171 5172 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5173 if (!err_node) 5174 return NULL; 5175 5176 INIT_LIST_HEAD(&err_node->node); 5177 5178 return err_node; 5179 } 5180 5181 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5182 { 5183 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5184 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5185 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5186 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5187 5188 if (unlikely(infoa->socket_id != infob->socket_id)) 5189 return infoa->socket_id - infob->socket_id; 5190 else 5191 return infoa->die_id - infob->die_id; 5192 5193 return 0; 5194 } 5195 5196 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5197 struct amdgpu_smuio_mcm_config_info *mcm_info) 5198 { 5199 struct ras_err_node *err_node; 5200 5201 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5202 if (err_node) 5203 return &err_node->err_info; 5204 5205 err_node = amdgpu_ras_error_node_new(); 5206 if (!err_node) 5207 return NULL; 5208 5209 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5210 5211 err_data->err_list_count++; 5212 list_add_tail(&err_node->node, &err_data->err_node_list); 5213 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5214 5215 return &err_node->err_info; 5216 } 5217 5218 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5219 struct amdgpu_smuio_mcm_config_info *mcm_info, 5220 u64 count) 5221 { 5222 struct ras_err_info *err_info; 5223 5224 if (!err_data || !mcm_info) 5225 return -EINVAL; 5226 5227 if (!count) 5228 return 0; 5229 5230 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5231 if (!err_info) 5232 return -EINVAL; 5233 5234 err_info->ue_count += count; 5235 err_data->ue_count += count; 5236 5237 return 0; 5238 } 5239 5240 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5241 struct amdgpu_smuio_mcm_config_info *mcm_info, 5242 u64 count) 5243 { 5244 struct ras_err_info *err_info; 5245 5246 if (!err_data || !mcm_info) 5247 return -EINVAL; 5248 5249 if (!count) 5250 return 0; 5251 5252 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5253 if (!err_info) 5254 return -EINVAL; 5255 5256 err_info->ce_count += count; 5257 err_data->ce_count += count; 5258 5259 return 0; 5260 } 5261 5262 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5263 struct amdgpu_smuio_mcm_config_info *mcm_info, 5264 u64 count) 5265 { 5266 struct ras_err_info *err_info; 5267 5268 if (!err_data || !mcm_info) 5269 return -EINVAL; 5270 5271 if (!count) 5272 return 0; 5273 5274 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5275 if (!err_info) 5276 return -EINVAL; 5277 5278 err_info->de_count += count; 5279 err_data->de_count += count; 5280 5281 return 0; 5282 } 5283 5284 #define mmMP0_SMN_C2PMSG_92 0x1609C 5285 #define mmMP0_SMN_C2PMSG_126 0x160BE 5286 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5287 u32 instance) 5288 { 5289 u32 socket_id, aid_id, hbm_id; 5290 u32 fw_status; 5291 u32 boot_error; 5292 u64 reg_addr; 5293 5294 /* The pattern for smn addressing in other SOC could be different from 5295 * the one for aqua_vanjaram. We should revisit the code if the pattern 5296 * is changed. In such case, replace the aqua_vanjaram implementation 5297 * with more common helper */ 5298 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5299 aqua_vanjaram_encode_ext_smn_addressing(instance); 5300 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5301 5302 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5303 aqua_vanjaram_encode_ext_smn_addressing(instance); 5304 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5305 5306 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5307 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5308 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5309 5310 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5311 dev_info(adev->dev, 5312 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5313 socket_id, aid_id, hbm_id, fw_status); 5314 5315 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5316 dev_info(adev->dev, 5317 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5318 socket_id, aid_id, fw_status); 5319 5320 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5321 dev_info(adev->dev, 5322 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5323 socket_id, aid_id, fw_status); 5324 5325 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5326 dev_info(adev->dev, 5327 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5328 socket_id, aid_id, fw_status); 5329 5330 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5331 dev_info(adev->dev, 5332 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5333 socket_id, aid_id, fw_status); 5334 5335 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5336 dev_info(adev->dev, 5337 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5338 socket_id, aid_id, fw_status); 5339 5340 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5341 dev_info(adev->dev, 5342 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5343 socket_id, aid_id, hbm_id, fw_status); 5344 5345 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5346 dev_info(adev->dev, 5347 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5348 socket_id, aid_id, hbm_id, fw_status); 5349 5350 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5351 dev_info(adev->dev, 5352 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5353 socket_id, aid_id, fw_status); 5354 5355 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5356 dev_info(adev->dev, 5357 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5358 socket_id, aid_id, fw_status); 5359 } 5360 5361 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5362 u32 instance) 5363 { 5364 u64 reg_addr; 5365 u32 reg_data; 5366 int retry_loop; 5367 5368 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5369 aqua_vanjaram_encode_ext_smn_addressing(instance); 5370 5371 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5372 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5373 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5374 return false; 5375 else 5376 msleep(1); 5377 } 5378 5379 return true; 5380 } 5381 5382 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5383 { 5384 u32 i; 5385 5386 for (i = 0; i < num_instances; i++) { 5387 if (amdgpu_ras_boot_error_detected(adev, i)) 5388 amdgpu_ras_boot_time_error_reporting(adev, i); 5389 } 5390 } 5391 5392 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5393 { 5394 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5395 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5396 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5397 int ret = 0; 5398 5399 if (amdgpu_ras_check_critical_address(adev, start)) 5400 return 0; 5401 5402 mutex_lock(&con->page_rsv_lock); 5403 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5404 if (ret == -ENOENT) 5405 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5406 mutex_unlock(&con->page_rsv_lock); 5407 5408 return ret; 5409 } 5410 5411 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5412 const char *fmt, ...) 5413 { 5414 struct va_format vaf; 5415 va_list args; 5416 5417 va_start(args, fmt); 5418 vaf.fmt = fmt; 5419 vaf.va = &args; 5420 5421 if (RAS_EVENT_ID_IS_VALID(event_id)) 5422 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5423 else 5424 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5425 5426 va_end(args); 5427 } 5428 5429 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5430 { 5431 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5432 5433 if (amdgpu_uniras_enabled(adev)) 5434 return amdgpu_ras_mgr_is_rma(adev); 5435 5436 if (!con) 5437 return false; 5438 5439 return con->is_rma; 5440 } 5441 5442 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5443 struct amdgpu_bo *bo) 5444 { 5445 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5446 struct amdgpu_vram_mgr_resource *vres; 5447 struct ras_critical_region *region; 5448 struct drm_buddy_block *block; 5449 int ret = 0; 5450 5451 if (!bo || !bo->tbo.resource) 5452 return -EINVAL; 5453 5454 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5455 5456 mutex_lock(&con->critical_region_lock); 5457 5458 /* Check if the bo had been recorded */ 5459 list_for_each_entry(region, &con->critical_region_head, node) 5460 if (region->bo == bo) 5461 goto out; 5462 5463 /* Record new critical amdgpu bo */ 5464 list_for_each_entry(block, &vres->blocks, link) { 5465 region = kzalloc(sizeof(*region), GFP_KERNEL); 5466 if (!region) { 5467 ret = -ENOMEM; 5468 goto out; 5469 } 5470 region->bo = bo; 5471 region->start = amdgpu_vram_mgr_block_start(block); 5472 region->size = amdgpu_vram_mgr_block_size(block); 5473 list_add_tail(®ion->node, &con->critical_region_head); 5474 } 5475 5476 out: 5477 mutex_unlock(&con->critical_region_lock); 5478 5479 return ret; 5480 } 5481 5482 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5483 { 5484 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5485 } 5486 5487 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5488 { 5489 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5490 struct ras_critical_region *region, *tmp; 5491 5492 mutex_lock(&con->critical_region_lock); 5493 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5494 list_del(®ion->node); 5495 kfree(region); 5496 } 5497 mutex_unlock(&con->critical_region_lock); 5498 } 5499 5500 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5501 { 5502 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5503 struct ras_critical_region *region; 5504 bool ret = false; 5505 5506 mutex_lock(&con->critical_region_lock); 5507 list_for_each_entry(region, &con->critical_region_head, node) { 5508 if ((region->start <= addr) && 5509 (addr < (region->start + region->size))) { 5510 ret = true; 5511 break; 5512 } 5513 } 5514 mutex_unlock(&con->critical_region_lock); 5515 5516 return ret; 5517 } 5518