1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 45 #ifdef CONFIG_X86_MCE_AMD 46 #include <asm/mce.h> 47 48 static bool notifier_registered; 49 #endif 50 static const char *RAS_FS_NAME = "ras"; 51 52 const char *ras_error_string[] = { 53 "none", 54 "parity", 55 "single_correctable", 56 "multi_uncorrectable", 57 "poison", 58 }; 59 60 const char *ras_block_string[] = { 61 "umc", 62 "sdma", 63 "gfx", 64 "mmhub", 65 "athub", 66 "pcie_bif", 67 "hdp", 68 "xgmi_wafl", 69 "df", 70 "smn", 71 "sem", 72 "mp0", 73 "mp1", 74 "fuse", 75 "mca", 76 "vcn", 77 "jpeg", 78 "ih", 79 "mpio", 80 "mmsch", 81 }; 82 83 const char *ras_mca_block_string[] = { 84 "mca_mp0", 85 "mca_mp1", 86 "mca_mpio", 87 "mca_iohc", 88 }; 89 90 struct amdgpu_ras_block_list { 91 /* ras block link */ 92 struct list_head node; 93 94 struct amdgpu_ras_block_object *ras_obj; 95 }; 96 97 const char *get_ras_block_str(struct ras_common_if *ras_block) 98 { 99 if (!ras_block) 100 return "NULL"; 101 102 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 103 ras_block->block >= ARRAY_SIZE(ras_block_string)) 104 return "OUT OF RANGE"; 105 106 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 107 return ras_mca_block_string[ras_block->sub_block_index]; 108 109 return ras_block_string[ras_block->block]; 110 } 111 112 #define ras_block_str(_BLOCK_) \ 113 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 114 115 #define ras_err_str(i) (ras_error_string[ffs(i)]) 116 117 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 118 119 /* inject address is 52 bits */ 120 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 121 122 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 123 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 124 125 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 126 127 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 128 129 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 130 131 #define BYPASS_ALLOCATED_ADDRESS 0x0 132 #define BYPASS_INITIALIZATION_ADDRESS 0x1 133 134 enum amdgpu_ras_retire_page_reservation { 135 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 136 AMDGPU_RAS_RETIRE_PAGE_PENDING, 137 AMDGPU_RAS_RETIRE_PAGE_FAULT, 138 }; 139 140 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 141 142 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 143 uint64_t addr); 144 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 145 uint64_t addr); 146 147 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 148 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 149 150 #ifdef CONFIG_X86_MCE_AMD 151 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 152 struct mce_notifier_adev_list { 153 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 154 int num_gpu; 155 }; 156 static struct mce_notifier_adev_list mce_adev_list; 157 #endif 158 159 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 160 { 161 if (adev && amdgpu_ras_get_context(adev)) 162 amdgpu_ras_get_context(adev)->error_query_ready = ready; 163 } 164 165 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 166 { 167 if (adev && amdgpu_ras_get_context(adev)) 168 return amdgpu_ras_get_context(adev)->error_query_ready; 169 170 return false; 171 } 172 173 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 174 { 175 struct ras_err_data err_data; 176 struct eeprom_table_record err_rec; 177 int ret; 178 179 ret = amdgpu_ras_check_bad_page(adev, address); 180 if (ret == -EINVAL) { 181 dev_warn(adev->dev, 182 "RAS WARN: input address 0x%llx is invalid.\n", 183 address); 184 return -EINVAL; 185 } else if (ret == 1) { 186 dev_warn(adev->dev, 187 "RAS WARN: 0x%llx has already been marked as bad page!\n", 188 address); 189 return 0; 190 } 191 192 ret = amdgpu_ras_error_data_init(&err_data); 193 if (ret) 194 return ret; 195 196 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 197 err_data.err_addr = &err_rec; 198 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 199 200 if (amdgpu_bad_page_threshold != 0) { 201 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 202 err_data.err_addr_cnt, false); 203 amdgpu_ras_save_bad_pages(adev, NULL); 204 } 205 206 amdgpu_ras_error_data_fini(&err_data); 207 208 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 209 dev_warn(adev->dev, "Clear EEPROM:\n"); 210 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 211 212 return 0; 213 } 214 215 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 216 uint64_t address, uint64_t flags) 217 { 218 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 219 struct amdgpu_vram_block_info blk_info; 220 uint64_t page_pfns[32] = {0}; 221 int i, ret, count; 222 bool hit = false; 223 224 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 225 return 0; 226 227 if (amdgpu_sriov_vf(adev)) { 228 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 229 return -EPERM; 230 return hit ? -EACCES : 0; 231 } 232 233 if ((address >= adev->gmc.mc_vram_size) || 234 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 235 return -EFAULT; 236 237 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 238 address, page_pfns, ARRAY_SIZE(page_pfns)); 239 if (count <= 0) 240 return -EPERM; 241 242 for (i = 0; i < count; i++) { 243 memset(&blk_info, 0, sizeof(blk_info)); 244 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 245 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 246 if (!ret) { 247 /* The input address that needs to be checked is allocated by 248 * current calling process, so it is necessary to exclude 249 * the calling process. 250 */ 251 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 252 ((blk_info.task.pid != task_pid_nr(current)) || 253 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 254 return -EACCES; 255 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 256 (blk_info.task.pid == con->init_task_pid) && 257 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 258 return -EACCES; 259 } 260 } 261 262 return 0; 263 } 264 265 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 266 size_t size, loff_t *pos) 267 { 268 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 269 struct ras_query_if info = { 270 .head = obj->head, 271 }; 272 ssize_t s; 273 char val[128]; 274 275 if (amdgpu_ras_query_error_status(obj->adev, &info)) 276 return -EINVAL; 277 278 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 279 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 280 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 281 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 282 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 283 } 284 285 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 286 "ue", info.ue_count, 287 "ce", info.ce_count); 288 if (*pos >= s) 289 return 0; 290 291 s -= *pos; 292 s = min_t(u64, s, size); 293 294 295 if (copy_to_user(buf, &val[*pos], s)) 296 return -EINVAL; 297 298 *pos += s; 299 300 return s; 301 } 302 303 static const struct file_operations amdgpu_ras_debugfs_ops = { 304 .owner = THIS_MODULE, 305 .read = amdgpu_ras_debugfs_read, 306 .write = NULL, 307 .llseek = default_llseek 308 }; 309 310 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 311 { 312 int i; 313 314 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 315 *block_id = i; 316 if (strcmp(name, ras_block_string[i]) == 0) 317 return 0; 318 } 319 return -EINVAL; 320 } 321 322 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 323 const char __user *buf, size_t size, 324 loff_t *pos, struct ras_debug_if *data) 325 { 326 ssize_t s = min_t(u64, 64, size); 327 char str[65]; 328 char block_name[33]; 329 char err[9] = "ue"; 330 int op = -1; 331 int block_id; 332 uint32_t sub_block; 333 u64 address, value; 334 /* default value is 0 if the mask is not set by user */ 335 u32 instance_mask = 0; 336 337 if (*pos) 338 return -EINVAL; 339 *pos = size; 340 341 memset(str, 0, sizeof(str)); 342 memset(data, 0, sizeof(*data)); 343 344 if (copy_from_user(str, buf, s)) 345 return -EINVAL; 346 347 if (sscanf(str, "disable %32s", block_name) == 1) 348 op = 0; 349 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 350 op = 1; 351 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 352 op = 2; 353 else if (strstr(str, "retire_page") != NULL) 354 op = 3; 355 else if (strstr(str, "check_address") != NULL) 356 op = 4; 357 else if (str[0] && str[1] && str[2] && str[3]) 358 /* ascii string, but commands are not matched. */ 359 return -EINVAL; 360 361 if (op != -1) { 362 if (op == 3) { 363 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 364 sscanf(str, "%*s %llu", &address) != 1) 365 return -EINVAL; 366 367 data->op = op; 368 data->inject.address = address; 369 370 return 0; 371 } else if (op == 4) { 372 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 373 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 374 return -EINVAL; 375 376 data->op = op; 377 data->inject.address = address; 378 data->inject.value = value; 379 return 0; 380 } 381 382 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 383 return -EINVAL; 384 385 data->head.block = block_id; 386 /* only ue, ce and poison errors are supported */ 387 if (!memcmp("ue", err, 2)) 388 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 389 else if (!memcmp("ce", err, 2)) 390 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 391 else if (!memcmp("poison", err, 6)) 392 data->head.type = AMDGPU_RAS_ERROR__POISON; 393 else 394 return -EINVAL; 395 396 data->op = op; 397 398 if (op == 2) { 399 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 400 &sub_block, &address, &value, &instance_mask) != 4 && 401 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 402 &sub_block, &address, &value, &instance_mask) != 4 && 403 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 404 &sub_block, &address, &value) != 3 && 405 sscanf(str, "%*s %*s %*s %u %llu %llu", 406 &sub_block, &address, &value) != 3) 407 return -EINVAL; 408 data->head.sub_block_index = sub_block; 409 data->inject.address = address; 410 data->inject.value = value; 411 data->inject.instance_mask = instance_mask; 412 } 413 } else { 414 if (size < sizeof(*data)) 415 return -EINVAL; 416 417 if (copy_from_user(data, buf, sizeof(*data))) 418 return -EINVAL; 419 } 420 421 return 0; 422 } 423 424 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 425 struct ras_debug_if *data) 426 { 427 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 428 uint32_t mask, inst_mask = data->inject.instance_mask; 429 430 /* no need to set instance mask if there is only one instance */ 431 if (num_xcc <= 1 && inst_mask) { 432 data->inject.instance_mask = 0; 433 dev_dbg(adev->dev, 434 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 435 inst_mask); 436 437 return; 438 } 439 440 switch (data->head.block) { 441 case AMDGPU_RAS_BLOCK__GFX: 442 mask = GENMASK(num_xcc - 1, 0); 443 break; 444 case AMDGPU_RAS_BLOCK__SDMA: 445 mask = GENMASK(adev->sdma.num_instances - 1, 0); 446 break; 447 case AMDGPU_RAS_BLOCK__VCN: 448 case AMDGPU_RAS_BLOCK__JPEG: 449 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 450 break; 451 default: 452 mask = inst_mask; 453 break; 454 } 455 456 /* remove invalid bits in instance mask */ 457 data->inject.instance_mask &= mask; 458 if (inst_mask != data->inject.instance_mask) 459 dev_dbg(adev->dev, 460 "Adjust RAS inject mask 0x%x to 0x%x\n", 461 inst_mask, data->inject.instance_mask); 462 } 463 464 /** 465 * DOC: AMDGPU RAS debugfs control interface 466 * 467 * The control interface accepts struct ras_debug_if which has two members. 468 * 469 * First member: ras_debug_if::head or ras_debug_if::inject. 470 * 471 * head is used to indicate which IP block will be under control. 472 * 473 * head has four members, they are block, type, sub_block_index, name. 474 * block: which IP will be under control. 475 * type: what kind of error will be enabled/disabled/injected. 476 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 477 * name: the name of IP. 478 * 479 * inject has three more members than head, they are address, value and mask. 480 * As their names indicate, inject operation will write the 481 * value to the address. 482 * 483 * The second member: struct ras_debug_if::op. 484 * It has three kinds of operations. 485 * 486 * - 0: disable RAS on the block. Take ::head as its data. 487 * - 1: enable RAS on the block. Take ::head as its data. 488 * - 2: inject errors on the block. Take ::inject as its data. 489 * 490 * How to use the interface? 491 * 492 * In a program 493 * 494 * Copy the struct ras_debug_if in your code and initialize it. 495 * Write the struct to the control interface. 496 * 497 * From shell 498 * 499 * .. code-block:: bash 500 * 501 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 502 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * 505 * Where N, is the card which you want to affect. 506 * 507 * "disable" requires only the block. 508 * "enable" requires the block and error type. 509 * "inject" requires the block, error type, address, and value. 510 * 511 * The block is one of: umc, sdma, gfx, etc. 512 * see ras_block_string[] for details 513 * 514 * The error type is one of: ue, ce and poison where, 515 * ue is multi-uncorrectable 516 * ce is single-correctable 517 * poison is poison 518 * 519 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 520 * The address and value are hexadecimal numbers, leading 0x is optional. 521 * The mask means instance mask, is optional, default value is 0x1. 522 * 523 * For instance, 524 * 525 * .. code-block:: bash 526 * 527 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 528 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * 531 * How to check the result of the operation? 532 * 533 * To check disable/enable, see "ras" features at, 534 * /sys/class/drm/card[0/1/2...]/device/ras/features 535 * 536 * To check inject, see the corresponding error count at, 537 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 538 * 539 * .. note:: 540 * Operations are only allowed on blocks which are supported. 541 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 542 * to see which blocks support RAS on a particular asic. 543 * 544 */ 545 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 546 const char __user *buf, 547 size_t size, loff_t *pos) 548 { 549 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 550 struct ras_debug_if data; 551 int ret = 0; 552 553 if (!amdgpu_ras_get_error_query_ready(adev)) { 554 dev_warn(adev->dev, "RAS WARN: error injection " 555 "currently inaccessible\n"); 556 return size; 557 } 558 559 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 560 if (ret) 561 return ret; 562 563 if (data.op == 3) { 564 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 565 if (!ret) 566 return size; 567 else 568 return ret; 569 } else if (data.op == 4) { 570 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 571 return ret ? ret : size; 572 } 573 574 if (!amdgpu_ras_is_supported(adev, data.head.block)) 575 return -EINVAL; 576 577 switch (data.op) { 578 case 0: 579 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 580 break; 581 case 1: 582 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 583 break; 584 case 2: 585 /* umc ce/ue error injection for a bad page is not allowed */ 586 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 587 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 588 if (ret == -EINVAL) { 589 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 590 data.inject.address); 591 break; 592 } else if (ret == 1) { 593 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 594 data.inject.address); 595 break; 596 } 597 598 amdgpu_ras_instance_mask_check(adev, &data); 599 600 /* data.inject.address is offset instead of absolute gpu address */ 601 ret = amdgpu_ras_error_inject(adev, &data.inject); 602 break; 603 default: 604 ret = -EINVAL; 605 break; 606 } 607 608 if (ret) 609 return ret; 610 611 return size; 612 } 613 614 /** 615 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 616 * 617 * Some boards contain an EEPROM which is used to persistently store a list of 618 * bad pages which experiences ECC errors in vram. This interface provides 619 * a way to reset the EEPROM, e.g., after testing error injection. 620 * 621 * Usage: 622 * 623 * .. code-block:: bash 624 * 625 * echo 1 > ../ras/ras_eeprom_reset 626 * 627 * will reset EEPROM table to 0 entries. 628 * 629 */ 630 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 631 const char __user *buf, 632 size_t size, loff_t *pos) 633 { 634 struct amdgpu_device *adev = 635 (struct amdgpu_device *)file_inode(f)->i_private; 636 int ret; 637 638 ret = amdgpu_ras_eeprom_reset_table( 639 &(amdgpu_ras_get_context(adev)->eeprom_control)); 640 641 if (!ret) { 642 /* Something was written to EEPROM. 643 */ 644 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 645 return size; 646 } else { 647 return ret; 648 } 649 } 650 651 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 652 .owner = THIS_MODULE, 653 .read = NULL, 654 .write = amdgpu_ras_debugfs_ctrl_write, 655 .llseek = default_llseek 656 }; 657 658 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 659 .owner = THIS_MODULE, 660 .read = NULL, 661 .write = amdgpu_ras_debugfs_eeprom_write, 662 .llseek = default_llseek 663 }; 664 665 /** 666 * DOC: AMDGPU RAS sysfs Error Count Interface 667 * 668 * It allows the user to read the error count for each IP block on the gpu through 669 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 670 * 671 * It outputs the multiple lines which report the uncorrected (ue) and corrected 672 * (ce) error counts. 673 * 674 * The format of one line is below, 675 * 676 * [ce|ue]: count 677 * 678 * Example: 679 * 680 * .. code-block:: bash 681 * 682 * ue: 0 683 * ce: 1 684 * 685 */ 686 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 687 struct device_attribute *attr, char *buf) 688 { 689 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 690 struct ras_query_if info = { 691 .head = obj->head, 692 }; 693 694 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 695 return sysfs_emit(buf, "Query currently inaccessible\n"); 696 697 if (amdgpu_ras_query_error_status(obj->adev, &info)) 698 return -EINVAL; 699 700 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 701 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 702 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 703 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 704 } 705 706 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 707 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 708 "ce", info.ce_count, "de", info.de_count); 709 else 710 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 711 "ce", info.ce_count); 712 } 713 714 /* obj begin */ 715 716 #define get_obj(obj) do { (obj)->use++; } while (0) 717 #define alive_obj(obj) ((obj)->use) 718 719 static inline void put_obj(struct ras_manager *obj) 720 { 721 if (obj && (--obj->use == 0)) { 722 list_del(&obj->node); 723 amdgpu_ras_error_data_fini(&obj->err_data); 724 } 725 726 if (obj && (obj->use < 0)) 727 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 728 } 729 730 /* make one obj and return it. */ 731 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 732 struct ras_common_if *head) 733 { 734 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 735 struct ras_manager *obj; 736 737 if (!adev->ras_enabled || !con) 738 return NULL; 739 740 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 741 return NULL; 742 743 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 744 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 745 return NULL; 746 747 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 748 } else 749 obj = &con->objs[head->block]; 750 751 /* already exist. return obj? */ 752 if (alive_obj(obj)) 753 return NULL; 754 755 if (amdgpu_ras_error_data_init(&obj->err_data)) 756 return NULL; 757 758 obj->head = *head; 759 obj->adev = adev; 760 list_add(&obj->node, &con->head); 761 get_obj(obj); 762 763 return obj; 764 } 765 766 /* return an obj equal to head, or the first when head is NULL */ 767 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 768 struct ras_common_if *head) 769 { 770 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 771 struct ras_manager *obj; 772 int i; 773 774 if (!adev->ras_enabled || !con) 775 return NULL; 776 777 if (head) { 778 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 779 return NULL; 780 781 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 782 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 783 return NULL; 784 785 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 786 } else 787 obj = &con->objs[head->block]; 788 789 if (alive_obj(obj)) 790 return obj; 791 } else { 792 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 793 obj = &con->objs[i]; 794 if (alive_obj(obj)) 795 return obj; 796 } 797 } 798 799 return NULL; 800 } 801 /* obj end */ 802 803 /* feature ctl begin */ 804 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 805 struct ras_common_if *head) 806 { 807 return adev->ras_hw_enabled & BIT(head->block); 808 } 809 810 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 811 struct ras_common_if *head) 812 { 813 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 814 815 return con->features & BIT(head->block); 816 } 817 818 /* 819 * if obj is not created, then create one. 820 * set feature enable flag. 821 */ 822 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 823 struct ras_common_if *head, int enable) 824 { 825 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 826 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 827 828 /* If hardware does not support ras, then do not create obj. 829 * But if hardware support ras, we can create the obj. 830 * Ras framework checks con->hw_supported to see if it need do 831 * corresponding initialization. 832 * IP checks con->support to see if it need disable ras. 833 */ 834 if (!amdgpu_ras_is_feature_allowed(adev, head)) 835 return 0; 836 837 if (enable) { 838 if (!obj) { 839 obj = amdgpu_ras_create_obj(adev, head); 840 if (!obj) 841 return -EINVAL; 842 } else { 843 /* In case we create obj somewhere else */ 844 get_obj(obj); 845 } 846 con->features |= BIT(head->block); 847 } else { 848 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 849 con->features &= ~BIT(head->block); 850 put_obj(obj); 851 } 852 } 853 854 return 0; 855 } 856 857 /* wrapper of psp_ras_enable_features */ 858 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 859 struct ras_common_if *head, bool enable) 860 { 861 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 862 union ta_ras_cmd_input *info; 863 int ret; 864 865 if (!con) 866 return -EINVAL; 867 868 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 869 /* For gfx ip, regardless of feature support status, */ 870 /* Force issue enable or disable ras feature commands */ 871 if (head->block != AMDGPU_RAS_BLOCK__GFX && 872 !amdgpu_ras_is_feature_allowed(adev, head)) 873 return 0; 874 875 /* Only enable gfx ras feature from host side */ 876 if (head->block == AMDGPU_RAS_BLOCK__GFX && 877 !amdgpu_sriov_vf(adev) && 878 !amdgpu_ras_intr_triggered()) { 879 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 880 if (!info) 881 return -ENOMEM; 882 883 if (!enable) { 884 info->disable_features = (struct ta_ras_disable_features_input) { 885 .block_id = amdgpu_ras_block_to_ta(head->block), 886 .error_type = amdgpu_ras_error_to_ta(head->type), 887 }; 888 } else { 889 info->enable_features = (struct ta_ras_enable_features_input) { 890 .block_id = amdgpu_ras_block_to_ta(head->block), 891 .error_type = amdgpu_ras_error_to_ta(head->type), 892 }; 893 } 894 895 ret = psp_ras_enable_features(&adev->psp, info, enable); 896 if (ret) { 897 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 898 enable ? "enable":"disable", 899 get_ras_block_str(head), 900 amdgpu_ras_is_poison_mode_supported(adev), ret); 901 kfree(info); 902 return ret; 903 } 904 905 kfree(info); 906 } 907 908 /* setup the obj */ 909 __amdgpu_ras_feature_enable(adev, head, enable); 910 911 return 0; 912 } 913 914 /* Only used in device probe stage and called only once. */ 915 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 916 struct ras_common_if *head, bool enable) 917 { 918 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 919 int ret; 920 921 if (!con) 922 return -EINVAL; 923 924 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 925 if (enable) { 926 /* There is no harm to issue a ras TA cmd regardless of 927 * the currecnt ras state. 928 * If current state == target state, it will do nothing 929 * But sometimes it requests driver to reset and repost 930 * with error code -EAGAIN. 931 */ 932 ret = amdgpu_ras_feature_enable(adev, head, 1); 933 /* With old ras TA, we might fail to enable ras. 934 * Log it and just setup the object. 935 * TODO need remove this WA in the future. 936 */ 937 if (ret == -EINVAL) { 938 ret = __amdgpu_ras_feature_enable(adev, head, 1); 939 if (!ret) 940 dev_info(adev->dev, 941 "RAS INFO: %s setup object\n", 942 get_ras_block_str(head)); 943 } 944 } else { 945 /* setup the object then issue a ras TA disable cmd.*/ 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (ret) 948 return ret; 949 950 /* gfx block ras disable cmd must send to ras-ta */ 951 if (head->block == AMDGPU_RAS_BLOCK__GFX) 952 con->features |= BIT(head->block); 953 954 ret = amdgpu_ras_feature_enable(adev, head, 0); 955 956 /* clean gfx block ras features flag */ 957 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 958 con->features &= ~BIT(head->block); 959 } 960 } else 961 ret = amdgpu_ras_feature_enable(adev, head, enable); 962 963 return ret; 964 } 965 966 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 967 bool bypass) 968 { 969 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 970 struct ras_manager *obj, *tmp; 971 972 list_for_each_entry_safe(obj, tmp, &con->head, node) { 973 /* bypass psp. 974 * aka just release the obj and corresponding flags 975 */ 976 if (bypass) { 977 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 978 break; 979 } else { 980 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 981 break; 982 } 983 } 984 985 return con->features; 986 } 987 988 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 989 bool bypass) 990 { 991 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 992 int i; 993 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 994 995 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 996 struct ras_common_if head = { 997 .block = i, 998 .type = default_ras_type, 999 .sub_block_index = 0, 1000 }; 1001 1002 if (i == AMDGPU_RAS_BLOCK__MCA) 1003 continue; 1004 1005 if (bypass) { 1006 /* 1007 * bypass psp. vbios enable ras for us. 1008 * so just create the obj 1009 */ 1010 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1011 break; 1012 } else { 1013 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1014 break; 1015 } 1016 } 1017 1018 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1019 struct ras_common_if head = { 1020 .block = AMDGPU_RAS_BLOCK__MCA, 1021 .type = default_ras_type, 1022 .sub_block_index = i, 1023 }; 1024 1025 if (bypass) { 1026 /* 1027 * bypass psp. vbios enable ras for us. 1028 * so just create the obj 1029 */ 1030 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1031 break; 1032 } else { 1033 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1034 break; 1035 } 1036 } 1037 1038 return con->features; 1039 } 1040 /* feature ctl end */ 1041 1042 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1043 enum amdgpu_ras_block block) 1044 { 1045 if (!block_obj) 1046 return -EINVAL; 1047 1048 if (block_obj->ras_comm.block == block) 1049 return 0; 1050 1051 return -EINVAL; 1052 } 1053 1054 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1055 enum amdgpu_ras_block block, uint32_t sub_block_index) 1056 { 1057 struct amdgpu_ras_block_list *node, *tmp; 1058 struct amdgpu_ras_block_object *obj; 1059 1060 if (block >= AMDGPU_RAS_BLOCK__LAST) 1061 return NULL; 1062 1063 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1064 if (!node->ras_obj) { 1065 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1066 continue; 1067 } 1068 1069 obj = node->ras_obj; 1070 if (obj->ras_block_match) { 1071 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1072 return obj; 1073 } else { 1074 if (amdgpu_ras_block_match_default(obj, block) == 0) 1075 return obj; 1076 } 1077 } 1078 1079 return NULL; 1080 } 1081 1082 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1083 { 1084 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1085 int ret = 0; 1086 1087 /* 1088 * choosing right query method according to 1089 * whether smu support query error information 1090 */ 1091 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1092 if (ret == -EOPNOTSUPP) { 1093 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1094 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1095 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1096 1097 /* umc query_ras_error_address is also responsible for clearing 1098 * error status 1099 */ 1100 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1101 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1103 } else if (!ret) { 1104 if (adev->umc.ras && 1105 adev->umc.ras->ecc_info_query_ras_error_count) 1106 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1107 1108 if (adev->umc.ras && 1109 adev->umc.ras->ecc_info_query_ras_error_address) 1110 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1111 } 1112 } 1113 1114 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1115 struct ras_manager *ras_mgr, 1116 struct ras_err_data *err_data, 1117 struct ras_query_context *qctx, 1118 const char *blk_name, 1119 bool is_ue, 1120 bool is_de) 1121 { 1122 struct amdgpu_smuio_mcm_config_info *mcm_info; 1123 struct ras_err_node *err_node; 1124 struct ras_err_info *err_info; 1125 u64 event_id = qctx->evid.event_id; 1126 1127 if (is_ue) { 1128 for_each_ras_error(err_node, err_data) { 1129 err_info = &err_node->err_info; 1130 mcm_info = &err_info->mcm_info; 1131 if (err_info->ue_count) { 1132 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1133 "%lld new uncorrectable hardware errors detected in %s block\n", 1134 mcm_info->socket_id, 1135 mcm_info->die_id, 1136 err_info->ue_count, 1137 blk_name); 1138 } 1139 } 1140 1141 for_each_ras_error(err_node, &ras_mgr->err_data) { 1142 err_info = &err_node->err_info; 1143 mcm_info = &err_info->mcm_info; 1144 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1145 "%lld uncorrectable hardware errors detected in total in %s block\n", 1146 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1147 } 1148 1149 } else { 1150 if (is_de) { 1151 for_each_ras_error(err_node, err_data) { 1152 err_info = &err_node->err_info; 1153 mcm_info = &err_info->mcm_info; 1154 if (err_info->de_count) { 1155 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1156 "%lld new deferred hardware errors detected in %s block\n", 1157 mcm_info->socket_id, 1158 mcm_info->die_id, 1159 err_info->de_count, 1160 blk_name); 1161 } 1162 } 1163 1164 for_each_ras_error(err_node, &ras_mgr->err_data) { 1165 err_info = &err_node->err_info; 1166 mcm_info = &err_info->mcm_info; 1167 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1168 "%lld deferred hardware errors detected in total in %s block\n", 1169 mcm_info->socket_id, mcm_info->die_id, 1170 err_info->de_count, blk_name); 1171 } 1172 } else { 1173 if (adev->debug_disable_ce_logs) 1174 return; 1175 1176 for_each_ras_error(err_node, err_data) { 1177 err_info = &err_node->err_info; 1178 mcm_info = &err_info->mcm_info; 1179 if (err_info->ce_count) { 1180 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1181 "%lld new correctable hardware errors detected in %s block\n", 1182 mcm_info->socket_id, 1183 mcm_info->die_id, 1184 err_info->ce_count, 1185 blk_name); 1186 } 1187 } 1188 1189 for_each_ras_error(err_node, &ras_mgr->err_data) { 1190 err_info = &err_node->err_info; 1191 mcm_info = &err_info->mcm_info; 1192 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1193 "%lld correctable hardware errors detected in total in %s block\n", 1194 mcm_info->socket_id, mcm_info->die_id, 1195 err_info->ce_count, blk_name); 1196 } 1197 } 1198 } 1199 } 1200 1201 static inline bool err_data_has_source_info(struct ras_err_data *data) 1202 { 1203 return !list_empty(&data->err_node_list); 1204 } 1205 1206 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1207 struct ras_query_if *query_if, 1208 struct ras_err_data *err_data, 1209 struct ras_query_context *qctx) 1210 { 1211 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1212 const char *blk_name = get_ras_block_str(&query_if->head); 1213 u64 event_id = qctx->evid.event_id; 1214 1215 if (err_data->ce_count) { 1216 if (err_data_has_source_info(err_data)) { 1217 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1218 blk_name, false, false); 1219 } else if (!adev->aid_mask && 1220 adev->smuio.funcs && 1221 adev->smuio.funcs->get_socket_id && 1222 adev->smuio.funcs->get_die_id) { 1223 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1224 "%ld correctable hardware errors " 1225 "detected in %s block\n", 1226 adev->smuio.funcs->get_socket_id(adev), 1227 adev->smuio.funcs->get_die_id(adev), 1228 ras_mgr->err_data.ce_count, 1229 blk_name); 1230 } else { 1231 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1232 "detected in %s block\n", 1233 ras_mgr->err_data.ce_count, 1234 blk_name); 1235 } 1236 } 1237 1238 if (err_data->ue_count) { 1239 if (err_data_has_source_info(err_data)) { 1240 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1241 blk_name, true, false); 1242 } else if (!adev->aid_mask && 1243 adev->smuio.funcs && 1244 adev->smuio.funcs->get_socket_id && 1245 adev->smuio.funcs->get_die_id) { 1246 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1247 "%ld uncorrectable hardware errors " 1248 "detected in %s block\n", 1249 adev->smuio.funcs->get_socket_id(adev), 1250 adev->smuio.funcs->get_die_id(adev), 1251 ras_mgr->err_data.ue_count, 1252 blk_name); 1253 } else { 1254 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1255 "detected in %s block\n", 1256 ras_mgr->err_data.ue_count, 1257 blk_name); 1258 } 1259 } 1260 1261 if (err_data->de_count) { 1262 if (err_data_has_source_info(err_data)) { 1263 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1264 blk_name, false, true); 1265 } else if (!adev->aid_mask && 1266 adev->smuio.funcs && 1267 adev->smuio.funcs->get_socket_id && 1268 adev->smuio.funcs->get_die_id) { 1269 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1270 "%ld deferred hardware errors " 1271 "detected in %s block\n", 1272 adev->smuio.funcs->get_socket_id(adev), 1273 adev->smuio.funcs->get_die_id(adev), 1274 ras_mgr->err_data.de_count, 1275 blk_name); 1276 } else { 1277 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1278 "detected in %s block\n", 1279 ras_mgr->err_data.de_count, 1280 blk_name); 1281 } 1282 } 1283 } 1284 1285 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1286 struct ras_query_if *query_if, 1287 struct ras_err_data *err_data, 1288 struct ras_query_context *qctx) 1289 { 1290 unsigned long new_ue, new_ce, new_de; 1291 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1292 const char *blk_name = get_ras_block_str(&query_if->head); 1293 u64 event_id = qctx->evid.event_id; 1294 1295 new_ce = err_data->ce_count - obj->err_data.ce_count; 1296 new_ue = err_data->ue_count - obj->err_data.ue_count; 1297 new_de = err_data->de_count - obj->err_data.de_count; 1298 1299 if (new_ce) { 1300 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1301 "detected in %s block\n", 1302 new_ce, 1303 blk_name); 1304 } 1305 1306 if (new_ue) { 1307 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1308 "detected in %s block\n", 1309 new_ue, 1310 blk_name); 1311 } 1312 1313 if (new_de) { 1314 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1315 "detected in %s block\n", 1316 new_de, 1317 blk_name); 1318 } 1319 } 1320 1321 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1322 { 1323 struct ras_err_node *err_node; 1324 struct ras_err_info *err_info; 1325 1326 if (err_data_has_source_info(err_data)) { 1327 for_each_ras_error(err_node, err_data) { 1328 err_info = &err_node->err_info; 1329 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1330 &err_info->mcm_info, err_info->de_count); 1331 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1332 &err_info->mcm_info, err_info->ce_count); 1333 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1334 &err_info->mcm_info, err_info->ue_count); 1335 } 1336 } else { 1337 /* for legacy asic path which doesn't has error source info */ 1338 obj->err_data.ue_count += err_data->ue_count; 1339 obj->err_data.ce_count += err_data->ce_count; 1340 obj->err_data.de_count += err_data->de_count; 1341 } 1342 } 1343 1344 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1345 struct ras_err_data *err_data) 1346 { 1347 /* Host reports absolute counts */ 1348 obj->err_data.ue_count = err_data->ue_count; 1349 obj->err_data.ce_count = err_data->ce_count; 1350 obj->err_data.de_count = err_data->de_count; 1351 } 1352 1353 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1354 { 1355 struct ras_common_if head; 1356 1357 memset(&head, 0, sizeof(head)); 1358 head.block = blk; 1359 1360 return amdgpu_ras_find_obj(adev, &head); 1361 } 1362 1363 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1364 const struct aca_info *aca_info, void *data) 1365 { 1366 struct ras_manager *obj; 1367 1368 /* in resume phase, no need to create aca fs node */ 1369 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1370 return 0; 1371 1372 obj = get_ras_manager(adev, blk); 1373 if (!obj) 1374 return -EINVAL; 1375 1376 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1377 } 1378 1379 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1380 { 1381 struct ras_manager *obj; 1382 1383 obj = get_ras_manager(adev, blk); 1384 if (!obj) 1385 return -EINVAL; 1386 1387 amdgpu_aca_remove_handle(&obj->aca_handle); 1388 1389 return 0; 1390 } 1391 1392 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1393 enum aca_error_type type, struct ras_err_data *err_data, 1394 struct ras_query_context *qctx) 1395 { 1396 struct ras_manager *obj; 1397 1398 obj = get_ras_manager(adev, blk); 1399 if (!obj) 1400 return -EINVAL; 1401 1402 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1403 } 1404 1405 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1406 struct aca_handle *handle, char *buf, void *data) 1407 { 1408 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1409 struct ras_query_if info = { 1410 .head = obj->head, 1411 }; 1412 1413 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1414 return sysfs_emit(buf, "Query currently inaccessible\n"); 1415 1416 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1417 return -EINVAL; 1418 1419 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1420 "ce", info.ce_count, "de", info.de_count); 1421 } 1422 1423 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1424 struct ras_query_if *info, 1425 struct ras_err_data *err_data, 1426 struct ras_query_context *qctx, 1427 unsigned int error_query_mode) 1428 { 1429 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1430 struct amdgpu_ras_block_object *block_obj = NULL; 1431 int ret; 1432 1433 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1434 return -EINVAL; 1435 1436 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1437 return -EINVAL; 1438 1439 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1440 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1441 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1442 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1443 amdgpu_ras_get_ecc_info(adev, err_data); 1444 } else { 1445 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1446 if (!block_obj || !block_obj->hw_ops) { 1447 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1448 get_ras_block_str(&info->head)); 1449 return -EINVAL; 1450 } 1451 1452 if (block_obj->hw_ops->query_ras_error_count) 1453 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1454 1455 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1456 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1457 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1458 if (block_obj->hw_ops->query_ras_error_status) 1459 block_obj->hw_ops->query_ras_error_status(adev); 1460 } 1461 } 1462 } else { 1463 if (amdgpu_aca_is_enabled(adev)) { 1464 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1465 if (ret) 1466 return ret; 1467 1468 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1469 if (ret) 1470 return ret; 1471 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 } else { 1476 /* FIXME: add code to check return value later */ 1477 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1478 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1479 } 1480 } 1481 1482 return 0; 1483 } 1484 1485 /* query/inject/cure begin */ 1486 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1487 struct ras_query_if *info, 1488 enum ras_event_type type) 1489 { 1490 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1491 struct ras_err_data err_data; 1492 struct ras_query_context qctx; 1493 unsigned int error_query_mode; 1494 int ret; 1495 1496 if (!obj) 1497 return -EINVAL; 1498 1499 ret = amdgpu_ras_error_data_init(&err_data); 1500 if (ret) 1501 return ret; 1502 1503 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1504 return -EINVAL; 1505 1506 memset(&qctx, 0, sizeof(qctx)); 1507 qctx.evid.type = type; 1508 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1509 1510 if (!down_read_trylock(&adev->reset_domain->sem)) { 1511 ret = -EIO; 1512 goto out_fini_err_data; 1513 } 1514 1515 ret = amdgpu_ras_query_error_status_helper(adev, info, 1516 &err_data, 1517 &qctx, 1518 error_query_mode); 1519 up_read(&adev->reset_domain->sem); 1520 if (ret) 1521 goto out_fini_err_data; 1522 1523 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1524 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1525 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1526 } else { 1527 /* Host provides absolute error counts. First generate the report 1528 * using the previous VF internal count against new host count. 1529 * Then Update VF internal count. 1530 */ 1531 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1532 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1533 } 1534 1535 info->ue_count = obj->err_data.ue_count; 1536 info->ce_count = obj->err_data.ce_count; 1537 info->de_count = obj->err_data.de_count; 1538 1539 out_fini_err_data: 1540 amdgpu_ras_error_data_fini(&err_data); 1541 1542 return ret; 1543 } 1544 1545 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1546 { 1547 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1548 } 1549 1550 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1551 enum amdgpu_ras_block block) 1552 { 1553 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1554 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1555 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1556 1557 if (!block_obj || !block_obj->hw_ops) { 1558 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1559 ras_block_str(block)); 1560 return -EOPNOTSUPP; 1561 } 1562 1563 if (!amdgpu_ras_is_supported(adev, block) || 1564 !amdgpu_ras_get_aca_debug_mode(adev)) 1565 return -EOPNOTSUPP; 1566 1567 if (amdgpu_sriov_vf(adev)) 1568 return -EOPNOTSUPP; 1569 1570 /* skip ras error reset in gpu reset */ 1571 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1572 ((smu_funcs && smu_funcs->set_debug_mode) || 1573 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1574 return -EOPNOTSUPP; 1575 1576 if (block_obj->hw_ops->reset_ras_error_count) 1577 block_obj->hw_ops->reset_ras_error_count(adev); 1578 1579 return 0; 1580 } 1581 1582 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1583 enum amdgpu_ras_block block) 1584 { 1585 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1586 1587 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1588 return 0; 1589 1590 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1591 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1592 if (block_obj->hw_ops->reset_ras_error_status) 1593 block_obj->hw_ops->reset_ras_error_status(adev); 1594 } 1595 1596 return 0; 1597 } 1598 1599 /* wrapper of psp_ras_trigger_error */ 1600 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1601 struct ras_inject_if *info) 1602 { 1603 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1604 struct ta_ras_trigger_error_input block_info = { 1605 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1606 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1607 .sub_block_index = info->head.sub_block_index, 1608 .address = info->address, 1609 .value = info->value, 1610 }; 1611 int ret = -EINVAL; 1612 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1613 info->head.block, 1614 info->head.sub_block_index); 1615 1616 /* inject on guest isn't allowed, return success directly */ 1617 if (amdgpu_sriov_vf(adev)) 1618 return 0; 1619 1620 if (!obj) 1621 return -EINVAL; 1622 1623 if (!block_obj || !block_obj->hw_ops) { 1624 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1625 get_ras_block_str(&info->head)); 1626 return -EINVAL; 1627 } 1628 1629 /* Calculate XGMI relative offset */ 1630 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1631 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1632 block_info.address = 1633 amdgpu_xgmi_get_relative_phy_addr(adev, 1634 block_info.address); 1635 } 1636 1637 if (block_obj->hw_ops->ras_error_inject) { 1638 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1639 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1640 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1641 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1642 info->instance_mask); 1643 } else { 1644 /* default path */ 1645 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1646 } 1647 1648 if (ret) 1649 dev_err(adev->dev, "ras inject %s failed %d\n", 1650 get_ras_block_str(&info->head), ret); 1651 1652 return ret; 1653 } 1654 1655 /** 1656 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1657 * @adev: pointer to AMD GPU device 1658 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1659 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1660 * @query_info: pointer to ras_query_if 1661 * 1662 * Return 0 for query success or do nothing, otherwise return an error 1663 * on failures 1664 */ 1665 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1666 unsigned long *ce_count, 1667 unsigned long *ue_count, 1668 struct ras_query_if *query_info) 1669 { 1670 int ret; 1671 1672 if (!query_info) 1673 /* do nothing if query_info is not specified */ 1674 return 0; 1675 1676 ret = amdgpu_ras_query_error_status(adev, query_info); 1677 if (ret) 1678 return ret; 1679 1680 *ce_count += query_info->ce_count; 1681 *ue_count += query_info->ue_count; 1682 1683 /* some hardware/IP supports read to clear 1684 * no need to explictly reset the err status after the query call */ 1685 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1686 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1687 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1688 dev_warn(adev->dev, 1689 "Failed to reset error counter and error status\n"); 1690 } 1691 1692 return 0; 1693 } 1694 1695 /** 1696 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1697 * @adev: pointer to AMD GPU device 1698 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1699 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1700 * errors. 1701 * @query_info: pointer to ras_query_if if the query request is only for 1702 * specific ip block; if info is NULL, then the qurey request is for 1703 * all the ip blocks that support query ras error counters/status 1704 * 1705 * If set, @ce_count or @ue_count, count and return the corresponding 1706 * error counts in those integer pointers. Return 0 if the device 1707 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1708 */ 1709 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1710 unsigned long *ce_count, 1711 unsigned long *ue_count, 1712 struct ras_query_if *query_info) 1713 { 1714 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1715 struct ras_manager *obj; 1716 unsigned long ce, ue; 1717 int ret; 1718 1719 if (!adev->ras_enabled || !con) 1720 return -EOPNOTSUPP; 1721 1722 /* Don't count since no reporting. 1723 */ 1724 if (!ce_count && !ue_count) 1725 return 0; 1726 1727 ce = 0; 1728 ue = 0; 1729 if (!query_info) { 1730 /* query all the ip blocks that support ras query interface */ 1731 list_for_each_entry(obj, &con->head, node) { 1732 struct ras_query_if info = { 1733 .head = obj->head, 1734 }; 1735 1736 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1737 } 1738 } else { 1739 /* query specific ip block */ 1740 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1741 } 1742 1743 if (ret) 1744 return ret; 1745 1746 if (ce_count) 1747 *ce_count = ce; 1748 1749 if (ue_count) 1750 *ue_count = ue; 1751 1752 return 0; 1753 } 1754 /* query/inject/cure end */ 1755 1756 1757 /* sysfs begin */ 1758 1759 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1760 struct ras_badpage **bps, unsigned int *count); 1761 1762 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1763 { 1764 switch (flags) { 1765 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1766 return "R"; 1767 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1768 return "P"; 1769 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1770 default: 1771 return "F"; 1772 } 1773 } 1774 1775 /** 1776 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1777 * 1778 * It allows user to read the bad pages of vram on the gpu through 1779 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1780 * 1781 * It outputs multiple lines, and each line stands for one gpu page. 1782 * 1783 * The format of one line is below, 1784 * gpu pfn : gpu page size : flags 1785 * 1786 * gpu pfn and gpu page size are printed in hex format. 1787 * flags can be one of below character, 1788 * 1789 * R: reserved, this gpu page is reserved and not able to use. 1790 * 1791 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1792 * in next window of page_reserve. 1793 * 1794 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1795 * 1796 * Examples: 1797 * 1798 * .. code-block:: bash 1799 * 1800 * 0x00000001 : 0x00001000 : R 1801 * 0x00000002 : 0x00001000 : P 1802 * 1803 */ 1804 1805 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1806 struct kobject *kobj, const struct bin_attribute *attr, 1807 char *buf, loff_t ppos, size_t count) 1808 { 1809 struct amdgpu_ras *con = 1810 container_of(attr, struct amdgpu_ras, badpages_attr); 1811 struct amdgpu_device *adev = con->adev; 1812 const unsigned int element_size = 1813 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1814 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1815 unsigned int end = div64_ul(ppos + count - 1, element_size); 1816 ssize_t s = 0; 1817 struct ras_badpage *bps = NULL; 1818 unsigned int bps_count = 0; 1819 1820 memset(buf, 0, count); 1821 1822 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1823 return 0; 1824 1825 for (; start < end && start < bps_count; start++) 1826 s += scnprintf(&buf[s], element_size + 1, 1827 "0x%08x : 0x%08x : %1s\n", 1828 bps[start].bp, 1829 bps[start].size, 1830 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1831 1832 kfree(bps); 1833 1834 return s; 1835 } 1836 1837 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1838 struct device_attribute *attr, char *buf) 1839 { 1840 struct amdgpu_ras *con = 1841 container_of(attr, struct amdgpu_ras, features_attr); 1842 1843 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1844 } 1845 1846 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1847 struct device_attribute *attr, char *buf) 1848 { 1849 struct amdgpu_ras *con = 1850 container_of(attr, struct amdgpu_ras, version_attr); 1851 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version); 1852 } 1853 1854 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1855 struct device_attribute *attr, char *buf) 1856 { 1857 struct amdgpu_ras *con = 1858 container_of(attr, struct amdgpu_ras, schema_attr); 1859 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1860 } 1861 1862 static struct { 1863 enum ras_event_type type; 1864 const char *name; 1865 } dump_event[] = { 1866 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 1867 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 1868 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 1869 }; 1870 1871 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 1872 struct device_attribute *attr, char *buf) 1873 { 1874 struct amdgpu_ras *con = 1875 container_of(attr, struct amdgpu_ras, event_state_attr); 1876 struct ras_event_manager *event_mgr = con->event_mgr; 1877 struct ras_event_state *event_state; 1878 int i, size = 0; 1879 1880 if (!event_mgr) 1881 return -EINVAL; 1882 1883 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 1884 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 1885 event_state = &event_mgr->event_state[dump_event[i].type]; 1886 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 1887 dump_event[i].name, 1888 atomic64_read(&event_state->count), 1889 event_state->last_seqno); 1890 } 1891 1892 return (ssize_t)size; 1893 } 1894 1895 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1896 { 1897 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1898 1899 if (adev->dev->kobj.sd) 1900 sysfs_remove_file_from_group(&adev->dev->kobj, 1901 &con->badpages_attr.attr, 1902 RAS_FS_NAME); 1903 } 1904 1905 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 1906 { 1907 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1908 struct attribute *attrs[] = { 1909 &con->features_attr.attr, 1910 &con->version_attr.attr, 1911 &con->schema_attr.attr, 1912 &con->event_state_attr.attr, 1913 NULL 1914 }; 1915 struct attribute_group group = { 1916 .name = RAS_FS_NAME, 1917 .attrs = attrs, 1918 }; 1919 1920 if (adev->dev->kobj.sd) 1921 sysfs_remove_group(&adev->dev->kobj, &group); 1922 1923 return 0; 1924 } 1925 1926 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1927 struct ras_common_if *head) 1928 { 1929 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1930 1931 if (amdgpu_aca_is_enabled(adev)) 1932 return 0; 1933 1934 if (!obj || obj->attr_inuse) 1935 return -EINVAL; 1936 1937 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 1938 return 0; 1939 1940 get_obj(obj); 1941 1942 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1943 "%s_err_count", head->name); 1944 1945 obj->sysfs_attr = (struct device_attribute){ 1946 .attr = { 1947 .name = obj->fs_data.sysfs_name, 1948 .mode = S_IRUGO, 1949 }, 1950 .show = amdgpu_ras_sysfs_read, 1951 }; 1952 sysfs_attr_init(&obj->sysfs_attr.attr); 1953 1954 if (sysfs_add_file_to_group(&adev->dev->kobj, 1955 &obj->sysfs_attr.attr, 1956 RAS_FS_NAME)) { 1957 put_obj(obj); 1958 return -EINVAL; 1959 } 1960 1961 obj->attr_inuse = 1; 1962 1963 return 0; 1964 } 1965 1966 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1967 struct ras_common_if *head) 1968 { 1969 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1970 1971 if (amdgpu_aca_is_enabled(adev)) 1972 return 0; 1973 1974 if (!obj || !obj->attr_inuse) 1975 return -EINVAL; 1976 1977 if (adev->dev->kobj.sd) 1978 sysfs_remove_file_from_group(&adev->dev->kobj, 1979 &obj->sysfs_attr.attr, 1980 RAS_FS_NAME); 1981 obj->attr_inuse = 0; 1982 put_obj(obj); 1983 1984 return 0; 1985 } 1986 1987 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1988 { 1989 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1990 struct ras_manager *obj, *tmp; 1991 1992 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1993 amdgpu_ras_sysfs_remove(adev, &obj->head); 1994 } 1995 1996 if (amdgpu_bad_page_threshold != 0) 1997 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1998 1999 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2000 2001 return 0; 2002 } 2003 /* sysfs end */ 2004 2005 /** 2006 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2007 * 2008 * Normally when there is an uncorrectable error, the driver will reset 2009 * the GPU to recover. However, in the event of an unrecoverable error, 2010 * the driver provides an interface to reboot the system automatically 2011 * in that event. 2012 * 2013 * The following file in debugfs provides that interface: 2014 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2015 * 2016 * Usage: 2017 * 2018 * .. code-block:: bash 2019 * 2020 * echo true > .../ras/auto_reboot 2021 * 2022 */ 2023 /* debugfs begin */ 2024 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2025 { 2026 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2027 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2028 struct drm_minor *minor = adev_to_drm(adev)->primary; 2029 struct dentry *dir; 2030 2031 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2032 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2033 &amdgpu_ras_debugfs_ctrl_ops); 2034 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2035 &amdgpu_ras_debugfs_eeprom_ops); 2036 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2037 &con->bad_page_cnt_threshold); 2038 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2039 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2040 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2041 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2042 &amdgpu_ras_debugfs_eeprom_size_ops); 2043 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2044 S_IRUGO, dir, adev, 2045 &amdgpu_ras_debugfs_eeprom_table_ops); 2046 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2047 2048 /* 2049 * After one uncorrectable error happens, usually GPU recovery will 2050 * be scheduled. But due to the known problem in GPU recovery failing 2051 * to bring GPU back, below interface provides one direct way to 2052 * user to reboot system automatically in such case within 2053 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2054 * will never be called. 2055 */ 2056 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2057 2058 /* 2059 * User could set this not to clean up hardware's error count register 2060 * of RAS IPs during ras recovery. 2061 */ 2062 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2063 &con->disable_ras_err_cnt_harvest); 2064 return dir; 2065 } 2066 2067 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2068 struct ras_fs_if *head, 2069 struct dentry *dir) 2070 { 2071 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2072 2073 if (!obj || !dir) 2074 return; 2075 2076 get_obj(obj); 2077 2078 memcpy(obj->fs_data.debugfs_name, 2079 head->debugfs_name, 2080 sizeof(obj->fs_data.debugfs_name)); 2081 2082 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2083 obj, &amdgpu_ras_debugfs_ops); 2084 } 2085 2086 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2087 { 2088 bool ret; 2089 2090 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2091 case IP_VERSION(13, 0, 6): 2092 case IP_VERSION(13, 0, 12): 2093 case IP_VERSION(13, 0, 14): 2094 ret = true; 2095 break; 2096 default: 2097 ret = false; 2098 break; 2099 } 2100 2101 return ret; 2102 } 2103 2104 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2105 { 2106 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2107 struct dentry *dir; 2108 struct ras_manager *obj; 2109 struct ras_fs_if fs_info; 2110 2111 /* 2112 * it won't be called in resume path, no need to check 2113 * suspend and gpu reset status 2114 */ 2115 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2116 return; 2117 2118 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2119 2120 list_for_each_entry(obj, &con->head, node) { 2121 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2122 (obj->attr_inuse == 1)) { 2123 sprintf(fs_info.debugfs_name, "%s_err_inject", 2124 get_ras_block_str(&obj->head)); 2125 fs_info.head = obj->head; 2126 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2127 } 2128 } 2129 2130 if (amdgpu_ras_aca_is_supported(adev)) { 2131 if (amdgpu_aca_is_enabled(adev)) 2132 amdgpu_aca_smu_debugfs_init(adev, dir); 2133 else 2134 amdgpu_mca_smu_debugfs_init(adev, dir); 2135 } 2136 } 2137 2138 /* debugfs end */ 2139 2140 /* ras fs */ 2141 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2142 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2143 static DEVICE_ATTR(features, S_IRUGO, 2144 amdgpu_ras_sysfs_features_read, NULL); 2145 static DEVICE_ATTR(version, 0444, 2146 amdgpu_ras_sysfs_version_show, NULL); 2147 static DEVICE_ATTR(schema, 0444, 2148 amdgpu_ras_sysfs_schema_show, NULL); 2149 static DEVICE_ATTR(event_state, 0444, 2150 amdgpu_ras_sysfs_event_state_show, NULL); 2151 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2152 { 2153 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2154 struct attribute_group group = { 2155 .name = RAS_FS_NAME, 2156 }; 2157 struct attribute *attrs[] = { 2158 &con->features_attr.attr, 2159 &con->version_attr.attr, 2160 &con->schema_attr.attr, 2161 &con->event_state_attr.attr, 2162 NULL 2163 }; 2164 const struct bin_attribute *bin_attrs[] = { 2165 NULL, 2166 NULL, 2167 }; 2168 int r; 2169 2170 group.attrs = attrs; 2171 2172 /* add features entry */ 2173 con->features_attr = dev_attr_features; 2174 sysfs_attr_init(attrs[0]); 2175 2176 /* add version entry */ 2177 con->version_attr = dev_attr_version; 2178 sysfs_attr_init(attrs[1]); 2179 2180 /* add schema entry */ 2181 con->schema_attr = dev_attr_schema; 2182 sysfs_attr_init(attrs[2]); 2183 2184 /* add event_state entry */ 2185 con->event_state_attr = dev_attr_event_state; 2186 sysfs_attr_init(attrs[3]); 2187 2188 if (amdgpu_bad_page_threshold != 0) { 2189 /* add bad_page_features entry */ 2190 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2191 sysfs_bin_attr_init(&con->badpages_attr); 2192 bin_attrs[0] = &con->badpages_attr; 2193 group.bin_attrs = bin_attrs; 2194 } 2195 2196 r = sysfs_create_group(&adev->dev->kobj, &group); 2197 if (r) 2198 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2199 2200 return 0; 2201 } 2202 2203 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2204 { 2205 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2206 struct ras_manager *con_obj, *ip_obj, *tmp; 2207 2208 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2209 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2210 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2211 if (ip_obj) 2212 put_obj(ip_obj); 2213 } 2214 } 2215 2216 amdgpu_ras_sysfs_remove_all(adev); 2217 return 0; 2218 } 2219 /* ras fs end */ 2220 2221 /* ih begin */ 2222 2223 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2224 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2225 * register to check whether the interrupt is triggered or not, and properly 2226 * ack the interrupt if it is there 2227 */ 2228 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2229 { 2230 /* Fatal error events are handled on host side */ 2231 if (amdgpu_sriov_vf(adev)) 2232 return; 2233 /* 2234 * If the current interrupt is caused by a non-fatal RAS error, skip 2235 * check for fatal error. For fatal errors, FED status of all devices 2236 * in XGMI hive gets set when the first device gets fatal error 2237 * interrupt. The error gets propagated to other devices as well, so 2238 * make sure to ack the interrupt regardless of FED status. 2239 */ 2240 if (!amdgpu_ras_get_fed_status(adev) && 2241 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2242 return; 2243 2244 if (adev->nbio.ras && 2245 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2246 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2247 2248 if (adev->nbio.ras && 2249 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2250 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2251 } 2252 2253 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2254 struct amdgpu_iv_entry *entry) 2255 { 2256 bool poison_stat = false; 2257 struct amdgpu_device *adev = obj->adev; 2258 struct amdgpu_ras_block_object *block_obj = 2259 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2260 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2261 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2262 u64 event_id; 2263 int ret; 2264 2265 if (!block_obj || !con) 2266 return; 2267 2268 ret = amdgpu_ras_mark_ras_event(adev, type); 2269 if (ret) 2270 return; 2271 2272 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2273 /* both query_poison_status and handle_poison_consumption are optional, 2274 * but at least one of them should be implemented if we need poison 2275 * consumption handler 2276 */ 2277 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2278 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2279 if (!poison_stat) { 2280 /* Not poison consumption interrupt, no need to handle it */ 2281 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2282 block_obj->ras_comm.name); 2283 2284 return; 2285 } 2286 } 2287 2288 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2289 2290 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2291 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2292 2293 /* gpu reset is fallback for failed and default cases. 2294 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2295 */ 2296 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2297 event_id = amdgpu_ras_acquire_event_id(adev, type); 2298 RAS_EVENT_LOG(adev, event_id, 2299 "GPU reset for %s RAS poison consumption is issued!\n", 2300 block_obj->ras_comm.name); 2301 amdgpu_ras_reset_gpu(adev); 2302 } 2303 2304 if (!poison_stat) 2305 amdgpu_gfx_poison_consumption_handler(adev, entry); 2306 } 2307 2308 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2309 struct amdgpu_iv_entry *entry) 2310 { 2311 struct amdgpu_device *adev = obj->adev; 2312 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2313 u64 event_id; 2314 int ret; 2315 2316 ret = amdgpu_ras_mark_ras_event(adev, type); 2317 if (ret) 2318 return; 2319 2320 event_id = amdgpu_ras_acquire_event_id(adev, type); 2321 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2322 2323 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2324 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2325 2326 atomic_inc(&con->page_retirement_req_cnt); 2327 atomic_inc(&con->poison_creation_count); 2328 2329 wake_up(&con->page_retirement_wq); 2330 } 2331 } 2332 2333 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2334 struct amdgpu_iv_entry *entry) 2335 { 2336 struct ras_ih_data *data = &obj->ih_data; 2337 struct ras_err_data err_data; 2338 int ret; 2339 2340 if (!data->cb) 2341 return; 2342 2343 ret = amdgpu_ras_error_data_init(&err_data); 2344 if (ret) 2345 return; 2346 2347 /* Let IP handle its data, maybe we need get the output 2348 * from the callback to update the error type/count, etc 2349 */ 2350 amdgpu_ras_set_fed(obj->adev, true); 2351 ret = data->cb(obj->adev, &err_data, entry); 2352 /* ue will trigger an interrupt, and in that case 2353 * we need do a reset to recovery the whole system. 2354 * But leave IP do that recovery, here we just dispatch 2355 * the error. 2356 */ 2357 if (ret == AMDGPU_RAS_SUCCESS) { 2358 /* these counts could be left as 0 if 2359 * some blocks do not count error number 2360 */ 2361 obj->err_data.ue_count += err_data.ue_count; 2362 obj->err_data.ce_count += err_data.ce_count; 2363 obj->err_data.de_count += err_data.de_count; 2364 } 2365 2366 amdgpu_ras_error_data_fini(&err_data); 2367 } 2368 2369 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2370 { 2371 struct ras_ih_data *data = &obj->ih_data; 2372 struct amdgpu_iv_entry entry; 2373 2374 while (data->rptr != data->wptr) { 2375 rmb(); 2376 memcpy(&entry, &data->ring[data->rptr], 2377 data->element_size); 2378 2379 wmb(); 2380 data->rptr = (data->aligned_element_size + 2381 data->rptr) % data->ring_size; 2382 2383 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2384 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2385 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2386 else 2387 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2388 } else { 2389 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2390 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2391 else 2392 dev_warn(obj->adev->dev, 2393 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2394 } 2395 } 2396 } 2397 2398 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2399 { 2400 struct ras_ih_data *data = 2401 container_of(work, struct ras_ih_data, ih_work); 2402 struct ras_manager *obj = 2403 container_of(data, struct ras_manager, ih_data); 2404 2405 amdgpu_ras_interrupt_handler(obj); 2406 } 2407 2408 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2409 struct ras_dispatch_if *info) 2410 { 2411 struct ras_manager *obj; 2412 struct ras_ih_data *data; 2413 2414 obj = amdgpu_ras_find_obj(adev, &info->head); 2415 if (!obj) 2416 return -EINVAL; 2417 2418 data = &obj->ih_data; 2419 2420 if (data->inuse == 0) 2421 return 0; 2422 2423 /* Might be overflow... */ 2424 memcpy(&data->ring[data->wptr], info->entry, 2425 data->element_size); 2426 2427 wmb(); 2428 data->wptr = (data->aligned_element_size + 2429 data->wptr) % data->ring_size; 2430 2431 schedule_work(&data->ih_work); 2432 2433 return 0; 2434 } 2435 2436 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2437 struct ras_common_if *head) 2438 { 2439 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2440 struct ras_ih_data *data; 2441 2442 if (!obj) 2443 return -EINVAL; 2444 2445 data = &obj->ih_data; 2446 if (data->inuse == 0) 2447 return 0; 2448 2449 cancel_work_sync(&data->ih_work); 2450 2451 kfree(data->ring); 2452 memset(data, 0, sizeof(*data)); 2453 put_obj(obj); 2454 2455 return 0; 2456 } 2457 2458 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2459 struct ras_common_if *head) 2460 { 2461 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2462 struct ras_ih_data *data; 2463 struct amdgpu_ras_block_object *ras_obj; 2464 2465 if (!obj) { 2466 /* in case we registe the IH before enable ras feature */ 2467 obj = amdgpu_ras_create_obj(adev, head); 2468 if (!obj) 2469 return -EINVAL; 2470 } else 2471 get_obj(obj); 2472 2473 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2474 2475 data = &obj->ih_data; 2476 /* add the callback.etc */ 2477 *data = (struct ras_ih_data) { 2478 .inuse = 0, 2479 .cb = ras_obj->ras_cb, 2480 .element_size = sizeof(struct amdgpu_iv_entry), 2481 .rptr = 0, 2482 .wptr = 0, 2483 }; 2484 2485 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2486 2487 data->aligned_element_size = ALIGN(data->element_size, 8); 2488 /* the ring can store 64 iv entries. */ 2489 data->ring_size = 64 * data->aligned_element_size; 2490 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2491 if (!data->ring) { 2492 put_obj(obj); 2493 return -ENOMEM; 2494 } 2495 2496 /* IH is ready */ 2497 data->inuse = 1; 2498 2499 return 0; 2500 } 2501 2502 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2503 { 2504 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2505 struct ras_manager *obj, *tmp; 2506 2507 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2508 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2509 } 2510 2511 return 0; 2512 } 2513 /* ih end */ 2514 2515 /* traversal all IPs except NBIO to query error counter */ 2516 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2517 { 2518 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2519 struct ras_manager *obj; 2520 2521 if (!adev->ras_enabled || !con) 2522 return; 2523 2524 list_for_each_entry(obj, &con->head, node) { 2525 struct ras_query_if info = { 2526 .head = obj->head, 2527 }; 2528 2529 /* 2530 * PCIE_BIF IP has one different isr by ras controller 2531 * interrupt, the specific ras counter query will be 2532 * done in that isr. So skip such block from common 2533 * sync flood interrupt isr calling. 2534 */ 2535 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2536 continue; 2537 2538 /* 2539 * this is a workaround for aldebaran, skip send msg to 2540 * smu to get ecc_info table due to smu handle get ecc 2541 * info table failed temporarily. 2542 * should be removed until smu fix handle ecc_info table. 2543 */ 2544 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2545 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2546 IP_VERSION(13, 0, 2))) 2547 continue; 2548 2549 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2550 2551 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2552 IP_VERSION(11, 0, 2) && 2553 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2554 IP_VERSION(11, 0, 4) && 2555 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2556 IP_VERSION(13, 0, 0)) { 2557 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2558 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2559 } 2560 } 2561 } 2562 2563 /* Parse RdRspStatus and WrRspStatus */ 2564 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2565 struct ras_query_if *info) 2566 { 2567 struct amdgpu_ras_block_object *block_obj; 2568 /* 2569 * Only two block need to query read/write 2570 * RspStatus at current state 2571 */ 2572 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2573 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2574 return; 2575 2576 block_obj = amdgpu_ras_get_ras_block(adev, 2577 info->head.block, 2578 info->head.sub_block_index); 2579 2580 if (!block_obj || !block_obj->hw_ops) { 2581 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2582 get_ras_block_str(&info->head)); 2583 return; 2584 } 2585 2586 if (block_obj->hw_ops->query_ras_error_status) 2587 block_obj->hw_ops->query_ras_error_status(adev); 2588 2589 } 2590 2591 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2592 { 2593 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2594 struct ras_manager *obj; 2595 2596 if (!adev->ras_enabled || !con) 2597 return; 2598 2599 list_for_each_entry(obj, &con->head, node) { 2600 struct ras_query_if info = { 2601 .head = obj->head, 2602 }; 2603 2604 amdgpu_ras_error_status_query(adev, &info); 2605 } 2606 } 2607 2608 /* recovery begin */ 2609 2610 /* return 0 on success. 2611 * caller need free bps. 2612 */ 2613 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2614 struct ras_badpage **bps, unsigned int *count) 2615 { 2616 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2617 struct ras_err_handler_data *data; 2618 int i = 0; 2619 int ret = 0, status; 2620 2621 if (!con || !con->eh_data || !bps || !count) 2622 return -EINVAL; 2623 2624 mutex_lock(&con->recovery_lock); 2625 data = con->eh_data; 2626 if (!data || data->count == 0) { 2627 *bps = NULL; 2628 ret = -EINVAL; 2629 goto out; 2630 } 2631 2632 *bps = kmalloc_array(data->count, sizeof(struct ras_badpage), GFP_KERNEL); 2633 if (!*bps) { 2634 ret = -ENOMEM; 2635 goto out; 2636 } 2637 2638 for (; i < data->count; i++) { 2639 if (!data->bps[i].ts) 2640 continue; 2641 2642 (*bps)[i] = (struct ras_badpage){ 2643 .bp = data->bps[i].retired_page, 2644 .size = AMDGPU_GPU_PAGE_SIZE, 2645 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 2646 }; 2647 2648 if (amdgpu_ras_check_critical_address(adev, 2649 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2650 continue; 2651 2652 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 2653 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT); 2654 if (status == -EBUSY) 2655 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 2656 else if (status == -ENOENT) 2657 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 2658 } 2659 2660 *count = con->bad_page_num; 2661 out: 2662 mutex_unlock(&con->recovery_lock); 2663 return ret; 2664 } 2665 2666 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2667 struct amdgpu_hive_info *hive, bool status) 2668 { 2669 struct amdgpu_device *tmp_adev; 2670 2671 if (hive) { 2672 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2673 amdgpu_ras_set_fed(tmp_adev, status); 2674 } else { 2675 amdgpu_ras_set_fed(adev, status); 2676 } 2677 } 2678 2679 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2680 { 2681 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2682 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2683 int hive_ras_recovery = 0; 2684 2685 if (hive) { 2686 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2687 amdgpu_put_xgmi_hive(hive); 2688 } 2689 2690 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2691 return true; 2692 2693 return false; 2694 } 2695 2696 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2697 { 2698 if (amdgpu_ras_intr_triggered()) 2699 return RAS_EVENT_TYPE_FATAL; 2700 else 2701 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2702 } 2703 2704 static void amdgpu_ras_do_recovery(struct work_struct *work) 2705 { 2706 struct amdgpu_ras *ras = 2707 container_of(work, struct amdgpu_ras, recovery_work); 2708 struct amdgpu_device *remote_adev = NULL; 2709 struct amdgpu_device *adev = ras->adev; 2710 struct list_head device_list, *device_list_handle = NULL; 2711 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2712 unsigned int error_query_mode; 2713 enum ras_event_type type; 2714 2715 if (hive) { 2716 atomic_set(&hive->ras_recovery, 1); 2717 2718 /* If any device which is part of the hive received RAS fatal 2719 * error interrupt, set fatal error status on all. This 2720 * condition will need a recovery, and flag will be cleared 2721 * as part of recovery. 2722 */ 2723 list_for_each_entry(remote_adev, &hive->device_list, 2724 gmc.xgmi.head) 2725 if (amdgpu_ras_get_fed_status(remote_adev)) { 2726 amdgpu_ras_set_fed_all(adev, hive, true); 2727 break; 2728 } 2729 } 2730 if (!ras->disable_ras_err_cnt_harvest) { 2731 2732 /* Build list of devices to query RAS related errors */ 2733 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2734 device_list_handle = &hive->device_list; 2735 } else { 2736 INIT_LIST_HEAD(&device_list); 2737 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2738 device_list_handle = &device_list; 2739 } 2740 2741 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2742 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2743 /* wait 500ms to ensure pmfw polling mca bank info done */ 2744 msleep(500); 2745 } 2746 } 2747 2748 type = amdgpu_ras_get_fatal_error_event(adev); 2749 list_for_each_entry(remote_adev, 2750 device_list_handle, gmc.xgmi.head) { 2751 amdgpu_ras_query_err_status(remote_adev); 2752 amdgpu_ras_log_on_err_counter(remote_adev, type); 2753 } 2754 2755 } 2756 2757 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2758 struct amdgpu_reset_context reset_context; 2759 memset(&reset_context, 0, sizeof(reset_context)); 2760 2761 reset_context.method = AMD_RESET_METHOD_NONE; 2762 reset_context.reset_req_dev = adev; 2763 reset_context.src = AMDGPU_RESET_SRC_RAS; 2764 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2765 2766 /* Perform full reset in fatal error mode */ 2767 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2768 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2769 else { 2770 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2771 2772 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2773 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2774 reset_context.method = AMD_RESET_METHOD_MODE2; 2775 } 2776 2777 /* Fatal error occurs in poison mode, mode1 reset is used to 2778 * recover gpu. 2779 */ 2780 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2781 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2782 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2783 2784 psp_fatal_error_recovery_quirk(&adev->psp); 2785 } 2786 } 2787 2788 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2789 } 2790 atomic_set(&ras->in_recovery, 0); 2791 if (hive) { 2792 atomic_set(&hive->ras_recovery, 0); 2793 amdgpu_put_xgmi_hive(hive); 2794 } 2795 } 2796 2797 /* alloc/realloc bps array */ 2798 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2799 struct ras_err_handler_data *data, int pages) 2800 { 2801 unsigned int old_space = data->count + data->space_left; 2802 unsigned int new_space = old_space + pages; 2803 unsigned int align_space = ALIGN(new_space, 512); 2804 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2805 2806 if (!bps) { 2807 return -ENOMEM; 2808 } 2809 2810 if (data->bps) { 2811 memcpy(bps, data->bps, 2812 data->count * sizeof(*data->bps)); 2813 kfree(data->bps); 2814 } 2815 2816 data->bps = bps; 2817 data->space_left += align_space - old_space; 2818 return 0; 2819 } 2820 2821 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2822 struct eeprom_table_record *bps, 2823 struct ras_err_data *err_data) 2824 { 2825 struct ta_ras_query_address_input addr_in; 2826 uint32_t socket = 0; 2827 int ret = 0; 2828 2829 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2830 socket = adev->smuio.funcs->get_socket_id(adev); 2831 2832 /* reinit err_data */ 2833 err_data->err_addr_cnt = 0; 2834 err_data->err_addr_len = adev->umc.retire_unit; 2835 2836 memset(&addr_in, 0, sizeof(addr_in)); 2837 addr_in.ma.err_addr = bps->address; 2838 addr_in.ma.socket_id = socket; 2839 addr_in.ma.ch_inst = bps->mem_channel; 2840 /* tell RAS TA the node instance is not used */ 2841 addr_in.ma.node_inst = TA_RAS_INV_NODE; 2842 2843 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2844 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 2845 &addr_in, NULL, false); 2846 2847 return ret; 2848 } 2849 2850 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 2851 struct eeprom_table_record *bps, 2852 struct ras_err_data *err_data) 2853 { 2854 struct ta_ras_query_address_input addr_in; 2855 uint32_t die_id, socket = 0; 2856 2857 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2858 socket = adev->smuio.funcs->get_socket_id(adev); 2859 2860 /* although die id is gotten from PA in nps1 mode, the id is 2861 * fitable for any nps mode 2862 */ 2863 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 2864 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 2865 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 2866 else 2867 return -EINVAL; 2868 2869 /* reinit err_data */ 2870 err_data->err_addr_cnt = 0; 2871 err_data->err_addr_len = adev->umc.retire_unit; 2872 2873 memset(&addr_in, 0, sizeof(addr_in)); 2874 addr_in.ma.err_addr = bps->address; 2875 addr_in.ma.ch_inst = bps->mem_channel; 2876 addr_in.ma.umc_inst = bps->mcumc_id; 2877 addr_in.ma.node_inst = die_id; 2878 addr_in.ma.socket_id = socket; 2879 2880 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2881 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 2882 &addr_in, NULL, false); 2883 else 2884 return -EINVAL; 2885 } 2886 2887 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 2888 struct eeprom_table_record *bps, int count) 2889 { 2890 int j; 2891 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2892 struct ras_err_handler_data *data = con->eh_data; 2893 2894 for (j = 0; j < count; j++) { 2895 if (amdgpu_ras_check_bad_page_unlock(con, 2896 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 2897 data->count++; 2898 data->space_left--; 2899 continue; 2900 } 2901 2902 if (!data->space_left && 2903 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2904 return -ENOMEM; 2905 } 2906 2907 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 2908 2909 memcpy(&data->bps[data->count], &(bps[j]), 2910 sizeof(struct eeprom_table_record)); 2911 data->count++; 2912 data->space_left--; 2913 con->bad_page_num++; 2914 } 2915 2916 return 0; 2917 } 2918 2919 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 2920 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2921 enum amdgpu_memory_partition nps) 2922 { 2923 int i = 0; 2924 enum amdgpu_memory_partition save_nps; 2925 2926 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2927 2928 /*old asics just have pa in eeprom*/ 2929 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2930 memcpy(err_data->err_addr, bps, 2931 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2932 goto out; 2933 } 2934 2935 for (i = 0; i < adev->umc.retire_unit; i++) 2936 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2937 2938 if (save_nps) { 2939 if (save_nps == nps) { 2940 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2941 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2942 return -EINVAL; 2943 for (i = 0; i < adev->umc.retire_unit; i++) { 2944 err_data->err_addr[i].address = bps[0].address; 2945 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 2946 err_data->err_addr[i].bank = bps[0].bank; 2947 err_data->err_addr[i].err_type = bps[0].err_type; 2948 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 2949 } 2950 } else { 2951 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 2952 return -EINVAL; 2953 } 2954 } else { 2955 if (bps[0].address == 0) { 2956 /* for specific old eeprom data, mca address is not stored, 2957 * calc it from pa 2958 */ 2959 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 2960 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 2961 return -EINVAL; 2962 } 2963 2964 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 2965 if (nps == AMDGPU_NPS1_PARTITION_MODE) 2966 memcpy(err_data->err_addr, bps, 2967 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2968 else 2969 return -EOPNOTSUPP; 2970 } 2971 } 2972 2973 out: 2974 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 2975 } 2976 2977 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 2978 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2979 enum amdgpu_memory_partition nps) 2980 { 2981 int i = 0; 2982 enum amdgpu_memory_partition save_nps; 2983 2984 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2985 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2986 2987 if (save_nps == nps) { 2988 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2989 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2990 return -EINVAL; 2991 for (i = 0; i < adev->umc.retire_unit; i++) { 2992 err_data->err_addr[i].address = bps->address; 2993 err_data->err_addr[i].mem_channel = bps->mem_channel; 2994 err_data->err_addr[i].bank = bps->bank; 2995 err_data->err_addr[i].err_type = bps->err_type; 2996 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 2997 } 2998 } else { 2999 if (bps->address) { 3000 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3001 return -EINVAL; 3002 } else { 3003 /* for specific old eeprom data, mca address is not stored, 3004 * calc it from pa 3005 */ 3006 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3007 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3008 return -EINVAL; 3009 3010 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3011 return -EOPNOTSUPP; 3012 } 3013 } 3014 3015 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3016 adev->umc.retire_unit); 3017 } 3018 3019 /* it deal with vram only. */ 3020 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3021 struct eeprom_table_record *bps, int pages, bool from_rom) 3022 { 3023 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3024 struct ras_err_data err_data; 3025 struct amdgpu_ras_eeprom_control *control = 3026 &adev->psp.ras_context.ras->eeprom_control; 3027 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3028 int ret = 0; 3029 uint32_t i = 0; 3030 3031 if (!con || !con->eh_data || !bps || pages <= 0) 3032 return 0; 3033 3034 if (from_rom) { 3035 err_data.err_addr = 3036 kcalloc(adev->umc.retire_unit, 3037 sizeof(struct eeprom_table_record), GFP_KERNEL); 3038 if (!err_data.err_addr) { 3039 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3040 return -ENOMEM; 3041 } 3042 3043 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3044 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3045 } 3046 3047 mutex_lock(&con->recovery_lock); 3048 3049 if (from_rom) { 3050 /* there is no pa recs in V3, so skip pa recs processing */ 3051 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3052 for (i = 0; i < pages; i++) { 3053 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3054 if ((bps[i].address == bps[i + 1].address) && 3055 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3056 /* deal with retire_unit records a time */ 3057 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3058 &bps[i], &err_data, nps); 3059 if (ret) 3060 con->bad_page_num -= adev->umc.retire_unit; 3061 i += (adev->umc.retire_unit - 1); 3062 } else { 3063 break; 3064 } 3065 } else { 3066 break; 3067 } 3068 } 3069 } 3070 for (; i < pages; i++) { 3071 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3072 &bps[i], &err_data, nps); 3073 if (ret) 3074 con->bad_page_num -= adev->umc.retire_unit; 3075 } 3076 3077 con->eh_data->count_saved = con->eh_data->count; 3078 } else { 3079 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3080 } 3081 3082 if (from_rom) 3083 kfree(err_data.err_addr); 3084 mutex_unlock(&con->recovery_lock); 3085 3086 return ret; 3087 } 3088 3089 /* 3090 * write error record array to eeprom, the function should be 3091 * protected by recovery_lock 3092 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3093 */ 3094 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3095 unsigned long *new_cnt) 3096 { 3097 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3098 struct ras_err_handler_data *data; 3099 struct amdgpu_ras_eeprom_control *control; 3100 int save_count, unit_num, i; 3101 3102 if (!con || !con->eh_data) { 3103 if (new_cnt) 3104 *new_cnt = 0; 3105 3106 return 0; 3107 } 3108 3109 if (!con->eeprom_control.is_eeprom_valid) { 3110 dev_warn(adev->dev, 3111 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3112 if (new_cnt) 3113 *new_cnt = 0; 3114 3115 return 0; 3116 } 3117 3118 mutex_lock(&con->recovery_lock); 3119 control = &con->eeprom_control; 3120 data = con->eh_data; 3121 unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; 3122 save_count = con->bad_page_num - control->ras_num_bad_pages; 3123 mutex_unlock(&con->recovery_lock); 3124 3125 if (new_cnt) 3126 *new_cnt = unit_num; 3127 3128 /* only new entries are saved */ 3129 if (unit_num > 0) { 3130 /*old asics only save pa to eeprom like before*/ 3131 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3132 if (amdgpu_ras_eeprom_append(control, 3133 &data->bps[data->count_saved], unit_num)) { 3134 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3135 return -EIO; 3136 } 3137 } else { 3138 for (i = 0; i < unit_num; i++) { 3139 if (amdgpu_ras_eeprom_append(control, 3140 &data->bps[data->count_saved + 3141 i * adev->umc.retire_unit], 1)) { 3142 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3143 return -EIO; 3144 } 3145 } 3146 } 3147 3148 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3149 data->count_saved = data->count; 3150 } 3151 3152 return 0; 3153 } 3154 3155 /* 3156 * read error record array in eeprom and reserve enough space for 3157 * storing new bad pages 3158 */ 3159 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3160 { 3161 struct amdgpu_ras_eeprom_control *control = 3162 &adev->psp.ras_context.ras->eeprom_control; 3163 struct eeprom_table_record *bps; 3164 int ret, i = 0; 3165 3166 /* no bad page record, skip eeprom access */ 3167 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3168 return 0; 3169 3170 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3171 if (!bps) 3172 return -ENOMEM; 3173 3174 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3175 if (ret) { 3176 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3177 } else { 3178 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3179 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3180 as pa recs, so add verion check to avoid it. 3181 */ 3182 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3183 for (i = 0; i < control->ras_num_recs; i++) { 3184 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3185 if ((bps[i].address == bps[i + 1].address) && 3186 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3187 control->ras_num_pa_recs += adev->umc.retire_unit; 3188 i += (adev->umc.retire_unit - 1); 3189 } else { 3190 control->ras_num_mca_recs += 3191 (control->ras_num_recs - i); 3192 break; 3193 } 3194 } else { 3195 control->ras_num_mca_recs += (control->ras_num_recs - i); 3196 break; 3197 } 3198 } 3199 } else { 3200 control->ras_num_mca_recs = control->ras_num_recs; 3201 } 3202 } 3203 3204 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3205 if (ret) 3206 goto out; 3207 3208 ret = amdgpu_ras_eeprom_check(control); 3209 if (ret) 3210 goto out; 3211 3212 /* HW not usable */ 3213 if (amdgpu_ras_is_rma(adev)) 3214 ret = -EHWPOISON; 3215 } 3216 3217 out: 3218 kfree(bps); 3219 return ret; 3220 } 3221 3222 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3223 uint64_t addr) 3224 { 3225 struct ras_err_handler_data *data = con->eh_data; 3226 struct amdgpu_device *adev = con->adev; 3227 int i; 3228 3229 if ((addr >= adev->gmc.mc_vram_size && 3230 adev->gmc.mc_vram_size) || 3231 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3232 return -EINVAL; 3233 3234 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3235 for (i = 0; i < data->count; i++) 3236 if (addr == data->bps[i].retired_page) 3237 return 1; 3238 3239 return 0; 3240 } 3241 3242 /* 3243 * check if an address belongs to bad page 3244 * 3245 * Note: this check is only for umc block 3246 */ 3247 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3248 uint64_t addr) 3249 { 3250 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3251 int ret = 0; 3252 3253 if (!con || !con->eh_data) 3254 return ret; 3255 3256 mutex_lock(&con->recovery_lock); 3257 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3258 mutex_unlock(&con->recovery_lock); 3259 return ret; 3260 } 3261 3262 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3263 uint32_t max_count) 3264 { 3265 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3266 3267 /* 3268 * amdgpu_bad_page_threshold is used to config 3269 * the threshold for the number of bad pages. 3270 * -1: Threshold is set to default value 3271 * Driver will issue a warning message when threshold is reached 3272 * and continue runtime services. 3273 * 0: Disable bad page retirement 3274 * Driver will not retire bad pages 3275 * which is intended for debugging purpose. 3276 * -2: Threshold is determined by a formula 3277 * that assumes 1 bad page per 100M of local memory. 3278 * Driver will continue runtime services when threhold is reached. 3279 * 0 < threshold < max number of bad page records in EEPROM, 3280 * A user-defined threshold is set 3281 * Driver will halt runtime services when this custom threshold is reached. 3282 */ 3283 if (amdgpu_bad_page_threshold == -2) { 3284 u64 val = adev->gmc.mc_vram_size; 3285 3286 do_div(val, RAS_BAD_PAGE_COVER); 3287 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3288 max_count); 3289 } else if (amdgpu_bad_page_threshold == -1) { 3290 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3291 } else { 3292 con->bad_page_cnt_threshold = min_t(int, max_count, 3293 amdgpu_bad_page_threshold); 3294 } 3295 } 3296 3297 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3298 enum amdgpu_ras_block block, uint16_t pasid, 3299 pasid_notify pasid_fn, void *data, uint32_t reset) 3300 { 3301 int ret = 0; 3302 struct ras_poison_msg poison_msg; 3303 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3304 3305 memset(&poison_msg, 0, sizeof(poison_msg)); 3306 poison_msg.block = block; 3307 poison_msg.pasid = pasid; 3308 poison_msg.reset = reset; 3309 poison_msg.pasid_fn = pasid_fn; 3310 poison_msg.data = data; 3311 3312 ret = kfifo_put(&con->poison_fifo, poison_msg); 3313 if (!ret) { 3314 dev_err(adev->dev, "Poison message fifo is full!\n"); 3315 return -ENOSPC; 3316 } 3317 3318 return 0; 3319 } 3320 3321 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3322 struct ras_poison_msg *poison_msg) 3323 { 3324 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3325 3326 return kfifo_get(&con->poison_fifo, poison_msg); 3327 } 3328 3329 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3330 { 3331 mutex_init(&ecc_log->lock); 3332 3333 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3334 ecc_log->de_queried_count = 0; 3335 ecc_log->consumption_q_count = 0; 3336 } 3337 3338 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3339 { 3340 struct radix_tree_iter iter; 3341 void __rcu **slot; 3342 struct ras_ecc_err *ecc_err; 3343 3344 mutex_lock(&ecc_log->lock); 3345 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3346 ecc_err = radix_tree_deref_slot(slot); 3347 kfree(ecc_err->err_pages.pfn); 3348 kfree(ecc_err); 3349 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3350 } 3351 mutex_unlock(&ecc_log->lock); 3352 3353 mutex_destroy(&ecc_log->lock); 3354 ecc_log->de_queried_count = 0; 3355 ecc_log->consumption_q_count = 0; 3356 } 3357 3358 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3359 uint32_t delayed_ms) 3360 { 3361 int ret; 3362 3363 mutex_lock(&con->umc_ecc_log.lock); 3364 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3365 UMC_ECC_NEW_DETECTED_TAG); 3366 mutex_unlock(&con->umc_ecc_log.lock); 3367 3368 if (ret) 3369 schedule_delayed_work(&con->page_retirement_dwork, 3370 msecs_to_jiffies(delayed_ms)); 3371 3372 return ret ? true : false; 3373 } 3374 3375 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3376 { 3377 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3378 page_retirement_dwork.work); 3379 struct amdgpu_device *adev = con->adev; 3380 struct ras_err_data err_data; 3381 3382 /* If gpu reset is ongoing, delay retiring the bad pages */ 3383 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3384 amdgpu_ras_schedule_retirement_dwork(con, 3385 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3386 return; 3387 } 3388 3389 amdgpu_ras_error_data_init(&err_data); 3390 3391 amdgpu_umc_handle_bad_pages(adev, &err_data); 3392 3393 amdgpu_ras_error_data_fini(&err_data); 3394 3395 amdgpu_ras_schedule_retirement_dwork(con, 3396 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3397 } 3398 3399 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3400 uint32_t poison_creation_count) 3401 { 3402 int ret = 0; 3403 struct ras_ecc_log_info *ecc_log; 3404 struct ras_query_if info; 3405 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3406 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3407 u64 de_queried_count; 3408 u64 consumption_q_count; 3409 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3410 3411 memset(&info, 0, sizeof(info)); 3412 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3413 3414 ecc_log = &ras->umc_ecc_log; 3415 ecc_log->de_queried_count = 0; 3416 ecc_log->consumption_q_count = 0; 3417 3418 do { 3419 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3420 if (ret) 3421 return ret; 3422 3423 de_queried_count = ecc_log->de_queried_count; 3424 consumption_q_count = ecc_log->consumption_q_count; 3425 3426 if (de_queried_count && consumption_q_count) 3427 break; 3428 3429 msleep(100); 3430 } while (--timeout); 3431 3432 if (de_queried_count) 3433 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3434 3435 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3436 amdgpu_ras_reset_gpu(adev); 3437 3438 return 0; 3439 } 3440 3441 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3442 { 3443 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3444 struct ras_poison_msg msg; 3445 int ret; 3446 3447 do { 3448 ret = kfifo_get(&con->poison_fifo, &msg); 3449 } while (ret); 3450 } 3451 3452 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3453 uint32_t msg_count, uint32_t *gpu_reset) 3454 { 3455 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3456 uint32_t reset_flags = 0, reset = 0; 3457 struct ras_poison_msg msg; 3458 int ret, i; 3459 3460 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3461 3462 for (i = 0; i < msg_count; i++) { 3463 ret = amdgpu_ras_get_poison_req(adev, &msg); 3464 if (!ret) 3465 continue; 3466 3467 if (msg.pasid_fn) 3468 msg.pasid_fn(adev, msg.pasid, msg.data); 3469 3470 reset_flags |= msg.reset; 3471 } 3472 3473 /* 3474 * Try to ensure poison creation handler is completed first 3475 * to set rma if bad page exceed threshold. 3476 */ 3477 flush_delayed_work(&con->page_retirement_dwork); 3478 3479 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3480 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3481 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3482 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3483 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3484 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3485 else 3486 reset = reset_flags; 3487 3488 con->gpu_reset_flags |= reset; 3489 amdgpu_ras_reset_gpu(adev); 3490 3491 *gpu_reset = reset; 3492 3493 /* Wait for gpu recovery to complete */ 3494 flush_work(&con->recovery_work); 3495 } 3496 3497 return 0; 3498 } 3499 3500 static int amdgpu_ras_page_retirement_thread(void *param) 3501 { 3502 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3503 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3504 uint32_t poison_creation_count, msg_count; 3505 uint32_t gpu_reset; 3506 int ret; 3507 3508 while (!kthread_should_stop()) { 3509 3510 wait_event_interruptible(con->page_retirement_wq, 3511 kthread_should_stop() || 3512 atomic_read(&con->page_retirement_req_cnt)); 3513 3514 if (kthread_should_stop()) 3515 break; 3516 3517 mutex_lock(&con->poison_lock); 3518 gpu_reset = 0; 3519 3520 do { 3521 poison_creation_count = atomic_read(&con->poison_creation_count); 3522 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3523 if (ret == -EIO) 3524 break; 3525 3526 if (poison_creation_count) { 3527 atomic_sub(poison_creation_count, &con->poison_creation_count); 3528 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3529 } 3530 } while (atomic_read(&con->poison_creation_count) && 3531 !atomic_read(&con->poison_consumption_count)); 3532 3533 if (ret != -EIO) { 3534 msg_count = kfifo_len(&con->poison_fifo); 3535 if (msg_count) { 3536 ret = amdgpu_ras_poison_consumption_handler(adev, 3537 msg_count, &gpu_reset); 3538 if ((ret != -EIO) && 3539 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3540 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3541 } 3542 } 3543 3544 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3545 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3546 /* Clear poison creation request */ 3547 atomic_set(&con->poison_creation_count, 0); 3548 atomic_set(&con->poison_consumption_count, 0); 3549 3550 /* Clear poison fifo */ 3551 amdgpu_ras_clear_poison_fifo(adev); 3552 3553 /* Clear all poison requests */ 3554 atomic_set(&con->page_retirement_req_cnt, 0); 3555 3556 if (ret == -EIO) { 3557 /* Wait for mode-1 reset to complete */ 3558 down_read(&adev->reset_domain->sem); 3559 up_read(&adev->reset_domain->sem); 3560 } 3561 3562 /* Wake up work to save bad pages to eeprom */ 3563 schedule_delayed_work(&con->page_retirement_dwork, 0); 3564 } else if (gpu_reset) { 3565 /* gpu just completed mode-2 reset or other reset */ 3566 /* Clear poison consumption messages cached in fifo */ 3567 msg_count = kfifo_len(&con->poison_fifo); 3568 if (msg_count) { 3569 amdgpu_ras_clear_poison_fifo(adev); 3570 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3571 } 3572 3573 atomic_set(&con->poison_consumption_count, 0); 3574 3575 /* Wake up work to save bad pages to eeprom */ 3576 schedule_delayed_work(&con->page_retirement_dwork, 0); 3577 } 3578 mutex_unlock(&con->poison_lock); 3579 } 3580 3581 return 0; 3582 } 3583 3584 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3585 { 3586 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3587 struct amdgpu_ras_eeprom_control *control; 3588 int ret; 3589 3590 if (!con || amdgpu_sriov_vf(adev)) 3591 return 0; 3592 3593 control = &con->eeprom_control; 3594 ret = amdgpu_ras_eeprom_init(control); 3595 control->is_eeprom_valid = !ret; 3596 3597 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3598 control->ras_num_pa_recs = control->ras_num_recs; 3599 3600 if (adev->umc.ras && 3601 adev->umc.ras->get_retire_flip_bits) 3602 adev->umc.ras->get_retire_flip_bits(adev); 3603 3604 if (control->ras_num_recs && control->is_eeprom_valid) { 3605 ret = amdgpu_ras_load_bad_pages(adev); 3606 if (ret) { 3607 control->is_eeprom_valid = false; 3608 return 0; 3609 } 3610 3611 amdgpu_dpm_send_hbm_bad_pages_num( 3612 adev, control->ras_num_bad_pages); 3613 3614 if (con->update_channel_flag == true) { 3615 amdgpu_dpm_send_hbm_bad_channel_flag( 3616 adev, control->bad_channel_bitmap); 3617 con->update_channel_flag = false; 3618 } 3619 3620 /* The format action is only applied to new ASICs */ 3621 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3622 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3623 if (!amdgpu_ras_eeprom_reset_table(control)) 3624 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3625 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3626 } 3627 3628 return 0; 3629 } 3630 3631 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3632 { 3633 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3634 struct ras_err_handler_data **data; 3635 u32 max_eeprom_records_count = 0; 3636 int ret; 3637 3638 if (!con || amdgpu_sriov_vf(adev)) 3639 return 0; 3640 3641 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3642 * supports RAS and debugfs is enabled, but when 3643 * adev->ras_enabled is unset, i.e. when "ras_enable" 3644 * module parameter is set to 0. 3645 */ 3646 con->adev = adev; 3647 3648 if (!adev->ras_enabled) 3649 return 0; 3650 3651 data = &con->eh_data; 3652 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3653 if (!*data) { 3654 ret = -ENOMEM; 3655 goto out; 3656 } 3657 3658 mutex_init(&con->recovery_lock); 3659 mutex_init(&con->poison_lock); 3660 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3661 atomic_set(&con->in_recovery, 0); 3662 atomic_set(&con->rma_in_recovery, 0); 3663 con->eeprom_control.bad_channel_bitmap = 0; 3664 3665 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3666 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3667 3668 if (init_bp_info) { 3669 ret = amdgpu_ras_init_badpage_info(adev); 3670 if (ret) 3671 goto free; 3672 } 3673 3674 mutex_init(&con->page_rsv_lock); 3675 INIT_KFIFO(con->poison_fifo); 3676 mutex_init(&con->page_retirement_lock); 3677 init_waitqueue_head(&con->page_retirement_wq); 3678 atomic_set(&con->page_retirement_req_cnt, 0); 3679 atomic_set(&con->poison_creation_count, 0); 3680 atomic_set(&con->poison_consumption_count, 0); 3681 con->page_retirement_thread = 3682 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3683 if (IS_ERR(con->page_retirement_thread)) { 3684 con->page_retirement_thread = NULL; 3685 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3686 } 3687 3688 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3689 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3690 #ifdef CONFIG_X86_MCE_AMD 3691 if ((adev->asic_type == CHIP_ALDEBARAN) && 3692 (adev->gmc.xgmi.connected_to_cpu)) 3693 amdgpu_register_bad_pages_mca_notifier(adev); 3694 #endif 3695 return 0; 3696 3697 free: 3698 kfree((*data)->bps); 3699 kfree(*data); 3700 con->eh_data = NULL; 3701 out: 3702 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3703 3704 /* 3705 * Except error threshold exceeding case, other failure cases in this 3706 * function would not fail amdgpu driver init. 3707 */ 3708 if (!amdgpu_ras_is_rma(adev)) 3709 ret = 0; 3710 else 3711 ret = -EINVAL; 3712 3713 return ret; 3714 } 3715 3716 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3717 { 3718 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3719 struct ras_err_handler_data *data = con->eh_data; 3720 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3721 bool ret; 3722 3723 /* recovery_init failed to init it, fini is useless */ 3724 if (!data) 3725 return 0; 3726 3727 /* Save all cached bad pages to eeprom */ 3728 do { 3729 flush_delayed_work(&con->page_retirement_dwork); 3730 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3731 } while (ret && max_flush_timeout--); 3732 3733 if (con->page_retirement_thread) 3734 kthread_stop(con->page_retirement_thread); 3735 3736 atomic_set(&con->page_retirement_req_cnt, 0); 3737 atomic_set(&con->poison_creation_count, 0); 3738 3739 mutex_destroy(&con->page_rsv_lock); 3740 3741 cancel_work_sync(&con->recovery_work); 3742 3743 cancel_delayed_work_sync(&con->page_retirement_dwork); 3744 3745 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3746 3747 mutex_lock(&con->recovery_lock); 3748 con->eh_data = NULL; 3749 kfree(data->bps); 3750 kfree(data); 3751 mutex_unlock(&con->recovery_lock); 3752 3753 amdgpu_ras_critical_region_init(adev); 3754 3755 return 0; 3756 } 3757 /* recovery end */ 3758 3759 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3760 { 3761 if (amdgpu_sriov_vf(adev)) { 3762 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3763 case IP_VERSION(13, 0, 2): 3764 case IP_VERSION(13, 0, 6): 3765 case IP_VERSION(13, 0, 12): 3766 case IP_VERSION(13, 0, 14): 3767 return true; 3768 default: 3769 return false; 3770 } 3771 } 3772 3773 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3774 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3775 case IP_VERSION(13, 0, 0): 3776 case IP_VERSION(13, 0, 6): 3777 case IP_VERSION(13, 0, 10): 3778 case IP_VERSION(13, 0, 12): 3779 case IP_VERSION(13, 0, 14): 3780 case IP_VERSION(14, 0, 3): 3781 return true; 3782 default: 3783 return false; 3784 } 3785 } 3786 3787 return adev->asic_type == CHIP_VEGA10 || 3788 adev->asic_type == CHIP_VEGA20 || 3789 adev->asic_type == CHIP_ARCTURUS || 3790 adev->asic_type == CHIP_ALDEBARAN || 3791 adev->asic_type == CHIP_SIENNA_CICHLID; 3792 } 3793 3794 /* 3795 * this is workaround for vega20 workstation sku, 3796 * force enable gfx ras, ignore vbios gfx ras flag 3797 * due to GC EDC can not write 3798 */ 3799 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3800 { 3801 struct atom_context *ctx = adev->mode_info.atom_context; 3802 3803 if (!ctx) 3804 return; 3805 3806 if (strnstr(ctx->vbios_pn, "D16406", 3807 sizeof(ctx->vbios_pn)) || 3808 strnstr(ctx->vbios_pn, "D36002", 3809 sizeof(ctx->vbios_pn))) 3810 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3811 } 3812 3813 /* Query ras capablity via atomfirmware interface */ 3814 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3815 { 3816 /* mem_ecc cap */ 3817 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3818 dev_info(adev->dev, "MEM ECC is active.\n"); 3819 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3820 1 << AMDGPU_RAS_BLOCK__DF); 3821 } else { 3822 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3823 } 3824 3825 /* sram_ecc cap */ 3826 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 3827 dev_info(adev->dev, "SRAM ECC is active.\n"); 3828 if (!amdgpu_sriov_vf(adev)) 3829 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 3830 1 << AMDGPU_RAS_BLOCK__DF); 3831 else 3832 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 3833 1 << AMDGPU_RAS_BLOCK__SDMA | 3834 1 << AMDGPU_RAS_BLOCK__GFX); 3835 3836 /* 3837 * VCN/JPEG RAS can be supported on both bare metal and 3838 * SRIOV environment 3839 */ 3840 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 3841 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 3842 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 3843 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 3844 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 3845 1 << AMDGPU_RAS_BLOCK__JPEG); 3846 else 3847 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 3848 1 << AMDGPU_RAS_BLOCK__JPEG); 3849 3850 /* 3851 * XGMI RAS is not supported if xgmi num physical nodes 3852 * is zero 3853 */ 3854 if (!adev->gmc.xgmi.num_physical_nodes) 3855 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 3856 } else { 3857 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 3858 } 3859 } 3860 3861 /* Query poison mode from umc/df IP callbacks */ 3862 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 3863 { 3864 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3865 bool df_poison, umc_poison; 3866 3867 /* poison setting is useless on SRIOV guest */ 3868 if (amdgpu_sriov_vf(adev) || !con) 3869 return; 3870 3871 /* Init poison supported flag, the default value is false */ 3872 if (adev->gmc.xgmi.connected_to_cpu || 3873 adev->gmc.is_app_apu) { 3874 /* enabled by default when GPU is connected to CPU */ 3875 con->poison_supported = true; 3876 } else if (adev->df.funcs && 3877 adev->df.funcs->query_ras_poison_mode && 3878 adev->umc.ras && 3879 adev->umc.ras->query_ras_poison_mode) { 3880 df_poison = 3881 adev->df.funcs->query_ras_poison_mode(adev); 3882 umc_poison = 3883 adev->umc.ras->query_ras_poison_mode(adev); 3884 3885 /* Only poison is set in both DF and UMC, we can support it */ 3886 if (df_poison && umc_poison) 3887 con->poison_supported = true; 3888 else if (df_poison != umc_poison) 3889 dev_warn(adev->dev, 3890 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 3891 df_poison, umc_poison); 3892 } 3893 } 3894 3895 /* 3896 * check hardware's ras ability which will be saved in hw_supported. 3897 * if hardware does not support ras, we can skip some ras initializtion and 3898 * forbid some ras operations from IP. 3899 * if software itself, say boot parameter, limit the ras ability. We still 3900 * need allow IP do some limited operations, like disable. In such case, 3901 * we have to initialize ras as normal. but need check if operation is 3902 * allowed or not in each function. 3903 */ 3904 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 3905 { 3906 adev->ras_hw_enabled = adev->ras_enabled = 0; 3907 3908 if (!amdgpu_ras_asic_supported(adev)) 3909 return; 3910 3911 if (amdgpu_sriov_vf(adev)) { 3912 if (amdgpu_virt_get_ras_capability(adev)) 3913 goto init_ras_enabled_flag; 3914 } 3915 3916 /* query ras capability from psp */ 3917 if (amdgpu_psp_get_ras_capability(&adev->psp)) 3918 goto init_ras_enabled_flag; 3919 3920 /* query ras capablity from bios */ 3921 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 3922 amdgpu_ras_query_ras_capablity_from_vbios(adev); 3923 } else { 3924 /* driver only manages a few IP blocks RAS feature 3925 * when GPU is connected cpu through XGMI */ 3926 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 3927 1 << AMDGPU_RAS_BLOCK__SDMA | 3928 1 << AMDGPU_RAS_BLOCK__MMHUB); 3929 } 3930 3931 /* apply asic specific settings (vega20 only for now) */ 3932 amdgpu_ras_get_quirks(adev); 3933 3934 /* query poison mode from umc/df ip callback */ 3935 amdgpu_ras_query_poison_mode(adev); 3936 3937 init_ras_enabled_flag: 3938 /* hw_supported needs to be aligned with RAS block mask. */ 3939 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 3940 3941 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 3942 adev->ras_hw_enabled & amdgpu_ras_mask; 3943 3944 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 3945 if (!amdgpu_sriov_vf(adev)) { 3946 adev->aca.is_enabled = 3947 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 3948 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 3949 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 3950 } 3951 3952 /* bad page feature is not applicable to specific app platform */ 3953 if (adev->gmc.is_app_apu && 3954 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 3955 amdgpu_bad_page_threshold = 0; 3956 } 3957 3958 static void amdgpu_ras_counte_dw(struct work_struct *work) 3959 { 3960 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3961 ras_counte_delay_work.work); 3962 struct amdgpu_device *adev = con->adev; 3963 struct drm_device *dev = adev_to_drm(adev); 3964 unsigned long ce_count, ue_count; 3965 int res; 3966 3967 res = pm_runtime_get_sync(dev->dev); 3968 if (res < 0) 3969 goto Out; 3970 3971 /* Cache new values. 3972 */ 3973 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 3974 atomic_set(&con->ras_ce_count, ce_count); 3975 atomic_set(&con->ras_ue_count, ue_count); 3976 } 3977 3978 pm_runtime_mark_last_busy(dev->dev); 3979 Out: 3980 pm_runtime_put_autosuspend(dev->dev); 3981 } 3982 3983 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 3984 { 3985 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 3986 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 3987 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 3988 AMDGPU_RAS_ERROR__PARITY; 3989 } 3990 3991 static void ras_event_mgr_init(struct ras_event_manager *mgr) 3992 { 3993 struct ras_event_state *event_state; 3994 int i; 3995 3996 memset(mgr, 0, sizeof(*mgr)); 3997 atomic64_set(&mgr->seqno, 0); 3998 3999 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4000 event_state = &mgr->event_state[i]; 4001 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4002 atomic64_set(&event_state->count, 0); 4003 } 4004 } 4005 4006 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4007 { 4008 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4009 struct amdgpu_hive_info *hive; 4010 4011 if (!ras) 4012 return; 4013 4014 hive = amdgpu_get_xgmi_hive(adev); 4015 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4016 4017 /* init event manager with node 0 on xgmi system */ 4018 if (!amdgpu_reset_in_recovery(adev)) { 4019 if (!hive || adev->gmc.xgmi.node_id == 0) 4020 ras_event_mgr_init(ras->event_mgr); 4021 } 4022 4023 if (hive) 4024 amdgpu_put_xgmi_hive(hive); 4025 } 4026 4027 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4028 { 4029 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4030 4031 if (!con || (adev->flags & AMD_IS_APU)) 4032 return; 4033 4034 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4035 case IP_VERSION(13, 0, 2): 4036 case IP_VERSION(13, 0, 6): 4037 case IP_VERSION(13, 0, 12): 4038 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4039 break; 4040 case IP_VERSION(13, 0, 14): 4041 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4042 break; 4043 default: 4044 break; 4045 } 4046 } 4047 4048 int amdgpu_ras_init(struct amdgpu_device *adev) 4049 { 4050 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4051 int r; 4052 4053 if (con) 4054 return 0; 4055 4056 con = kzalloc(sizeof(*con) + 4057 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4058 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4059 GFP_KERNEL); 4060 if (!con) 4061 return -ENOMEM; 4062 4063 con->adev = adev; 4064 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4065 atomic_set(&con->ras_ce_count, 0); 4066 atomic_set(&con->ras_ue_count, 0); 4067 4068 con->objs = (struct ras_manager *)(con + 1); 4069 4070 amdgpu_ras_set_context(adev, con); 4071 4072 amdgpu_ras_check_supported(adev); 4073 4074 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4075 /* set gfx block ras context feature for VEGA20 Gaming 4076 * send ras disable cmd to ras ta during ras late init. 4077 */ 4078 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4079 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4080 4081 return 0; 4082 } 4083 4084 r = 0; 4085 goto release_con; 4086 } 4087 4088 con->update_channel_flag = false; 4089 con->features = 0; 4090 con->schema = 0; 4091 INIT_LIST_HEAD(&con->head); 4092 /* Might need get this flag from vbios. */ 4093 con->flags = RAS_DEFAULT_FLAGS; 4094 4095 /* initialize nbio ras function ahead of any other 4096 * ras functions so hardware fatal error interrupt 4097 * can be enabled as early as possible */ 4098 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4099 case IP_VERSION(7, 4, 0): 4100 case IP_VERSION(7, 4, 1): 4101 case IP_VERSION(7, 4, 4): 4102 if (!adev->gmc.xgmi.connected_to_cpu) 4103 adev->nbio.ras = &nbio_v7_4_ras; 4104 break; 4105 case IP_VERSION(4, 3, 0): 4106 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4107 /* unlike other generation of nbio ras, 4108 * nbio v4_3 only support fatal error interrupt 4109 * to inform software that DF is freezed due to 4110 * system fatal error event. driver should not 4111 * enable nbio ras in such case. Instead, 4112 * check DF RAS */ 4113 adev->nbio.ras = &nbio_v4_3_ras; 4114 break; 4115 case IP_VERSION(6, 3, 1): 4116 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4117 /* unlike other generation of nbio ras, 4118 * nbif v6_3_1 only support fatal error interrupt 4119 * to inform software that DF is freezed due to 4120 * system fatal error event. driver should not 4121 * enable nbio ras in such case. Instead, 4122 * check DF RAS 4123 */ 4124 adev->nbio.ras = &nbif_v6_3_1_ras; 4125 break; 4126 case IP_VERSION(7, 9, 0): 4127 case IP_VERSION(7, 9, 1): 4128 if (!adev->gmc.is_app_apu) 4129 adev->nbio.ras = &nbio_v7_9_ras; 4130 break; 4131 default: 4132 /* nbio ras is not available */ 4133 break; 4134 } 4135 4136 /* nbio ras block needs to be enabled ahead of other ras blocks 4137 * to handle fatal error */ 4138 r = amdgpu_nbio_ras_sw_init(adev); 4139 if (r) 4140 return r; 4141 4142 if (adev->nbio.ras && 4143 adev->nbio.ras->init_ras_controller_interrupt) { 4144 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4145 if (r) 4146 goto release_con; 4147 } 4148 4149 if (adev->nbio.ras && 4150 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4151 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4152 if (r) 4153 goto release_con; 4154 } 4155 4156 /* Packed socket_id to ras feature mask bits[31:29] */ 4157 if (adev->smuio.funcs && 4158 adev->smuio.funcs->get_socket_id) 4159 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4160 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4161 4162 /* Get RAS schema for particular SOC */ 4163 con->schema = amdgpu_get_ras_schema(adev); 4164 4165 amdgpu_ras_init_reserved_vram_size(adev); 4166 4167 if (amdgpu_ras_fs_init(adev)) { 4168 r = -EINVAL; 4169 goto release_con; 4170 } 4171 4172 if (amdgpu_ras_aca_is_supported(adev)) { 4173 if (amdgpu_aca_is_enabled(adev)) 4174 r = amdgpu_aca_init(adev); 4175 else 4176 r = amdgpu_mca_init(adev); 4177 if (r) 4178 goto release_con; 4179 } 4180 4181 con->init_task_pid = task_pid_nr(current); 4182 get_task_comm(con->init_task_comm, current); 4183 4184 mutex_init(&con->critical_region_lock); 4185 INIT_LIST_HEAD(&con->critical_region_head); 4186 4187 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4188 "hardware ability[%x] ras_mask[%x]\n", 4189 adev->ras_hw_enabled, adev->ras_enabled); 4190 4191 return 0; 4192 release_con: 4193 amdgpu_ras_set_context(adev, NULL); 4194 kfree(con); 4195 4196 return r; 4197 } 4198 4199 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4200 { 4201 if (adev->gmc.xgmi.connected_to_cpu || 4202 adev->gmc.is_app_apu) 4203 return 1; 4204 return 0; 4205 } 4206 4207 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4208 struct ras_common_if *ras_block) 4209 { 4210 struct ras_query_if info = { 4211 .head = *ras_block, 4212 }; 4213 4214 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4215 return 0; 4216 4217 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4218 DRM_WARN("RAS init harvest failure"); 4219 4220 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4221 DRM_WARN("RAS init harvest reset failure"); 4222 4223 return 0; 4224 } 4225 4226 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4227 { 4228 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4229 4230 if (!con) 4231 return false; 4232 4233 return con->poison_supported; 4234 } 4235 4236 /* helper function to handle common stuff in ip late init phase */ 4237 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4238 struct ras_common_if *ras_block) 4239 { 4240 struct amdgpu_ras_block_object *ras_obj = NULL; 4241 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4242 struct ras_query_if *query_info; 4243 unsigned long ue_count, ce_count; 4244 int r; 4245 4246 /* disable RAS feature per IP block if it is not supported */ 4247 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4248 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4249 return 0; 4250 } 4251 4252 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4253 if (r) { 4254 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4255 /* in resume phase, if fail to enable ras, 4256 * clean up all ras fs nodes, and disable ras */ 4257 goto cleanup; 4258 } else 4259 return r; 4260 } 4261 4262 /* check for errors on warm reset edc persisant supported ASIC */ 4263 amdgpu_persistent_edc_harvesting(adev, ras_block); 4264 4265 /* in resume phase, no need to create ras fs node */ 4266 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4267 return 0; 4268 4269 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4270 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4271 (ras_obj->hw_ops->query_poison_status || 4272 ras_obj->hw_ops->handle_poison_consumption))) { 4273 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4274 if (r) 4275 goto cleanup; 4276 } 4277 4278 if (ras_obj->hw_ops && 4279 (ras_obj->hw_ops->query_ras_error_count || 4280 ras_obj->hw_ops->query_ras_error_status)) { 4281 r = amdgpu_ras_sysfs_create(adev, ras_block); 4282 if (r) 4283 goto interrupt; 4284 4285 /* Those are the cached values at init. 4286 */ 4287 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4288 if (!query_info) 4289 return -ENOMEM; 4290 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4291 4292 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4293 atomic_set(&con->ras_ce_count, ce_count); 4294 atomic_set(&con->ras_ue_count, ue_count); 4295 } 4296 4297 kfree(query_info); 4298 } 4299 4300 return 0; 4301 4302 interrupt: 4303 if (ras_obj->ras_cb) 4304 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4305 cleanup: 4306 amdgpu_ras_feature_enable(adev, ras_block, 0); 4307 return r; 4308 } 4309 4310 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4311 struct ras_common_if *ras_block) 4312 { 4313 return amdgpu_ras_block_late_init(adev, ras_block); 4314 } 4315 4316 /* helper function to remove ras fs node and interrupt handler */ 4317 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4318 struct ras_common_if *ras_block) 4319 { 4320 struct amdgpu_ras_block_object *ras_obj; 4321 if (!ras_block) 4322 return; 4323 4324 amdgpu_ras_sysfs_remove(adev, ras_block); 4325 4326 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4327 if (ras_obj->ras_cb) 4328 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4329 } 4330 4331 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4332 struct ras_common_if *ras_block) 4333 { 4334 return amdgpu_ras_block_late_fini(adev, ras_block); 4335 } 4336 4337 /* do some init work after IP late init as dependence. 4338 * and it runs in resume/gpu reset/booting up cases. 4339 */ 4340 void amdgpu_ras_resume(struct amdgpu_device *adev) 4341 { 4342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4343 struct ras_manager *obj, *tmp; 4344 4345 if (!adev->ras_enabled || !con) { 4346 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4347 amdgpu_release_ras_context(adev); 4348 4349 return; 4350 } 4351 4352 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4353 /* Set up all other IPs which are not implemented. There is a 4354 * tricky thing that IP's actual ras error type should be 4355 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4356 * ERROR_NONE make sense anyway. 4357 */ 4358 amdgpu_ras_enable_all_features(adev, 1); 4359 4360 /* We enable ras on all hw_supported block, but as boot 4361 * parameter might disable some of them and one or more IP has 4362 * not implemented yet. So we disable them on behalf. 4363 */ 4364 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4365 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4366 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4367 /* there should be no any reference. */ 4368 WARN_ON(alive_obj(obj)); 4369 } 4370 } 4371 } 4372 } 4373 4374 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4375 { 4376 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4377 4378 if (!adev->ras_enabled || !con) 4379 return; 4380 4381 amdgpu_ras_disable_all_features(adev, 0); 4382 /* Make sure all ras objects are disabled. */ 4383 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4384 amdgpu_ras_disable_all_features(adev, 1); 4385 } 4386 4387 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4388 { 4389 struct amdgpu_ras_block_list *node, *tmp; 4390 struct amdgpu_ras_block_object *obj; 4391 int r; 4392 4393 amdgpu_ras_event_mgr_init(adev); 4394 4395 if (amdgpu_ras_aca_is_supported(adev)) { 4396 if (amdgpu_reset_in_recovery(adev)) { 4397 if (amdgpu_aca_is_enabled(adev)) 4398 r = amdgpu_aca_reset(adev); 4399 else 4400 r = amdgpu_mca_reset(adev); 4401 if (r) 4402 return r; 4403 } 4404 4405 if (!amdgpu_sriov_vf(adev)) { 4406 if (amdgpu_aca_is_enabled(adev)) 4407 amdgpu_ras_set_aca_debug_mode(adev, false); 4408 else 4409 amdgpu_ras_set_mca_debug_mode(adev, false); 4410 } 4411 } 4412 4413 /* Guest side doesn't need init ras feature */ 4414 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4415 return 0; 4416 4417 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4418 obj = node->ras_obj; 4419 if (!obj) { 4420 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4421 continue; 4422 } 4423 4424 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4425 continue; 4426 4427 if (obj->ras_late_init) { 4428 r = obj->ras_late_init(adev, &obj->ras_comm); 4429 if (r) { 4430 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4431 obj->ras_comm.name, r); 4432 return r; 4433 } 4434 } else 4435 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4436 } 4437 4438 return 0; 4439 } 4440 4441 /* do some fini work before IP fini as dependence */ 4442 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4443 { 4444 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4445 4446 if (!adev->ras_enabled || !con) 4447 return 0; 4448 4449 4450 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4451 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4452 amdgpu_ras_disable_all_features(adev, 0); 4453 amdgpu_ras_recovery_fini(adev); 4454 return 0; 4455 } 4456 4457 int amdgpu_ras_fini(struct amdgpu_device *adev) 4458 { 4459 struct amdgpu_ras_block_list *ras_node, *tmp; 4460 struct amdgpu_ras_block_object *obj = NULL; 4461 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4462 4463 if (!adev->ras_enabled || !con) 4464 return 0; 4465 4466 amdgpu_ras_critical_region_fini(adev); 4467 mutex_destroy(&con->critical_region_lock); 4468 4469 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4470 if (ras_node->ras_obj) { 4471 obj = ras_node->ras_obj; 4472 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4473 obj->ras_fini) 4474 obj->ras_fini(adev, &obj->ras_comm); 4475 else 4476 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4477 } 4478 4479 /* Clear ras blocks from ras_list and free ras block list node */ 4480 list_del(&ras_node->node); 4481 kfree(ras_node); 4482 } 4483 4484 amdgpu_ras_fs_fini(adev); 4485 amdgpu_ras_interrupt_remove_all(adev); 4486 4487 if (amdgpu_ras_aca_is_supported(adev)) { 4488 if (amdgpu_aca_is_enabled(adev)) 4489 amdgpu_aca_fini(adev); 4490 else 4491 amdgpu_mca_fini(adev); 4492 } 4493 4494 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4495 4496 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4497 amdgpu_ras_disable_all_features(adev, 0); 4498 4499 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4500 4501 amdgpu_ras_set_context(adev, NULL); 4502 kfree(con); 4503 4504 return 0; 4505 } 4506 4507 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4508 { 4509 struct amdgpu_ras *ras; 4510 4511 ras = amdgpu_ras_get_context(adev); 4512 if (!ras) 4513 return false; 4514 4515 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4516 } 4517 4518 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4519 { 4520 struct amdgpu_ras *ras; 4521 4522 ras = amdgpu_ras_get_context(adev); 4523 if (ras) { 4524 if (status) 4525 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4526 else 4527 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4528 } 4529 } 4530 4531 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4532 { 4533 struct amdgpu_ras *ras; 4534 4535 ras = amdgpu_ras_get_context(adev); 4536 if (ras) { 4537 ras->ras_err_state = 0; 4538 ras->gpu_reset_flags = 0; 4539 } 4540 } 4541 4542 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4543 enum amdgpu_ras_block block) 4544 { 4545 struct amdgpu_ras *ras; 4546 4547 ras = amdgpu_ras_get_context(adev); 4548 if (ras) 4549 set_bit(block, &ras->ras_err_state); 4550 } 4551 4552 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4553 { 4554 struct amdgpu_ras *ras; 4555 4556 ras = amdgpu_ras_get_context(adev); 4557 if (ras) { 4558 if (block == AMDGPU_RAS_BLOCK__ANY) 4559 return (ras->ras_err_state != 0); 4560 else 4561 return test_bit(block, &ras->ras_err_state) || 4562 test_bit(AMDGPU_RAS_BLOCK__LAST, 4563 &ras->ras_err_state); 4564 } 4565 4566 return false; 4567 } 4568 4569 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4570 { 4571 struct amdgpu_ras *ras; 4572 4573 ras = amdgpu_ras_get_context(adev); 4574 if (!ras) 4575 return NULL; 4576 4577 return ras->event_mgr; 4578 } 4579 4580 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4581 const void *caller) 4582 { 4583 struct ras_event_manager *event_mgr; 4584 struct ras_event_state *event_state; 4585 int ret = 0; 4586 4587 if (type >= RAS_EVENT_TYPE_COUNT) { 4588 ret = -EINVAL; 4589 goto out; 4590 } 4591 4592 event_mgr = __get_ras_event_mgr(adev); 4593 if (!event_mgr) { 4594 ret = -EINVAL; 4595 goto out; 4596 } 4597 4598 event_state = &event_mgr->event_state[type]; 4599 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4600 atomic64_inc(&event_state->count); 4601 4602 out: 4603 if (ret && caller) 4604 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4605 (int)type, caller, ret); 4606 4607 return ret; 4608 } 4609 4610 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4611 { 4612 struct ras_event_manager *event_mgr; 4613 u64 id; 4614 4615 if (type >= RAS_EVENT_TYPE_COUNT) 4616 return RAS_EVENT_INVALID_ID; 4617 4618 switch (type) { 4619 case RAS_EVENT_TYPE_FATAL: 4620 case RAS_EVENT_TYPE_POISON_CREATION: 4621 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4622 event_mgr = __get_ras_event_mgr(adev); 4623 if (!event_mgr) 4624 return RAS_EVENT_INVALID_ID; 4625 4626 id = event_mgr->event_state[type].last_seqno; 4627 break; 4628 case RAS_EVENT_TYPE_INVALID: 4629 default: 4630 id = RAS_EVENT_INVALID_ID; 4631 break; 4632 } 4633 4634 return id; 4635 } 4636 4637 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4638 { 4639 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4640 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4641 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4642 u64 event_id; 4643 4644 if (amdgpu_ras_mark_ras_event(adev, type)) { 4645 dev_err(adev->dev, 4646 "uncorrectable hardware error (ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4647 return; 4648 } 4649 4650 event_id = amdgpu_ras_acquire_event_id(adev, type); 4651 4652 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4653 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4654 4655 amdgpu_ras_set_fed(adev, true); 4656 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4657 amdgpu_ras_reset_gpu(adev); 4658 } 4659 } 4660 4661 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4662 { 4663 if (adev->asic_type == CHIP_VEGA20 && 4664 adev->pm.fw_version <= 0x283400) { 4665 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4666 amdgpu_ras_intr_triggered(); 4667 } 4668 4669 return false; 4670 } 4671 4672 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4673 { 4674 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4675 4676 if (!con) 4677 return; 4678 4679 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4680 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4681 amdgpu_ras_set_context(adev, NULL); 4682 kfree(con); 4683 } 4684 } 4685 4686 #ifdef CONFIG_X86_MCE_AMD 4687 static struct amdgpu_device *find_adev(uint32_t node_id) 4688 { 4689 int i; 4690 struct amdgpu_device *adev = NULL; 4691 4692 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4693 adev = mce_adev_list.devs[i]; 4694 4695 if (adev && adev->gmc.xgmi.connected_to_cpu && 4696 adev->gmc.xgmi.physical_node_id == node_id) 4697 break; 4698 adev = NULL; 4699 } 4700 4701 return adev; 4702 } 4703 4704 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4705 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4706 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4707 #define GPU_ID_OFFSET 8 4708 4709 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4710 unsigned long val, void *data) 4711 { 4712 struct mce *m = (struct mce *)data; 4713 struct amdgpu_device *adev = NULL; 4714 uint32_t gpu_id = 0; 4715 uint32_t umc_inst = 0, ch_inst = 0; 4716 4717 /* 4718 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4719 * and error occurred in DramECC (Extended error code = 0) then only 4720 * process the error, else bail out. 4721 */ 4722 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4723 (XEC(m->status, 0x3f) == 0x0))) 4724 return NOTIFY_DONE; 4725 4726 /* 4727 * If it is correctable error, return. 4728 */ 4729 if (mce_is_correctable(m)) 4730 return NOTIFY_OK; 4731 4732 /* 4733 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4734 */ 4735 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4736 4737 adev = find_adev(gpu_id); 4738 if (!adev) { 4739 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4740 gpu_id); 4741 return NOTIFY_DONE; 4742 } 4743 4744 /* 4745 * If it is uncorrectable error, then find out UMC instance and 4746 * channel index. 4747 */ 4748 umc_inst = GET_UMC_INST(m->ipid); 4749 ch_inst = GET_CHAN_INDEX(m->ipid); 4750 4751 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4752 umc_inst, ch_inst); 4753 4754 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4755 return NOTIFY_OK; 4756 else 4757 return NOTIFY_DONE; 4758 } 4759 4760 static struct notifier_block amdgpu_bad_page_nb = { 4761 .notifier_call = amdgpu_bad_page_notifier, 4762 .priority = MCE_PRIO_UC, 4763 }; 4764 4765 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4766 { 4767 /* 4768 * Add the adev to the mce_adev_list. 4769 * During mode2 reset, amdgpu device is temporarily 4770 * removed from the mgpu_info list which can cause 4771 * page retirement to fail. 4772 * Use this list instead of mgpu_info to find the amdgpu 4773 * device on which the UMC error was reported. 4774 */ 4775 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4776 4777 /* 4778 * Register the x86 notifier only once 4779 * with MCE subsystem. 4780 */ 4781 if (notifier_registered == false) { 4782 mce_register_decode_chain(&amdgpu_bad_page_nb); 4783 notifier_registered = true; 4784 } 4785 } 4786 #endif 4787 4788 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4789 { 4790 if (!adev) 4791 return NULL; 4792 4793 return adev->psp.ras_context.ras; 4794 } 4795 4796 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4797 { 4798 if (!adev) 4799 return -EINVAL; 4800 4801 adev->psp.ras_context.ras = ras_con; 4802 return 0; 4803 } 4804 4805 /* check if ras is supported on block, say, sdma, gfx */ 4806 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4807 unsigned int block) 4808 { 4809 int ret = 0; 4810 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4811 4812 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4813 return 0; 4814 4815 ret = ras && (adev->ras_enabled & (1 << block)); 4816 4817 /* For the special asic with mem ecc enabled but sram ecc 4818 * not enabled, even if the ras block is not supported on 4819 * .ras_enabled, if the asic supports poison mode and the 4820 * ras block has ras configuration, it can be considered 4821 * that the ras block supports ras function. 4822 */ 4823 if (!ret && 4824 (block == AMDGPU_RAS_BLOCK__GFX || 4825 block == AMDGPU_RAS_BLOCK__SDMA || 4826 block == AMDGPU_RAS_BLOCK__VCN || 4827 block == AMDGPU_RAS_BLOCK__JPEG) && 4828 (amdgpu_ras_mask & (1 << block)) && 4829 amdgpu_ras_is_poison_mode_supported(adev) && 4830 amdgpu_ras_get_ras_block(adev, block, 0)) 4831 ret = 1; 4832 4833 return ret; 4834 } 4835 4836 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 4837 { 4838 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4839 4840 /* mode1 is the only selection for RMA status */ 4841 if (amdgpu_ras_is_rma(adev)) { 4842 ras->gpu_reset_flags = 0; 4843 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4844 } 4845 4846 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 4847 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4848 int hive_ras_recovery = 0; 4849 4850 if (hive) { 4851 hive_ras_recovery = atomic_read(&hive->ras_recovery); 4852 amdgpu_put_xgmi_hive(hive); 4853 } 4854 /* In the case of multiple GPUs, after a GPU has started 4855 * resetting all GPUs on hive, other GPUs do not need to 4856 * trigger GPU reset again. 4857 */ 4858 if (!hive_ras_recovery) 4859 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4860 else 4861 atomic_set(&ras->in_recovery, 0); 4862 } else { 4863 flush_work(&ras->recovery_work); 4864 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4865 } 4866 4867 return 0; 4868 } 4869 4870 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 4871 { 4872 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4873 int ret = 0; 4874 4875 if (con) { 4876 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4877 if (!ret) 4878 con->is_aca_debug_mode = enable; 4879 } 4880 4881 return ret; 4882 } 4883 4884 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 4885 { 4886 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4887 int ret = 0; 4888 4889 if (con) { 4890 if (amdgpu_aca_is_enabled(adev)) 4891 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 4892 else 4893 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4894 if (!ret) 4895 con->is_aca_debug_mode = enable; 4896 } 4897 4898 return ret; 4899 } 4900 4901 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 4902 { 4903 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4904 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4905 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4906 4907 if (!con) 4908 return false; 4909 4910 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 4911 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 4912 return con->is_aca_debug_mode; 4913 else 4914 return true; 4915 } 4916 4917 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 4918 unsigned int *error_query_mode) 4919 { 4920 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4921 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4922 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4923 4924 if (!con) { 4925 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 4926 return false; 4927 } 4928 4929 if (amdgpu_sriov_vf(adev)) { 4930 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 4931 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 4932 *error_query_mode = 4933 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 4934 } else { 4935 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 4936 } 4937 4938 return true; 4939 } 4940 4941 /* Register each ip ras block into amdgpu ras */ 4942 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 4943 struct amdgpu_ras_block_object *ras_block_obj) 4944 { 4945 struct amdgpu_ras_block_list *ras_node; 4946 if (!adev || !ras_block_obj) 4947 return -EINVAL; 4948 4949 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 4950 if (!ras_node) 4951 return -ENOMEM; 4952 4953 INIT_LIST_HEAD(&ras_node->node); 4954 ras_node->ras_obj = ras_block_obj; 4955 list_add_tail(&ras_node->node, &adev->ras_list); 4956 4957 return 0; 4958 } 4959 4960 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 4961 { 4962 if (!err_type_name) 4963 return; 4964 4965 switch (err_type) { 4966 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 4967 sprintf(err_type_name, "correctable"); 4968 break; 4969 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 4970 sprintf(err_type_name, "uncorrectable"); 4971 break; 4972 default: 4973 sprintf(err_type_name, "unknown"); 4974 break; 4975 } 4976 } 4977 4978 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 4979 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4980 uint32_t instance, 4981 uint32_t *memory_id) 4982 { 4983 uint32_t err_status_lo_data, err_status_lo_offset; 4984 4985 if (!reg_entry) 4986 return false; 4987 4988 err_status_lo_offset = 4989 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4990 reg_entry->seg_lo, reg_entry->reg_lo); 4991 err_status_lo_data = RREG32(err_status_lo_offset); 4992 4993 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 4994 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 4995 return false; 4996 4997 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 4998 4999 return true; 5000 } 5001 5002 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5003 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5004 uint32_t instance, 5005 unsigned long *err_cnt) 5006 { 5007 uint32_t err_status_hi_data, err_status_hi_offset; 5008 5009 if (!reg_entry) 5010 return false; 5011 5012 err_status_hi_offset = 5013 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5014 reg_entry->seg_hi, reg_entry->reg_hi); 5015 err_status_hi_data = RREG32(err_status_hi_offset); 5016 5017 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5018 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5019 /* keep the check here in case we need to refer to the result later */ 5020 dev_dbg(adev->dev, "Invalid err_info field\n"); 5021 5022 /* read err count */ 5023 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5024 5025 return true; 5026 } 5027 5028 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5029 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5030 uint32_t reg_list_size, 5031 const struct amdgpu_ras_memory_id_entry *mem_list, 5032 uint32_t mem_list_size, 5033 uint32_t instance, 5034 uint32_t err_type, 5035 unsigned long *err_count) 5036 { 5037 uint32_t memory_id; 5038 unsigned long err_cnt; 5039 char err_type_name[16]; 5040 uint32_t i, j; 5041 5042 for (i = 0; i < reg_list_size; i++) { 5043 /* query memory_id from err_status_lo */ 5044 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5045 instance, &memory_id)) 5046 continue; 5047 5048 /* query err_cnt from err_status_hi */ 5049 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5050 instance, &err_cnt) || 5051 !err_cnt) 5052 continue; 5053 5054 *err_count += err_cnt; 5055 5056 /* log the errors */ 5057 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5058 if (!mem_list) { 5059 /* memory_list is not supported */ 5060 dev_info(adev->dev, 5061 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5062 err_cnt, err_type_name, 5063 reg_list[i].block_name, 5064 instance, memory_id); 5065 } else { 5066 for (j = 0; j < mem_list_size; j++) { 5067 if (memory_id == mem_list[j].memory_id) { 5068 dev_info(adev->dev, 5069 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5070 err_cnt, err_type_name, 5071 reg_list[i].block_name, 5072 instance, mem_list[j].name); 5073 break; 5074 } 5075 } 5076 } 5077 } 5078 } 5079 5080 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5081 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5082 uint32_t reg_list_size, 5083 uint32_t instance) 5084 { 5085 uint32_t err_status_lo_offset, err_status_hi_offset; 5086 uint32_t i; 5087 5088 for (i = 0; i < reg_list_size; i++) { 5089 err_status_lo_offset = 5090 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5091 reg_list[i].seg_lo, reg_list[i].reg_lo); 5092 err_status_hi_offset = 5093 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5094 reg_list[i].seg_hi, reg_list[i].reg_hi); 5095 WREG32(err_status_lo_offset, 0); 5096 WREG32(err_status_hi_offset, 0); 5097 } 5098 } 5099 5100 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5101 { 5102 memset(err_data, 0, sizeof(*err_data)); 5103 5104 INIT_LIST_HEAD(&err_data->err_node_list); 5105 5106 return 0; 5107 } 5108 5109 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5110 { 5111 if (!err_node) 5112 return; 5113 5114 list_del(&err_node->node); 5115 kvfree(err_node); 5116 } 5117 5118 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5119 { 5120 struct ras_err_node *err_node, *tmp; 5121 5122 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5123 amdgpu_ras_error_node_release(err_node); 5124 } 5125 5126 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5127 struct amdgpu_smuio_mcm_config_info *mcm_info) 5128 { 5129 struct ras_err_node *err_node; 5130 struct amdgpu_smuio_mcm_config_info *ref_id; 5131 5132 if (!err_data || !mcm_info) 5133 return NULL; 5134 5135 for_each_ras_error(err_node, err_data) { 5136 ref_id = &err_node->err_info.mcm_info; 5137 5138 if (mcm_info->socket_id == ref_id->socket_id && 5139 mcm_info->die_id == ref_id->die_id) 5140 return err_node; 5141 } 5142 5143 return NULL; 5144 } 5145 5146 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5147 { 5148 struct ras_err_node *err_node; 5149 5150 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5151 if (!err_node) 5152 return NULL; 5153 5154 INIT_LIST_HEAD(&err_node->node); 5155 5156 return err_node; 5157 } 5158 5159 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5160 { 5161 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5162 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5163 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5164 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5165 5166 if (unlikely(infoa->socket_id != infob->socket_id)) 5167 return infoa->socket_id - infob->socket_id; 5168 else 5169 return infoa->die_id - infob->die_id; 5170 5171 return 0; 5172 } 5173 5174 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5175 struct amdgpu_smuio_mcm_config_info *mcm_info) 5176 { 5177 struct ras_err_node *err_node; 5178 5179 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5180 if (err_node) 5181 return &err_node->err_info; 5182 5183 err_node = amdgpu_ras_error_node_new(); 5184 if (!err_node) 5185 return NULL; 5186 5187 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5188 5189 err_data->err_list_count++; 5190 list_add_tail(&err_node->node, &err_data->err_node_list); 5191 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5192 5193 return &err_node->err_info; 5194 } 5195 5196 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5197 struct amdgpu_smuio_mcm_config_info *mcm_info, 5198 u64 count) 5199 { 5200 struct ras_err_info *err_info; 5201 5202 if (!err_data || !mcm_info) 5203 return -EINVAL; 5204 5205 if (!count) 5206 return 0; 5207 5208 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5209 if (!err_info) 5210 return -EINVAL; 5211 5212 err_info->ue_count += count; 5213 err_data->ue_count += count; 5214 5215 return 0; 5216 } 5217 5218 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5219 struct amdgpu_smuio_mcm_config_info *mcm_info, 5220 u64 count) 5221 { 5222 struct ras_err_info *err_info; 5223 5224 if (!err_data || !mcm_info) 5225 return -EINVAL; 5226 5227 if (!count) 5228 return 0; 5229 5230 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5231 if (!err_info) 5232 return -EINVAL; 5233 5234 err_info->ce_count += count; 5235 err_data->ce_count += count; 5236 5237 return 0; 5238 } 5239 5240 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5241 struct amdgpu_smuio_mcm_config_info *mcm_info, 5242 u64 count) 5243 { 5244 struct ras_err_info *err_info; 5245 5246 if (!err_data || !mcm_info) 5247 return -EINVAL; 5248 5249 if (!count) 5250 return 0; 5251 5252 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5253 if (!err_info) 5254 return -EINVAL; 5255 5256 err_info->de_count += count; 5257 err_data->de_count += count; 5258 5259 return 0; 5260 } 5261 5262 #define mmMP0_SMN_C2PMSG_92 0x1609C 5263 #define mmMP0_SMN_C2PMSG_126 0x160BE 5264 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5265 u32 instance) 5266 { 5267 u32 socket_id, aid_id, hbm_id; 5268 u32 fw_status; 5269 u32 boot_error; 5270 u64 reg_addr; 5271 5272 /* The pattern for smn addressing in other SOC could be different from 5273 * the one for aqua_vanjaram. We should revisit the code if the pattern 5274 * is changed. In such case, replace the aqua_vanjaram implementation 5275 * with more common helper */ 5276 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5277 aqua_vanjaram_encode_ext_smn_addressing(instance); 5278 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5279 5280 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5281 aqua_vanjaram_encode_ext_smn_addressing(instance); 5282 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5283 5284 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5285 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5286 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5287 5288 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5289 dev_info(adev->dev, 5290 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5291 socket_id, aid_id, hbm_id, fw_status); 5292 5293 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5294 dev_info(adev->dev, 5295 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5296 socket_id, aid_id, fw_status); 5297 5298 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5299 dev_info(adev->dev, 5300 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5301 socket_id, aid_id, fw_status); 5302 5303 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5304 dev_info(adev->dev, 5305 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5306 socket_id, aid_id, fw_status); 5307 5308 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5309 dev_info(adev->dev, 5310 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5311 socket_id, aid_id, fw_status); 5312 5313 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5314 dev_info(adev->dev, 5315 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5316 socket_id, aid_id, fw_status); 5317 5318 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5319 dev_info(adev->dev, 5320 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5321 socket_id, aid_id, hbm_id, fw_status); 5322 5323 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5324 dev_info(adev->dev, 5325 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5326 socket_id, aid_id, hbm_id, fw_status); 5327 5328 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5329 dev_info(adev->dev, 5330 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5331 socket_id, aid_id, fw_status); 5332 5333 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5334 dev_info(adev->dev, 5335 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5336 socket_id, aid_id, fw_status); 5337 } 5338 5339 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5340 u32 instance) 5341 { 5342 u64 reg_addr; 5343 u32 reg_data; 5344 int retry_loop; 5345 5346 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5347 aqua_vanjaram_encode_ext_smn_addressing(instance); 5348 5349 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5350 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5351 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5352 return false; 5353 else 5354 msleep(1); 5355 } 5356 5357 return true; 5358 } 5359 5360 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5361 { 5362 u32 i; 5363 5364 for (i = 0; i < num_instances; i++) { 5365 if (amdgpu_ras_boot_error_detected(adev, i)) 5366 amdgpu_ras_boot_time_error_reporting(adev, i); 5367 } 5368 } 5369 5370 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5371 { 5372 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5373 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5374 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5375 int ret = 0; 5376 5377 if (amdgpu_ras_check_critical_address(adev, start)) 5378 return 0; 5379 5380 mutex_lock(&con->page_rsv_lock); 5381 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5382 if (ret == -ENOENT) 5383 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5384 mutex_unlock(&con->page_rsv_lock); 5385 5386 return ret; 5387 } 5388 5389 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5390 const char *fmt, ...) 5391 { 5392 struct va_format vaf; 5393 va_list args; 5394 5395 va_start(args, fmt); 5396 vaf.fmt = fmt; 5397 vaf.va = &args; 5398 5399 if (RAS_EVENT_ID_IS_VALID(event_id)) 5400 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5401 else 5402 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5403 5404 va_end(args); 5405 } 5406 5407 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5408 { 5409 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5410 5411 if (!con) 5412 return false; 5413 5414 return con->is_rma; 5415 } 5416 5417 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5418 struct amdgpu_bo *bo) 5419 { 5420 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5421 struct amdgpu_vram_mgr_resource *vres; 5422 struct ras_critical_region *region; 5423 struct drm_buddy_block *block; 5424 int ret = 0; 5425 5426 if (!bo || !bo->tbo.resource) 5427 return -EINVAL; 5428 5429 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5430 5431 mutex_lock(&con->critical_region_lock); 5432 5433 /* Check if the bo had been recorded */ 5434 list_for_each_entry(region, &con->critical_region_head, node) 5435 if (region->bo == bo) 5436 goto out; 5437 5438 /* Record new critical amdgpu bo */ 5439 list_for_each_entry(block, &vres->blocks, link) { 5440 region = kzalloc(sizeof(*region), GFP_KERNEL); 5441 if (!region) { 5442 ret = -ENOMEM; 5443 goto out; 5444 } 5445 region->bo = bo; 5446 region->start = amdgpu_vram_mgr_block_start(block); 5447 region->size = amdgpu_vram_mgr_block_size(block); 5448 list_add_tail(®ion->node, &con->critical_region_head); 5449 } 5450 5451 out: 5452 mutex_unlock(&con->critical_region_lock); 5453 5454 return ret; 5455 } 5456 5457 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5458 { 5459 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5460 } 5461 5462 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5463 { 5464 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5465 struct ras_critical_region *region, *tmp; 5466 5467 mutex_lock(&con->critical_region_lock); 5468 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5469 list_del(®ion->node); 5470 kfree(region); 5471 } 5472 mutex_unlock(&con->critical_region_lock); 5473 } 5474 5475 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5476 { 5477 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5478 struct ras_critical_region *region; 5479 bool ret = false; 5480 5481 mutex_lock(&con->critical_region_lock); 5482 list_for_each_entry(region, &con->critical_region_head, node) { 5483 if ((region->start <= addr) && 5484 (addr < (region->start + region->size))) { 5485 ret = true; 5486 break; 5487 } 5488 } 5489 mutex_unlock(&con->critical_region_lock); 5490 5491 return ret; 5492 } 5493