1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 616 617 /** 618 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 619 * 620 * Some boards contain an EEPROM which is used to persistently store a list of 621 * bad pages which experiences ECC errors in vram. This interface provides 622 * a way to reset the EEPROM, e.g., after testing error injection. 623 * 624 * Usage: 625 * 626 * .. code-block:: bash 627 * 628 * echo 1 > ../ras/ras_eeprom_reset 629 * 630 * will reset EEPROM table to 0 entries. 631 * 632 */ 633 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 634 const char __user *buf, 635 size_t size, loff_t *pos) 636 { 637 struct amdgpu_device *adev = 638 (struct amdgpu_device *)file_inode(f)->i_private; 639 int ret; 640 641 if (amdgpu_uniras_enabled(adev)) { 642 ret = amdgpu_uniras_clear_badpages_info(adev); 643 return ret ? ret : size; 644 } 645 646 ret = amdgpu_ras_eeprom_reset_table( 647 &(amdgpu_ras_get_context(adev)->eeprom_control)); 648 649 if (!ret) { 650 /* Something was written to EEPROM. 651 */ 652 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 653 return size; 654 } else { 655 return ret; 656 } 657 } 658 659 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_ctrl_write, 663 .llseek = default_llseek 664 }; 665 666 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 667 .owner = THIS_MODULE, 668 .read = NULL, 669 .write = amdgpu_ras_debugfs_eeprom_write, 670 .llseek = default_llseek 671 }; 672 673 /** 674 * DOC: AMDGPU RAS sysfs Error Count Interface 675 * 676 * It allows the user to read the error count for each IP block on the gpu through 677 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 678 * 679 * It outputs the multiple lines which report the uncorrected (ue) and corrected 680 * (ce) error counts. 681 * 682 * The format of one line is below, 683 * 684 * [ce|ue]: count 685 * 686 * Example: 687 * 688 * .. code-block:: bash 689 * 690 * ue: 0 691 * ce: 1 692 * 693 */ 694 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 695 struct device_attribute *attr, char *buf) 696 { 697 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 698 struct ras_query_if info = { 699 .head = obj->head, 700 }; 701 702 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 703 return sysfs_emit(buf, "Query currently inaccessible\n"); 704 705 if (amdgpu_ras_query_error_status(obj->adev, &info)) 706 return -EINVAL; 707 708 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 709 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 710 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 711 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 712 } 713 714 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 715 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 716 "ce", info.ce_count, "de", info.de_count); 717 else 718 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 719 "ce", info.ce_count); 720 } 721 722 /* obj begin */ 723 724 #define get_obj(obj) do { (obj)->use++; } while (0) 725 #define alive_obj(obj) ((obj)->use) 726 727 static inline void put_obj(struct ras_manager *obj) 728 { 729 if (obj && (--obj->use == 0)) { 730 list_del(&obj->node); 731 amdgpu_ras_error_data_fini(&obj->err_data); 732 } 733 734 if (obj && (obj->use < 0)) 735 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 736 } 737 738 /* make one obj and return it. */ 739 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 740 struct ras_common_if *head) 741 { 742 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 743 struct ras_manager *obj; 744 745 if (!adev->ras_enabled || !con) 746 return NULL; 747 748 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 749 return NULL; 750 751 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 752 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 753 return NULL; 754 755 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 756 } else 757 obj = &con->objs[head->block]; 758 759 /* already exist. return obj? */ 760 if (alive_obj(obj)) 761 return NULL; 762 763 if (amdgpu_ras_error_data_init(&obj->err_data)) 764 return NULL; 765 766 obj->head = *head; 767 obj->adev = adev; 768 list_add(&obj->node, &con->head); 769 get_obj(obj); 770 771 return obj; 772 } 773 774 /* return an obj equal to head, or the first when head is NULL */ 775 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 776 struct ras_common_if *head) 777 { 778 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 779 struct ras_manager *obj; 780 int i; 781 782 if (!adev->ras_enabled || !con) 783 return NULL; 784 785 if (head) { 786 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 787 return NULL; 788 789 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 790 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 791 return NULL; 792 793 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 794 } else 795 obj = &con->objs[head->block]; 796 797 if (alive_obj(obj)) 798 return obj; 799 } else { 800 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 801 obj = &con->objs[i]; 802 if (alive_obj(obj)) 803 return obj; 804 } 805 } 806 807 return NULL; 808 } 809 /* obj end */ 810 811 /* feature ctl begin */ 812 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 813 struct ras_common_if *head) 814 { 815 return adev->ras_hw_enabled & BIT(head->block); 816 } 817 818 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 819 struct ras_common_if *head) 820 { 821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 822 823 return con->features & BIT(head->block); 824 } 825 826 /* 827 * if obj is not created, then create one. 828 * set feature enable flag. 829 */ 830 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 831 struct ras_common_if *head, int enable) 832 { 833 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 834 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 835 836 /* If hardware does not support ras, then do not create obj. 837 * But if hardware support ras, we can create the obj. 838 * Ras framework checks con->hw_supported to see if it need do 839 * corresponding initialization. 840 * IP checks con->support to see if it need disable ras. 841 */ 842 if (!amdgpu_ras_is_feature_allowed(adev, head)) 843 return 0; 844 845 if (enable) { 846 if (!obj) { 847 obj = amdgpu_ras_create_obj(adev, head); 848 if (!obj) 849 return -EINVAL; 850 } else { 851 /* In case we create obj somewhere else */ 852 get_obj(obj); 853 } 854 con->features |= BIT(head->block); 855 } else { 856 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 857 con->features &= ~BIT(head->block); 858 put_obj(obj); 859 } 860 } 861 862 return 0; 863 } 864 865 /* wrapper of psp_ras_enable_features */ 866 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 867 struct ras_common_if *head, bool enable) 868 { 869 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 870 union ta_ras_cmd_input *info; 871 int ret; 872 873 if (!con) 874 return -EINVAL; 875 876 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 877 /* For gfx ip, regardless of feature support status, */ 878 /* Force issue enable or disable ras feature commands */ 879 if (head->block != AMDGPU_RAS_BLOCK__GFX && 880 !amdgpu_ras_is_feature_allowed(adev, head)) 881 return 0; 882 883 /* Only enable gfx ras feature from host side */ 884 if (head->block == AMDGPU_RAS_BLOCK__GFX && 885 !amdgpu_sriov_vf(adev) && 886 !amdgpu_ras_intr_triggered()) { 887 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 888 if (!info) 889 return -ENOMEM; 890 891 if (!enable) { 892 info->disable_features = (struct ta_ras_disable_features_input) { 893 .block_id = amdgpu_ras_block_to_ta(head->block), 894 .error_type = amdgpu_ras_error_to_ta(head->type), 895 }; 896 } else { 897 info->enable_features = (struct ta_ras_enable_features_input) { 898 .block_id = amdgpu_ras_block_to_ta(head->block), 899 .error_type = amdgpu_ras_error_to_ta(head->type), 900 }; 901 } 902 903 ret = psp_ras_enable_features(&adev->psp, info, enable); 904 if (ret) { 905 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 906 enable ? "enable":"disable", 907 get_ras_block_str(head), 908 amdgpu_ras_is_poison_mode_supported(adev), ret); 909 kfree(info); 910 return ret; 911 } 912 913 kfree(info); 914 } 915 916 /* setup the obj */ 917 __amdgpu_ras_feature_enable(adev, head, enable); 918 919 return 0; 920 } 921 922 /* Only used in device probe stage and called only once. */ 923 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 924 struct ras_common_if *head, bool enable) 925 { 926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 927 int ret; 928 929 if (!con) 930 return -EINVAL; 931 932 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 933 if (enable) { 934 /* There is no harm to issue a ras TA cmd regardless of 935 * the currecnt ras state. 936 * If current state == target state, it will do nothing 937 * But sometimes it requests driver to reset and repost 938 * with error code -EAGAIN. 939 */ 940 ret = amdgpu_ras_feature_enable(adev, head, 1); 941 /* With old ras TA, we might fail to enable ras. 942 * Log it and just setup the object. 943 * TODO need remove this WA in the future. 944 */ 945 if (ret == -EINVAL) { 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (!ret) 948 dev_info(adev->dev, 949 "RAS INFO: %s setup object\n", 950 get_ras_block_str(head)); 951 } 952 } else { 953 /* setup the object then issue a ras TA disable cmd.*/ 954 ret = __amdgpu_ras_feature_enable(adev, head, 1); 955 if (ret) 956 return ret; 957 958 /* gfx block ras disable cmd must send to ras-ta */ 959 if (head->block == AMDGPU_RAS_BLOCK__GFX) 960 con->features |= BIT(head->block); 961 962 ret = amdgpu_ras_feature_enable(adev, head, 0); 963 964 /* clean gfx block ras features flag */ 965 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 966 con->features &= ~BIT(head->block); 967 } 968 } else 969 ret = amdgpu_ras_feature_enable(adev, head, enable); 970 971 return ret; 972 } 973 974 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 975 bool bypass) 976 { 977 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 978 struct ras_manager *obj, *tmp; 979 980 list_for_each_entry_safe(obj, tmp, &con->head, node) { 981 /* bypass psp. 982 * aka just release the obj and corresponding flags 983 */ 984 if (bypass) { 985 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 986 break; 987 } else { 988 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 989 break; 990 } 991 } 992 993 return con->features; 994 } 995 996 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 997 bool bypass) 998 { 999 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1000 int i; 1001 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1002 1003 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1004 struct ras_common_if head = { 1005 .block = i, 1006 .type = default_ras_type, 1007 .sub_block_index = 0, 1008 }; 1009 1010 if (i == AMDGPU_RAS_BLOCK__MCA) 1011 continue; 1012 1013 if (bypass) { 1014 /* 1015 * bypass psp. vbios enable ras for us. 1016 * so just create the obj 1017 */ 1018 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1019 break; 1020 } else { 1021 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1022 break; 1023 } 1024 } 1025 1026 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1027 struct ras_common_if head = { 1028 .block = AMDGPU_RAS_BLOCK__MCA, 1029 .type = default_ras_type, 1030 .sub_block_index = i, 1031 }; 1032 1033 if (bypass) { 1034 /* 1035 * bypass psp. vbios enable ras for us. 1036 * so just create the obj 1037 */ 1038 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1039 break; 1040 } else { 1041 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1042 break; 1043 } 1044 } 1045 1046 return con->features; 1047 } 1048 /* feature ctl end */ 1049 1050 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1051 enum amdgpu_ras_block block) 1052 { 1053 if (!block_obj) 1054 return -EINVAL; 1055 1056 if (block_obj->ras_comm.block == block) 1057 return 0; 1058 1059 return -EINVAL; 1060 } 1061 1062 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1063 enum amdgpu_ras_block block, uint32_t sub_block_index) 1064 { 1065 struct amdgpu_ras_block_list *node, *tmp; 1066 struct amdgpu_ras_block_object *obj; 1067 1068 if (block >= AMDGPU_RAS_BLOCK__LAST) 1069 return NULL; 1070 1071 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1072 if (!node->ras_obj) { 1073 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1074 continue; 1075 } 1076 1077 obj = node->ras_obj; 1078 if (obj->ras_block_match) { 1079 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1080 return obj; 1081 } else { 1082 if (amdgpu_ras_block_match_default(obj, block) == 0) 1083 return obj; 1084 } 1085 } 1086 1087 return NULL; 1088 } 1089 1090 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1091 { 1092 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1093 int ret = 0; 1094 1095 /* 1096 * choosing right query method according to 1097 * whether smu support query error information 1098 */ 1099 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1100 if (ret == -EOPNOTSUPP) { 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1104 1105 /* umc query_ras_error_address is also responsible for clearing 1106 * error status 1107 */ 1108 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1109 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1110 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1111 } else if (!ret) { 1112 if (adev->umc.ras && 1113 adev->umc.ras->ecc_info_query_ras_error_count) 1114 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1115 1116 if (adev->umc.ras && 1117 adev->umc.ras->ecc_info_query_ras_error_address) 1118 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1119 } 1120 } 1121 1122 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1123 struct ras_manager *ras_mgr, 1124 struct ras_err_data *err_data, 1125 struct ras_query_context *qctx, 1126 const char *blk_name, 1127 bool is_ue, 1128 bool is_de) 1129 { 1130 struct amdgpu_smuio_mcm_config_info *mcm_info; 1131 struct ras_err_node *err_node; 1132 struct ras_err_info *err_info; 1133 u64 event_id = qctx->evid.event_id; 1134 1135 if (is_ue) { 1136 for_each_ras_error(err_node, err_data) { 1137 err_info = &err_node->err_info; 1138 mcm_info = &err_info->mcm_info; 1139 if (err_info->ue_count) { 1140 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1141 "%lld new uncorrectable hardware errors detected in %s block\n", 1142 mcm_info->socket_id, 1143 mcm_info->die_id, 1144 err_info->ue_count, 1145 blk_name); 1146 } 1147 } 1148 1149 for_each_ras_error(err_node, &ras_mgr->err_data) { 1150 err_info = &err_node->err_info; 1151 mcm_info = &err_info->mcm_info; 1152 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1153 "%lld uncorrectable hardware errors detected in total in %s block\n", 1154 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1155 } 1156 1157 } else { 1158 if (is_de) { 1159 for_each_ras_error(err_node, err_data) { 1160 err_info = &err_node->err_info; 1161 mcm_info = &err_info->mcm_info; 1162 if (err_info->de_count) { 1163 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1164 "%lld new deferred hardware errors detected in %s block\n", 1165 mcm_info->socket_id, 1166 mcm_info->die_id, 1167 err_info->de_count, 1168 blk_name); 1169 } 1170 } 1171 1172 for_each_ras_error(err_node, &ras_mgr->err_data) { 1173 err_info = &err_node->err_info; 1174 mcm_info = &err_info->mcm_info; 1175 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1176 "%lld deferred hardware errors detected in total in %s block\n", 1177 mcm_info->socket_id, mcm_info->die_id, 1178 err_info->de_count, blk_name); 1179 } 1180 } else { 1181 if (adev->debug_disable_ce_logs) 1182 return; 1183 1184 for_each_ras_error(err_node, err_data) { 1185 err_info = &err_node->err_info; 1186 mcm_info = &err_info->mcm_info; 1187 if (err_info->ce_count) { 1188 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1189 "%lld new correctable hardware errors detected in %s block\n", 1190 mcm_info->socket_id, 1191 mcm_info->die_id, 1192 err_info->ce_count, 1193 blk_name); 1194 } 1195 } 1196 1197 for_each_ras_error(err_node, &ras_mgr->err_data) { 1198 err_info = &err_node->err_info; 1199 mcm_info = &err_info->mcm_info; 1200 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1201 "%lld correctable hardware errors detected in total in %s block\n", 1202 mcm_info->socket_id, mcm_info->die_id, 1203 err_info->ce_count, blk_name); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline bool err_data_has_source_info(struct ras_err_data *data) 1210 { 1211 return !list_empty(&data->err_node_list); 1212 } 1213 1214 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1215 struct ras_query_if *query_if, 1216 struct ras_err_data *err_data, 1217 struct ras_query_context *qctx) 1218 { 1219 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1220 const char *blk_name = get_ras_block_str(&query_if->head); 1221 u64 event_id = qctx->evid.event_id; 1222 1223 if (err_data->ce_count) { 1224 if (err_data_has_source_info(err_data)) { 1225 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1226 blk_name, false, false); 1227 } else if (!adev->aid_mask && 1228 adev->smuio.funcs && 1229 adev->smuio.funcs->get_socket_id && 1230 adev->smuio.funcs->get_die_id) { 1231 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1232 "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 adev->smuio.funcs->get_socket_id(adev), 1235 adev->smuio.funcs->get_die_id(adev), 1236 ras_mgr->err_data.ce_count, 1237 blk_name); 1238 } else { 1239 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1240 "detected in %s block\n", 1241 ras_mgr->err_data.ce_count, 1242 blk_name); 1243 } 1244 } 1245 1246 if (err_data->ue_count) { 1247 if (err_data_has_source_info(err_data)) { 1248 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1249 blk_name, true, false); 1250 } else if (!adev->aid_mask && 1251 adev->smuio.funcs && 1252 adev->smuio.funcs->get_socket_id && 1253 adev->smuio.funcs->get_die_id) { 1254 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1255 "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 adev->smuio.funcs->get_socket_id(adev), 1258 adev->smuio.funcs->get_die_id(adev), 1259 ras_mgr->err_data.ue_count, 1260 blk_name); 1261 } else { 1262 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1263 "detected in %s block\n", 1264 ras_mgr->err_data.ue_count, 1265 blk_name); 1266 } 1267 } 1268 1269 if (err_data->de_count) { 1270 if (err_data_has_source_info(err_data)) { 1271 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1272 blk_name, false, true); 1273 } else if (!adev->aid_mask && 1274 adev->smuio.funcs && 1275 adev->smuio.funcs->get_socket_id && 1276 adev->smuio.funcs->get_die_id) { 1277 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1278 "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 adev->smuio.funcs->get_socket_id(adev), 1281 adev->smuio.funcs->get_die_id(adev), 1282 ras_mgr->err_data.de_count, 1283 blk_name); 1284 } else { 1285 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1286 "detected in %s block\n", 1287 ras_mgr->err_data.de_count, 1288 blk_name); 1289 } 1290 } 1291 } 1292 1293 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1294 struct ras_query_if *query_if, 1295 struct ras_err_data *err_data, 1296 struct ras_query_context *qctx) 1297 { 1298 unsigned long new_ue, new_ce, new_de; 1299 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1300 const char *blk_name = get_ras_block_str(&query_if->head); 1301 u64 event_id = qctx->evid.event_id; 1302 1303 new_ce = err_data->ce_count - obj->err_data.ce_count; 1304 new_ue = err_data->ue_count - obj->err_data.ue_count; 1305 new_de = err_data->de_count - obj->err_data.de_count; 1306 1307 if (new_ce) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1309 "detected in %s block\n", 1310 new_ce, 1311 blk_name); 1312 } 1313 1314 if (new_ue) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1316 "detected in %s block\n", 1317 new_ue, 1318 blk_name); 1319 } 1320 1321 if (new_de) { 1322 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1323 "detected in %s block\n", 1324 new_de, 1325 blk_name); 1326 } 1327 } 1328 1329 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1330 { 1331 struct ras_err_node *err_node; 1332 struct ras_err_info *err_info; 1333 1334 if (err_data_has_source_info(err_data)) { 1335 for_each_ras_error(err_node, err_data) { 1336 err_info = &err_node->err_info; 1337 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1338 &err_info->mcm_info, err_info->de_count); 1339 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->ce_count); 1341 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ue_count); 1343 } 1344 } else { 1345 /* for legacy asic path which doesn't has error source info */ 1346 obj->err_data.ue_count += err_data->ue_count; 1347 obj->err_data.ce_count += err_data->ce_count; 1348 obj->err_data.de_count += err_data->de_count; 1349 } 1350 } 1351 1352 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1353 struct ras_err_data *err_data) 1354 { 1355 /* Host reports absolute counts */ 1356 obj->err_data.ue_count = err_data->ue_count; 1357 obj->err_data.ce_count = err_data->ce_count; 1358 obj->err_data.de_count = err_data->de_count; 1359 } 1360 1361 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1362 { 1363 struct ras_common_if head; 1364 1365 memset(&head, 0, sizeof(head)); 1366 head.block = blk; 1367 1368 return amdgpu_ras_find_obj(adev, &head); 1369 } 1370 1371 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1372 const struct aca_info *aca_info, void *data) 1373 { 1374 struct ras_manager *obj; 1375 1376 /* in resume phase, no need to create aca fs node */ 1377 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1378 return 0; 1379 1380 obj = get_ras_manager(adev, blk); 1381 if (!obj) 1382 return -EINVAL; 1383 1384 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1385 } 1386 1387 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1388 { 1389 struct ras_manager *obj; 1390 1391 obj = get_ras_manager(adev, blk); 1392 if (!obj) 1393 return -EINVAL; 1394 1395 amdgpu_aca_remove_handle(&obj->aca_handle); 1396 1397 return 0; 1398 } 1399 1400 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1401 enum aca_error_type type, struct ras_err_data *err_data, 1402 struct ras_query_context *qctx) 1403 { 1404 struct ras_manager *obj; 1405 1406 obj = get_ras_manager(adev, blk); 1407 if (!obj) 1408 return -EINVAL; 1409 1410 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1411 } 1412 1413 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1414 struct aca_handle *handle, char *buf, void *data) 1415 { 1416 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1417 struct ras_query_if info = { 1418 .head = obj->head, 1419 }; 1420 1421 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1422 return sysfs_emit(buf, "Query currently inaccessible\n"); 1423 1424 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1425 return -EINVAL; 1426 1427 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1428 "ce", info.ce_count, "de", info.de_count); 1429 } 1430 1431 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1432 struct ras_query_if *info, 1433 struct ras_err_data *err_data, 1434 struct ras_query_context *qctx, 1435 unsigned int error_query_mode) 1436 { 1437 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1438 struct amdgpu_ras_block_object *block_obj = NULL; 1439 int ret; 1440 1441 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1442 return -EINVAL; 1443 1444 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1445 return -EINVAL; 1446 1447 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1448 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1449 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1450 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1451 amdgpu_ras_get_ecc_info(adev, err_data); 1452 } else { 1453 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1454 if (!block_obj || !block_obj->hw_ops) { 1455 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1456 get_ras_block_str(&info->head)); 1457 return -EINVAL; 1458 } 1459 1460 if (block_obj->hw_ops->query_ras_error_count) 1461 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1462 1463 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1464 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1465 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1466 if (block_obj->hw_ops->query_ras_error_status) 1467 block_obj->hw_ops->query_ras_error_status(adev); 1468 } 1469 } 1470 } else { 1471 if (amdgpu_aca_is_enabled(adev)) { 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 1476 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1477 if (ret) 1478 return ret; 1479 1480 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1481 if (ret) 1482 return ret; 1483 } else { 1484 /* FIXME: add code to check return value later */ 1485 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1486 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1487 } 1488 } 1489 1490 return 0; 1491 } 1492 1493 /* query/inject/cure begin */ 1494 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1495 struct ras_query_if *info, 1496 enum ras_event_type type) 1497 { 1498 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1499 struct ras_err_data err_data; 1500 struct ras_query_context qctx; 1501 unsigned int error_query_mode; 1502 int ret; 1503 1504 if (!obj) 1505 return -EINVAL; 1506 1507 ret = amdgpu_ras_error_data_init(&err_data); 1508 if (ret) 1509 return ret; 1510 1511 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1512 return -EINVAL; 1513 1514 memset(&qctx, 0, sizeof(qctx)); 1515 qctx.evid.type = type; 1516 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1517 1518 if (!down_read_trylock(&adev->reset_domain->sem)) { 1519 ret = -EIO; 1520 goto out_fini_err_data; 1521 } 1522 1523 ret = amdgpu_ras_query_error_status_helper(adev, info, 1524 &err_data, 1525 &qctx, 1526 error_query_mode); 1527 up_read(&adev->reset_domain->sem); 1528 if (ret) 1529 goto out_fini_err_data; 1530 1531 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1532 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1533 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1534 } else { 1535 /* Host provides absolute error counts. First generate the report 1536 * using the previous VF internal count against new host count. 1537 * Then Update VF internal count. 1538 */ 1539 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1540 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1541 } 1542 1543 info->ue_count = obj->err_data.ue_count; 1544 info->ce_count = obj->err_data.ce_count; 1545 info->de_count = obj->err_data.de_count; 1546 1547 out_fini_err_data: 1548 amdgpu_ras_error_data_fini(&err_data); 1549 1550 return ret; 1551 } 1552 1553 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1554 { 1555 struct ras_cmd_dev_handle req = {0}; 1556 int ret; 1557 1558 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1559 &req, sizeof(req), NULL, 0); 1560 if (ret) { 1561 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1562 return ret; 1563 } 1564 1565 return 0; 1566 } 1567 1568 static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1569 struct ras_query_if *info) 1570 { 1571 struct ras_cmd_block_ecc_info_req req = {0}; 1572 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1573 int ret; 1574 1575 if (!info) 1576 return -EINVAL; 1577 1578 req.block_id = info->head.block; 1579 req.subblock_id = info->head.sub_block_index; 1580 1581 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1582 &req, sizeof(req), &rsp, sizeof(rsp)); 1583 if (!ret) { 1584 info->ce_count = rsp.ce_count; 1585 info->ue_count = rsp.ue_count; 1586 info->de_count = rsp.de_count; 1587 } 1588 1589 return ret; 1590 } 1591 1592 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1593 { 1594 if (amdgpu_uniras_enabled(adev)) 1595 return amdgpu_uniras_query_block_ecc(adev, info); 1596 else 1597 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1598 } 1599 1600 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1601 enum amdgpu_ras_block block) 1602 { 1603 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1604 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1605 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1606 1607 if (!block_obj || !block_obj->hw_ops) { 1608 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1609 ras_block_str(block)); 1610 return -EOPNOTSUPP; 1611 } 1612 1613 if (!amdgpu_ras_is_supported(adev, block) || 1614 !amdgpu_ras_get_aca_debug_mode(adev)) 1615 return -EOPNOTSUPP; 1616 1617 if (amdgpu_sriov_vf(adev)) 1618 return -EOPNOTSUPP; 1619 1620 /* skip ras error reset in gpu reset */ 1621 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1622 ((smu_funcs && smu_funcs->set_debug_mode) || 1623 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1624 return -EOPNOTSUPP; 1625 1626 if (block_obj->hw_ops->reset_ras_error_count) 1627 block_obj->hw_ops->reset_ras_error_count(adev); 1628 1629 return 0; 1630 } 1631 1632 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1633 enum amdgpu_ras_block block) 1634 { 1635 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1636 1637 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1638 return 0; 1639 1640 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1641 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1642 if (block_obj->hw_ops->reset_ras_error_status) 1643 block_obj->hw_ops->reset_ras_error_status(adev); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1650 struct ras_inject_if *info) 1651 { 1652 struct ras_cmd_inject_error_req inject_req; 1653 struct ras_cmd_inject_error_rsp rsp; 1654 1655 if (!info) 1656 return -EINVAL; 1657 1658 memset(&inject_req, 0, sizeof(inject_req)); 1659 inject_req.block_id = info->head.block; 1660 inject_req.subblock_id = info->head.sub_block_index; 1661 inject_req.address = info->address; 1662 inject_req.error_type = info->head.type; 1663 inject_req.instance_mask = info->instance_mask; 1664 inject_req.method = info->value; 1665 1666 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1667 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1668 } 1669 1670 /* wrapper of psp_ras_trigger_error */ 1671 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1672 struct ras_inject_if *info) 1673 { 1674 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1675 struct ta_ras_trigger_error_input block_info = { 1676 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1677 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1678 .sub_block_index = info->head.sub_block_index, 1679 .address = info->address, 1680 .value = info->value, 1681 }; 1682 int ret = -EINVAL; 1683 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1684 info->head.block, 1685 info->head.sub_block_index); 1686 1687 if (amdgpu_uniras_enabled(adev)) 1688 return amdgpu_uniras_error_inject(adev, info); 1689 1690 /* inject on guest isn't allowed, return success directly */ 1691 if (amdgpu_sriov_vf(adev)) 1692 return 0; 1693 1694 if (!obj) 1695 return -EINVAL; 1696 1697 if (!block_obj || !block_obj->hw_ops) { 1698 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1699 get_ras_block_str(&info->head)); 1700 return -EINVAL; 1701 } 1702 1703 /* Calculate XGMI relative offset */ 1704 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1705 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1706 block_info.address = 1707 amdgpu_xgmi_get_relative_phy_addr(adev, 1708 block_info.address); 1709 } 1710 1711 if (block_obj->hw_ops->ras_error_inject) { 1712 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1713 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1714 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1715 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1716 info->instance_mask); 1717 } else { 1718 /* default path */ 1719 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1720 } 1721 1722 if (ret) 1723 dev_err(adev->dev, "ras inject %s failed %d\n", 1724 get_ras_block_str(&info->head), ret); 1725 1726 return ret; 1727 } 1728 1729 /** 1730 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1731 * @adev: pointer to AMD GPU device 1732 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1733 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1734 * @query_info: pointer to ras_query_if 1735 * 1736 * Return 0 for query success or do nothing, otherwise return an error 1737 * on failures 1738 */ 1739 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1740 unsigned long *ce_count, 1741 unsigned long *ue_count, 1742 struct ras_query_if *query_info) 1743 { 1744 int ret; 1745 1746 if (!query_info) 1747 /* do nothing if query_info is not specified */ 1748 return 0; 1749 1750 ret = amdgpu_ras_query_error_status(adev, query_info); 1751 if (ret) 1752 return ret; 1753 1754 *ce_count += query_info->ce_count; 1755 *ue_count += query_info->ue_count; 1756 1757 /* some hardware/IP supports read to clear 1758 * no need to explictly reset the err status after the query call */ 1759 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1760 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1761 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1762 dev_warn(adev->dev, 1763 "Failed to reset error counter and error status\n"); 1764 } 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1771 * @adev: pointer to AMD GPU device 1772 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1773 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1774 * errors. 1775 * @query_info: pointer to ras_query_if if the query request is only for 1776 * specific ip block; if info is NULL, then the qurey request is for 1777 * all the ip blocks that support query ras error counters/status 1778 * 1779 * If set, @ce_count or @ue_count, count and return the corresponding 1780 * error counts in those integer pointers. Return 0 if the device 1781 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1782 */ 1783 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1784 unsigned long *ce_count, 1785 unsigned long *ue_count, 1786 struct ras_query_if *query_info) 1787 { 1788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1789 struct ras_manager *obj; 1790 unsigned long ce, ue; 1791 int ret; 1792 1793 if (!adev->ras_enabled || !con) 1794 return -EOPNOTSUPP; 1795 1796 /* Don't count since no reporting. 1797 */ 1798 if (!ce_count && !ue_count) 1799 return 0; 1800 1801 ce = 0; 1802 ue = 0; 1803 if (!query_info) { 1804 /* query all the ip blocks that support ras query interface */ 1805 list_for_each_entry(obj, &con->head, node) { 1806 struct ras_query_if info = { 1807 .head = obj->head, 1808 }; 1809 1810 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1811 } 1812 } else { 1813 /* query specific ip block */ 1814 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1815 } 1816 1817 if (ret) 1818 return ret; 1819 1820 if (ce_count) 1821 *ce_count = ce; 1822 1823 if (ue_count) 1824 *ue_count = ue; 1825 1826 return 0; 1827 } 1828 /* query/inject/cure end */ 1829 1830 1831 /* sysfs begin */ 1832 1833 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1834 struct ras_badpage *bps, uint32_t count, uint32_t start); 1835 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837 1838 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1839 { 1840 switch (flags) { 1841 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1842 return "R"; 1843 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1844 return "P"; 1845 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1846 default: 1847 return "F"; 1848 } 1849 } 1850 1851 /** 1852 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1853 * 1854 * It allows user to read the bad pages of vram on the gpu through 1855 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1856 * 1857 * It outputs multiple lines, and each line stands for one gpu page. 1858 * 1859 * The format of one line is below, 1860 * gpu pfn : gpu page size : flags 1861 * 1862 * gpu pfn and gpu page size are printed in hex format. 1863 * flags can be one of below character, 1864 * 1865 * R: reserved, this gpu page is reserved and not able to use. 1866 * 1867 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1868 * in next window of page_reserve. 1869 * 1870 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1871 * 1872 * Examples: 1873 * 1874 * .. code-block:: bash 1875 * 1876 * 0x00000001 : 0x00001000 : R 1877 * 0x00000002 : 0x00001000 : P 1878 * 1879 */ 1880 1881 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1882 struct kobject *kobj, const struct bin_attribute *attr, 1883 char *buf, loff_t ppos, size_t count) 1884 { 1885 struct amdgpu_ras *con = 1886 container_of(attr, struct amdgpu_ras, badpages_attr); 1887 struct amdgpu_device *adev = con->adev; 1888 const unsigned int element_size = 1889 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1890 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1891 unsigned int end = div64_ul(ppos + count - 1, element_size); 1892 ssize_t s = 0; 1893 struct ras_badpage *bps = NULL; 1894 int bps_count = 0, i, status; 1895 uint64_t address; 1896 1897 memset(buf, 0, count); 1898 1899 bps_count = end - start; 1900 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1901 if (!bps) 1902 return 0; 1903 1904 memset(bps, 0, sizeof(*bps) * bps_count); 1905 1906 if (amdgpu_uniras_enabled(adev)) 1907 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1908 else 1909 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1910 1911 if (bps_count <= 0) { 1912 kfree(bps); 1913 return 0; 1914 } 1915 1916 for (i = 0; i < bps_count; i++) { 1917 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1918 if (amdgpu_ras_check_critical_address(adev, address)) 1919 continue; 1920 1921 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1922 1923 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1924 address); 1925 if (status == -EBUSY) 1926 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1927 else if (status == -ENOENT) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1929 else 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1931 1932 s += scnprintf(&buf[s], element_size + 1, 1933 "0x%08x : 0x%08x : %1s\n", 1934 bps[i].bp, 1935 bps[i].size, 1936 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1937 } 1938 1939 kfree(bps); 1940 1941 return s; 1942 } 1943 1944 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1945 struct device_attribute *attr, char *buf) 1946 { 1947 struct amdgpu_ras *con = 1948 container_of(attr, struct amdgpu_ras, features_attr); 1949 1950 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1951 } 1952 1953 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1954 u32 *minor, u32 *rev) 1955 { 1956 int i; 1957 1958 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1959 return false; 1960 1961 for (i = 0; i < adev->num_ip_blocks; i++) { 1962 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1963 *major = adev->ip_blocks[i].version->major; 1964 *minor = adev->ip_blocks[i].version->minor; 1965 *rev = adev->ip_blocks[i].version->rev; 1966 return true; 1967 } 1968 } 1969 1970 return false; 1971 } 1972 1973 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1974 struct device_attribute *attr, char *buf) 1975 { 1976 struct amdgpu_ras *con = 1977 container_of(attr, struct amdgpu_ras, version_attr); 1978 u32 major, minor, rev; 1979 ssize_t size = 0; 1980 1981 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1982 con->eeprom_control.tbl_hdr.version); 1983 1984 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1985 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1986 major, minor, rev); 1987 1988 return size; 1989 } 1990 1991 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1992 struct device_attribute *attr, char *buf) 1993 { 1994 struct amdgpu_ras *con = 1995 container_of(attr, struct amdgpu_ras, schema_attr); 1996 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1997 } 1998 1999 static struct { 2000 enum ras_event_type type; 2001 const char *name; 2002 } dump_event[] = { 2003 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2004 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2005 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2006 }; 2007 2008 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2009 struct device_attribute *attr, char *buf) 2010 { 2011 struct amdgpu_ras *con = 2012 container_of(attr, struct amdgpu_ras, event_state_attr); 2013 struct ras_event_manager *event_mgr = con->event_mgr; 2014 struct ras_event_state *event_state; 2015 int i, size = 0; 2016 2017 if (!event_mgr) 2018 return -EINVAL; 2019 2020 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2021 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2022 event_state = &event_mgr->event_state[dump_event[i].type]; 2023 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2024 dump_event[i].name, 2025 atomic64_read(&event_state->count), 2026 event_state->last_seqno); 2027 } 2028 2029 return (ssize_t)size; 2030 } 2031 2032 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2033 { 2034 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2035 2036 if (adev->dev->kobj.sd) 2037 sysfs_remove_file_from_group(&adev->dev->kobj, 2038 &con->badpages_attr.attr, 2039 RAS_FS_NAME); 2040 } 2041 2042 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2043 { 2044 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2045 struct attribute *attrs[] = { 2046 &con->features_attr.attr, 2047 &con->version_attr.attr, 2048 &con->schema_attr.attr, 2049 &con->event_state_attr.attr, 2050 NULL 2051 }; 2052 struct attribute_group group = { 2053 .name = RAS_FS_NAME, 2054 .attrs = attrs, 2055 }; 2056 2057 if (adev->dev->kobj.sd) 2058 sysfs_remove_group(&adev->dev->kobj, &group); 2059 2060 return 0; 2061 } 2062 2063 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2064 struct ras_common_if *head) 2065 { 2066 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2067 2068 if (amdgpu_aca_is_enabled(adev)) 2069 return 0; 2070 2071 if (!obj || obj->attr_inuse) 2072 return -EINVAL; 2073 2074 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2075 return 0; 2076 2077 get_obj(obj); 2078 2079 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2080 "%s_err_count", head->name); 2081 2082 obj->sysfs_attr = (struct device_attribute){ 2083 .attr = { 2084 .name = obj->fs_data.sysfs_name, 2085 .mode = S_IRUGO, 2086 }, 2087 .show = amdgpu_ras_sysfs_read, 2088 }; 2089 sysfs_attr_init(&obj->sysfs_attr.attr); 2090 2091 if (sysfs_add_file_to_group(&adev->dev->kobj, 2092 &obj->sysfs_attr.attr, 2093 RAS_FS_NAME)) { 2094 put_obj(obj); 2095 return -EINVAL; 2096 } 2097 2098 obj->attr_inuse = 1; 2099 2100 return 0; 2101 } 2102 2103 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2104 struct ras_common_if *head) 2105 { 2106 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2107 2108 if (amdgpu_aca_is_enabled(adev)) 2109 return 0; 2110 2111 if (!obj || !obj->attr_inuse) 2112 return -EINVAL; 2113 2114 if (adev->dev->kobj.sd) 2115 sysfs_remove_file_from_group(&adev->dev->kobj, 2116 &obj->sysfs_attr.attr, 2117 RAS_FS_NAME); 2118 obj->attr_inuse = 0; 2119 put_obj(obj); 2120 2121 return 0; 2122 } 2123 2124 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2125 { 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2127 struct ras_manager *obj, *tmp; 2128 2129 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2130 amdgpu_ras_sysfs_remove(adev, &obj->head); 2131 } 2132 2133 if (amdgpu_bad_page_threshold != 0) 2134 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2135 2136 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2137 2138 return 0; 2139 } 2140 /* sysfs end */ 2141 2142 /** 2143 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2144 * 2145 * Normally when there is an uncorrectable error, the driver will reset 2146 * the GPU to recover. However, in the event of an unrecoverable error, 2147 * the driver provides an interface to reboot the system automatically 2148 * in that event. 2149 * 2150 * The following file in debugfs provides that interface: 2151 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2152 * 2153 * Usage: 2154 * 2155 * .. code-block:: bash 2156 * 2157 * echo true > .../ras/auto_reboot 2158 * 2159 */ 2160 /* debugfs begin */ 2161 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2162 { 2163 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2164 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2165 struct drm_minor *minor = adev_to_drm(adev)->primary; 2166 struct dentry *dir; 2167 2168 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2169 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2170 &amdgpu_ras_debugfs_ctrl_ops); 2171 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_eeprom_ops); 2173 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2174 &con->bad_page_cnt_threshold); 2175 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2176 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2177 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2178 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2179 &amdgpu_ras_debugfs_eeprom_size_ops); 2180 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2181 S_IRUGO, dir, adev, 2182 &amdgpu_ras_debugfs_eeprom_table_ops); 2183 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2184 2185 /* 2186 * After one uncorrectable error happens, usually GPU recovery will 2187 * be scheduled. But due to the known problem in GPU recovery failing 2188 * to bring GPU back, below interface provides one direct way to 2189 * user to reboot system automatically in such case within 2190 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2191 * will never be called. 2192 */ 2193 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2194 2195 /* 2196 * User could set this not to clean up hardware's error count register 2197 * of RAS IPs during ras recovery. 2198 */ 2199 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2200 &con->disable_ras_err_cnt_harvest); 2201 return dir; 2202 } 2203 2204 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2205 struct ras_fs_if *head, 2206 struct dentry *dir) 2207 { 2208 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2209 2210 if (!obj || !dir) 2211 return; 2212 2213 get_obj(obj); 2214 2215 memcpy(obj->fs_data.debugfs_name, 2216 head->debugfs_name, 2217 sizeof(obj->fs_data.debugfs_name)); 2218 2219 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2220 obj, &amdgpu_ras_debugfs_ops); 2221 } 2222 2223 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2224 { 2225 bool ret; 2226 2227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2228 case IP_VERSION(13, 0, 6): 2229 case IP_VERSION(13, 0, 12): 2230 case IP_VERSION(13, 0, 14): 2231 ret = true; 2232 break; 2233 default: 2234 ret = false; 2235 break; 2236 } 2237 2238 return ret; 2239 } 2240 2241 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2242 { 2243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2244 struct dentry *dir; 2245 struct ras_manager *obj; 2246 struct ras_fs_if fs_info; 2247 2248 /* 2249 * it won't be called in resume path, no need to check 2250 * suspend and gpu reset status 2251 */ 2252 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2253 return; 2254 2255 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2256 2257 list_for_each_entry(obj, &con->head, node) { 2258 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2259 (obj->attr_inuse == 1)) { 2260 sprintf(fs_info.debugfs_name, "%s_err_inject", 2261 get_ras_block_str(&obj->head)); 2262 fs_info.head = obj->head; 2263 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2264 } 2265 } 2266 2267 if (amdgpu_ras_aca_is_supported(adev)) { 2268 if (amdgpu_aca_is_enabled(adev)) 2269 amdgpu_aca_smu_debugfs_init(adev, dir); 2270 else 2271 amdgpu_mca_smu_debugfs_init(adev, dir); 2272 } 2273 } 2274 2275 /* debugfs end */ 2276 2277 /* ras fs */ 2278 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2279 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2280 static DEVICE_ATTR(features, S_IRUGO, 2281 amdgpu_ras_sysfs_features_read, NULL); 2282 static DEVICE_ATTR(version, 0444, 2283 amdgpu_ras_sysfs_version_show, NULL); 2284 static DEVICE_ATTR(schema, 0444, 2285 amdgpu_ras_sysfs_schema_show, NULL); 2286 static DEVICE_ATTR(event_state, 0444, 2287 amdgpu_ras_sysfs_event_state_show, NULL); 2288 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2289 { 2290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2291 struct attribute_group group = { 2292 .name = RAS_FS_NAME, 2293 }; 2294 struct attribute *attrs[] = { 2295 &con->features_attr.attr, 2296 &con->version_attr.attr, 2297 &con->schema_attr.attr, 2298 &con->event_state_attr.attr, 2299 NULL 2300 }; 2301 const struct bin_attribute *bin_attrs[] = { 2302 NULL, 2303 NULL, 2304 }; 2305 int r; 2306 2307 group.attrs = attrs; 2308 2309 /* add features entry */ 2310 con->features_attr = dev_attr_features; 2311 sysfs_attr_init(attrs[0]); 2312 2313 /* add version entry */ 2314 con->version_attr = dev_attr_version; 2315 sysfs_attr_init(attrs[1]); 2316 2317 /* add schema entry */ 2318 con->schema_attr = dev_attr_schema; 2319 sysfs_attr_init(attrs[2]); 2320 2321 /* add event_state entry */ 2322 con->event_state_attr = dev_attr_event_state; 2323 sysfs_attr_init(attrs[3]); 2324 2325 if (amdgpu_bad_page_threshold != 0) { 2326 /* add bad_page_features entry */ 2327 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2328 sysfs_bin_attr_init(&con->badpages_attr); 2329 bin_attrs[0] = &con->badpages_attr; 2330 group.bin_attrs = bin_attrs; 2331 } 2332 2333 r = sysfs_create_group(&adev->dev->kobj, &group); 2334 if (r) 2335 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2336 2337 return 0; 2338 } 2339 2340 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2341 { 2342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2343 struct ras_manager *con_obj, *ip_obj, *tmp; 2344 2345 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2346 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2347 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2348 if (ip_obj) 2349 put_obj(ip_obj); 2350 } 2351 } 2352 2353 amdgpu_ras_sysfs_remove_all(adev); 2354 return 0; 2355 } 2356 /* ras fs end */ 2357 2358 /* ih begin */ 2359 2360 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2361 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2362 * register to check whether the interrupt is triggered or not, and properly 2363 * ack the interrupt if it is there 2364 */ 2365 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2366 { 2367 /* Fatal error events are handled on host side */ 2368 if (amdgpu_sriov_vf(adev)) 2369 return; 2370 /* 2371 * If the current interrupt is caused by a non-fatal RAS error, skip 2372 * check for fatal error. For fatal errors, FED status of all devices 2373 * in XGMI hive gets set when the first device gets fatal error 2374 * interrupt. The error gets propagated to other devices as well, so 2375 * make sure to ack the interrupt regardless of FED status. 2376 */ 2377 if (!amdgpu_ras_get_fed_status(adev) && 2378 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2379 return; 2380 2381 if (amdgpu_uniras_enabled(adev)) { 2382 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2383 return; 2384 } 2385 2386 if (adev->nbio.ras && 2387 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2388 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2389 2390 if (adev->nbio.ras && 2391 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2392 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2393 } 2394 2395 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2396 struct amdgpu_iv_entry *entry) 2397 { 2398 bool poison_stat = false; 2399 struct amdgpu_device *adev = obj->adev; 2400 struct amdgpu_ras_block_object *block_obj = 2401 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2402 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2403 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2404 u64 event_id; 2405 int ret; 2406 2407 if (!block_obj || !con) 2408 return; 2409 2410 ret = amdgpu_ras_mark_ras_event(adev, type); 2411 if (ret) 2412 return; 2413 2414 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2415 /* both query_poison_status and handle_poison_consumption are optional, 2416 * but at least one of them should be implemented if we need poison 2417 * consumption handler 2418 */ 2419 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2420 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2421 if (!poison_stat) { 2422 /* Not poison consumption interrupt, no need to handle it */ 2423 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2424 block_obj->ras_comm.name); 2425 2426 return; 2427 } 2428 } 2429 2430 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2431 2432 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2433 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2434 2435 /* gpu reset is fallback for failed and default cases. 2436 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2437 */ 2438 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2439 event_id = amdgpu_ras_acquire_event_id(adev, type); 2440 RAS_EVENT_LOG(adev, event_id, 2441 "GPU reset for %s RAS poison consumption is issued!\n", 2442 block_obj->ras_comm.name); 2443 amdgpu_ras_reset_gpu(adev); 2444 } 2445 2446 if (!poison_stat) 2447 amdgpu_gfx_poison_consumption_handler(adev, entry); 2448 } 2449 2450 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2451 struct amdgpu_iv_entry *entry) 2452 { 2453 struct amdgpu_device *adev = obj->adev; 2454 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2455 u64 event_id; 2456 int ret; 2457 2458 ret = amdgpu_ras_mark_ras_event(adev, type); 2459 if (ret) 2460 return; 2461 2462 event_id = amdgpu_ras_acquire_event_id(adev, type); 2463 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2464 2465 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2466 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2467 2468 atomic_inc(&con->page_retirement_req_cnt); 2469 atomic_inc(&con->poison_creation_count); 2470 2471 wake_up(&con->page_retirement_wq); 2472 } 2473 } 2474 2475 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2476 struct amdgpu_iv_entry *entry) 2477 { 2478 struct ras_ih_data *data = &obj->ih_data; 2479 struct ras_err_data err_data; 2480 int ret; 2481 2482 if (!data->cb) 2483 return; 2484 2485 ret = amdgpu_ras_error_data_init(&err_data); 2486 if (ret) 2487 return; 2488 2489 /* Let IP handle its data, maybe we need get the output 2490 * from the callback to update the error type/count, etc 2491 */ 2492 amdgpu_ras_set_fed(obj->adev, true); 2493 ret = data->cb(obj->adev, &err_data, entry); 2494 /* ue will trigger an interrupt, and in that case 2495 * we need do a reset to recovery the whole system. 2496 * But leave IP do that recovery, here we just dispatch 2497 * the error. 2498 */ 2499 if (ret == AMDGPU_RAS_SUCCESS) { 2500 /* these counts could be left as 0 if 2501 * some blocks do not count error number 2502 */ 2503 obj->err_data.ue_count += err_data.ue_count; 2504 obj->err_data.ce_count += err_data.ce_count; 2505 obj->err_data.de_count += err_data.de_count; 2506 } 2507 2508 amdgpu_ras_error_data_fini(&err_data); 2509 } 2510 2511 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2512 { 2513 struct ras_ih_data *data = &obj->ih_data; 2514 struct amdgpu_iv_entry entry; 2515 2516 while (data->rptr != data->wptr) { 2517 rmb(); 2518 memcpy(&entry, &data->ring[data->rptr], 2519 data->element_size); 2520 2521 wmb(); 2522 data->rptr = (data->aligned_element_size + 2523 data->rptr) % data->ring_size; 2524 2525 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2526 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2527 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2528 else 2529 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2530 } else { 2531 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2532 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2533 else 2534 dev_warn(obj->adev->dev, 2535 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2536 } 2537 } 2538 } 2539 2540 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2541 { 2542 struct ras_ih_data *data = 2543 container_of(work, struct ras_ih_data, ih_work); 2544 struct ras_manager *obj = 2545 container_of(data, struct ras_manager, ih_data); 2546 2547 amdgpu_ras_interrupt_handler(obj); 2548 } 2549 2550 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2551 struct ras_dispatch_if *info) 2552 { 2553 struct ras_manager *obj; 2554 struct ras_ih_data *data; 2555 2556 if (amdgpu_uniras_enabled(adev)) { 2557 struct ras_ih_info ih_info; 2558 2559 memset(&ih_info, 0, sizeof(ih_info)); 2560 ih_info.block = info->head.block; 2561 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2562 2563 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2564 } 2565 2566 obj = amdgpu_ras_find_obj(adev, &info->head); 2567 if (!obj) 2568 return -EINVAL; 2569 2570 data = &obj->ih_data; 2571 2572 if (data->inuse == 0) 2573 return 0; 2574 2575 /* Might be overflow... */ 2576 memcpy(&data->ring[data->wptr], info->entry, 2577 data->element_size); 2578 2579 wmb(); 2580 data->wptr = (data->aligned_element_size + 2581 data->wptr) % data->ring_size; 2582 2583 schedule_work(&data->ih_work); 2584 2585 return 0; 2586 } 2587 2588 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2589 struct ras_common_if *head) 2590 { 2591 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2592 struct ras_ih_data *data; 2593 2594 if (!obj) 2595 return -EINVAL; 2596 2597 data = &obj->ih_data; 2598 if (data->inuse == 0) 2599 return 0; 2600 2601 cancel_work_sync(&data->ih_work); 2602 2603 kfree(data->ring); 2604 memset(data, 0, sizeof(*data)); 2605 put_obj(obj); 2606 2607 return 0; 2608 } 2609 2610 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2611 struct ras_common_if *head) 2612 { 2613 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2614 struct ras_ih_data *data; 2615 struct amdgpu_ras_block_object *ras_obj; 2616 2617 if (!obj) { 2618 /* in case we registe the IH before enable ras feature */ 2619 obj = amdgpu_ras_create_obj(adev, head); 2620 if (!obj) 2621 return -EINVAL; 2622 } else 2623 get_obj(obj); 2624 2625 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2626 2627 data = &obj->ih_data; 2628 /* add the callback.etc */ 2629 *data = (struct ras_ih_data) { 2630 .inuse = 0, 2631 .cb = ras_obj->ras_cb, 2632 .element_size = sizeof(struct amdgpu_iv_entry), 2633 .rptr = 0, 2634 .wptr = 0, 2635 }; 2636 2637 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2638 2639 data->aligned_element_size = ALIGN(data->element_size, 8); 2640 /* the ring can store 64 iv entries. */ 2641 data->ring_size = 64 * data->aligned_element_size; 2642 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2643 if (!data->ring) { 2644 put_obj(obj); 2645 return -ENOMEM; 2646 } 2647 2648 /* IH is ready */ 2649 data->inuse = 1; 2650 2651 return 0; 2652 } 2653 2654 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2655 { 2656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2657 struct ras_manager *obj, *tmp; 2658 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2660 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2661 } 2662 2663 return 0; 2664 } 2665 /* ih end */ 2666 2667 /* traversal all IPs except NBIO to query error counter */ 2668 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2669 { 2670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2671 struct ras_manager *obj; 2672 2673 if (!adev->ras_enabled || !con) 2674 return; 2675 2676 list_for_each_entry(obj, &con->head, node) { 2677 struct ras_query_if info = { 2678 .head = obj->head, 2679 }; 2680 2681 /* 2682 * PCIE_BIF IP has one different isr by ras controller 2683 * interrupt, the specific ras counter query will be 2684 * done in that isr. So skip such block from common 2685 * sync flood interrupt isr calling. 2686 */ 2687 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2688 continue; 2689 2690 /* 2691 * this is a workaround for aldebaran, skip send msg to 2692 * smu to get ecc_info table due to smu handle get ecc 2693 * info table failed temporarily. 2694 * should be removed until smu fix handle ecc_info table. 2695 */ 2696 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2697 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2698 IP_VERSION(13, 0, 2))) 2699 continue; 2700 2701 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2702 2703 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2704 IP_VERSION(11, 0, 2) && 2705 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 4) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(13, 0, 0)) { 2709 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2710 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2711 } 2712 } 2713 } 2714 2715 /* Parse RdRspStatus and WrRspStatus */ 2716 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2717 struct ras_query_if *info) 2718 { 2719 struct amdgpu_ras_block_object *block_obj; 2720 /* 2721 * Only two block need to query read/write 2722 * RspStatus at current state 2723 */ 2724 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2725 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2726 return; 2727 2728 block_obj = amdgpu_ras_get_ras_block(adev, 2729 info->head.block, 2730 info->head.sub_block_index); 2731 2732 if (!block_obj || !block_obj->hw_ops) { 2733 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2734 get_ras_block_str(&info->head)); 2735 return; 2736 } 2737 2738 if (block_obj->hw_ops->query_ras_error_status) 2739 block_obj->hw_ops->query_ras_error_status(adev); 2740 2741 } 2742 2743 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2744 { 2745 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2746 struct ras_manager *obj; 2747 2748 if (!adev->ras_enabled || !con) 2749 return; 2750 2751 list_for_each_entry(obj, &con->head, node) { 2752 struct ras_query_if info = { 2753 .head = obj->head, 2754 }; 2755 2756 amdgpu_ras_error_status_query(adev, &info); 2757 } 2758 } 2759 2760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2761 struct ras_badpage *bps, uint32_t count, uint32_t start) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_err_handler_data *data; 2765 int r = 0; 2766 uint32_t i; 2767 2768 if (!con || !con->eh_data || !bps || !count) 2769 return -EINVAL; 2770 2771 mutex_lock(&con->recovery_lock); 2772 data = con->eh_data; 2773 if (start < data->count) { 2774 for (i = start; i < data->count; i++) { 2775 if (!data->bps[i].ts) 2776 continue; 2777 2778 bps[r].bp = data->bps[i].retired_page; 2779 r++; 2780 if (r >= count) 2781 break; 2782 } 2783 } 2784 mutex_unlock(&con->recovery_lock); 2785 2786 return r; 2787 } 2788 2789 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2790 struct ras_badpage *bps, uint32_t count, uint32_t start) 2791 { 2792 struct ras_cmd_bad_pages_info_req cmd_input; 2793 struct ras_cmd_bad_pages_info_rsp *output; 2794 uint32_t group, start_group, end_group; 2795 uint32_t pos, pos_in_group; 2796 int r = 0, i; 2797 2798 if (!bps || !count) 2799 return -EINVAL; 2800 2801 output = kmalloc(sizeof(*output), GFP_KERNEL); 2802 if (!output) 2803 return -ENOMEM; 2804 2805 memset(&cmd_input, 0, sizeof(cmd_input)); 2806 2807 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2808 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2809 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2810 2811 pos = start; 2812 for (group = start_group; group < end_group; group++) { 2813 memset(output, 0, sizeof(*output)); 2814 cmd_input.group_index = group; 2815 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2816 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2817 goto out; 2818 2819 if (pos >= output->bp_total_cnt) 2820 goto out; 2821 2822 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2823 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2824 if (!output->records[i].ts) 2825 continue; 2826 2827 bps[r].bp = output->records[i].retired_page; 2828 r++; 2829 if (r >= count) 2830 goto out; 2831 } 2832 } 2833 2834 out: 2835 kfree(output); 2836 return r; 2837 } 2838 2839 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2840 struct amdgpu_hive_info *hive, bool status) 2841 { 2842 struct amdgpu_device *tmp_adev; 2843 2844 if (hive) { 2845 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2846 amdgpu_ras_set_fed(tmp_adev, status); 2847 } else { 2848 amdgpu_ras_set_fed(adev, status); 2849 } 2850 } 2851 2852 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2853 { 2854 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2856 int hive_ras_recovery = 0; 2857 2858 if (hive) { 2859 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2860 amdgpu_put_xgmi_hive(hive); 2861 } 2862 2863 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2864 return true; 2865 2866 return false; 2867 } 2868 2869 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2870 { 2871 if (amdgpu_ras_intr_triggered()) 2872 return RAS_EVENT_TYPE_FATAL; 2873 else 2874 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2875 } 2876 2877 static void amdgpu_ras_do_recovery(struct work_struct *work) 2878 { 2879 struct amdgpu_ras *ras = 2880 container_of(work, struct amdgpu_ras, recovery_work); 2881 struct amdgpu_device *remote_adev = NULL; 2882 struct amdgpu_device *adev = ras->adev; 2883 struct list_head device_list, *device_list_handle = NULL; 2884 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2885 unsigned int error_query_mode; 2886 enum ras_event_type type; 2887 2888 if (hive) { 2889 atomic_set(&hive->ras_recovery, 1); 2890 2891 /* If any device which is part of the hive received RAS fatal 2892 * error interrupt, set fatal error status on all. This 2893 * condition will need a recovery, and flag will be cleared 2894 * as part of recovery. 2895 */ 2896 list_for_each_entry(remote_adev, &hive->device_list, 2897 gmc.xgmi.head) 2898 if (amdgpu_ras_get_fed_status(remote_adev)) { 2899 amdgpu_ras_set_fed_all(adev, hive, true); 2900 break; 2901 } 2902 } 2903 if (!ras->disable_ras_err_cnt_harvest) { 2904 2905 /* Build list of devices to query RAS related errors */ 2906 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2907 device_list_handle = &hive->device_list; 2908 } else { 2909 INIT_LIST_HEAD(&device_list); 2910 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2911 device_list_handle = &device_list; 2912 } 2913 2914 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2915 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2916 /* wait 500ms to ensure pmfw polling mca bank info done */ 2917 msleep(500); 2918 } 2919 } 2920 2921 type = amdgpu_ras_get_fatal_error_event(adev); 2922 list_for_each_entry(remote_adev, 2923 device_list_handle, gmc.xgmi.head) { 2924 if (amdgpu_uniras_enabled(remote_adev)) { 2925 amdgpu_ras_mgr_update_ras_ecc(remote_adev); 2926 } else { 2927 amdgpu_ras_query_err_status(remote_adev); 2928 amdgpu_ras_log_on_err_counter(remote_adev, type); 2929 } 2930 } 2931 2932 } 2933 2934 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2935 struct amdgpu_reset_context reset_context; 2936 memset(&reset_context, 0, sizeof(reset_context)); 2937 2938 reset_context.method = AMD_RESET_METHOD_NONE; 2939 reset_context.reset_req_dev = adev; 2940 reset_context.src = AMDGPU_RESET_SRC_RAS; 2941 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2942 2943 /* Perform full reset in fatal error mode */ 2944 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2945 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2946 else { 2947 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2948 2949 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2950 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2951 reset_context.method = AMD_RESET_METHOD_MODE2; 2952 } 2953 2954 /* Fatal error occurs in poison mode, mode1 reset is used to 2955 * recover gpu. 2956 */ 2957 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2958 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2959 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2960 2961 psp_fatal_error_recovery_quirk(&adev->psp); 2962 } 2963 } 2964 2965 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2966 } 2967 atomic_set(&ras->in_recovery, 0); 2968 if (hive) { 2969 atomic_set(&hive->ras_recovery, 0); 2970 amdgpu_put_xgmi_hive(hive); 2971 } 2972 } 2973 2974 /* alloc/realloc bps array */ 2975 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2976 struct ras_err_handler_data *data, int pages) 2977 { 2978 unsigned int old_space = data->count + data->space_left; 2979 unsigned int new_space = old_space + pages; 2980 unsigned int align_space = ALIGN(new_space, 512); 2981 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2982 2983 if (!bps) { 2984 return -ENOMEM; 2985 } 2986 2987 if (data->bps) { 2988 memcpy(bps, data->bps, 2989 data->count * sizeof(*data->bps)); 2990 kfree(data->bps); 2991 } 2992 2993 data->bps = bps; 2994 data->space_left += align_space - old_space; 2995 return 0; 2996 } 2997 2998 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2999 struct eeprom_table_record *bps, 3000 struct ras_err_data *err_data) 3001 { 3002 struct ta_ras_query_address_input addr_in; 3003 uint32_t socket = 0; 3004 int ret = 0; 3005 3006 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3007 socket = adev->smuio.funcs->get_socket_id(adev); 3008 3009 /* reinit err_data */ 3010 err_data->err_addr_cnt = 0; 3011 err_data->err_addr_len = adev->umc.retire_unit; 3012 3013 memset(&addr_in, 0, sizeof(addr_in)); 3014 addr_in.ma.err_addr = bps->address; 3015 addr_in.ma.socket_id = socket; 3016 addr_in.ma.ch_inst = bps->mem_channel; 3017 /* tell RAS TA the node instance is not used */ 3018 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3019 3020 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3021 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3022 &addr_in, NULL, false); 3023 3024 return ret; 3025 } 3026 3027 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3028 struct eeprom_table_record *bps, 3029 struct ras_err_data *err_data) 3030 { 3031 struct ta_ras_query_address_input addr_in; 3032 uint32_t die_id, socket = 0; 3033 3034 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3035 socket = adev->smuio.funcs->get_socket_id(adev); 3036 3037 /* although die id is gotten from PA in nps1 mode, the id is 3038 * fitable for any nps mode 3039 */ 3040 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3041 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3042 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3043 else 3044 return -EINVAL; 3045 3046 /* reinit err_data */ 3047 err_data->err_addr_cnt = 0; 3048 err_data->err_addr_len = adev->umc.retire_unit; 3049 3050 memset(&addr_in, 0, sizeof(addr_in)); 3051 addr_in.ma.err_addr = bps->address; 3052 addr_in.ma.ch_inst = bps->mem_channel; 3053 addr_in.ma.umc_inst = bps->mcumc_id; 3054 addr_in.ma.node_inst = die_id; 3055 addr_in.ma.socket_id = socket; 3056 3057 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3058 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3059 &addr_in, NULL, false); 3060 else 3061 return -EINVAL; 3062 } 3063 3064 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3065 struct eeprom_table_record *bps, int count) 3066 { 3067 int j; 3068 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3069 struct ras_err_handler_data *data = con->eh_data; 3070 3071 for (j = 0; j < count; j++) { 3072 if (amdgpu_ras_check_bad_page_unlock(con, 3073 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3074 data->count++; 3075 data->space_left--; 3076 continue; 3077 } 3078 3079 if (!data->space_left && 3080 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3081 return -ENOMEM; 3082 } 3083 3084 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3085 3086 memcpy(&data->bps[data->count], &(bps[j]), 3087 sizeof(struct eeprom_table_record)); 3088 data->count++; 3089 data->space_left--; 3090 con->bad_page_num++; 3091 } 3092 3093 return 0; 3094 } 3095 3096 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3097 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3098 enum amdgpu_memory_partition nps) 3099 { 3100 int i = 0; 3101 enum amdgpu_memory_partition save_nps; 3102 3103 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3104 3105 /*old asics just have pa in eeprom*/ 3106 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3107 memcpy(err_data->err_addr, bps, 3108 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3109 goto out; 3110 } 3111 3112 for (i = 0; i < adev->umc.retire_unit; i++) 3113 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3114 3115 if (save_nps) { 3116 if (save_nps == nps) { 3117 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3118 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3119 return -EINVAL; 3120 for (i = 0; i < adev->umc.retire_unit; i++) { 3121 err_data->err_addr[i].address = bps[0].address; 3122 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3123 err_data->err_addr[i].bank = bps[0].bank; 3124 err_data->err_addr[i].err_type = bps[0].err_type; 3125 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3126 } 3127 } else { 3128 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3129 return -EINVAL; 3130 } 3131 } else { 3132 if (bps[0].address == 0) { 3133 /* for specific old eeprom data, mca address is not stored, 3134 * calc it from pa 3135 */ 3136 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3137 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3138 return -EINVAL; 3139 } 3140 3141 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3142 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3143 memcpy(err_data->err_addr, bps, 3144 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3145 else 3146 return -EOPNOTSUPP; 3147 } 3148 } 3149 3150 out: 3151 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3152 } 3153 3154 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3155 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3156 enum amdgpu_memory_partition nps) 3157 { 3158 int i = 0; 3159 enum amdgpu_memory_partition save_nps; 3160 3161 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3162 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3163 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3164 } else { 3165 save_nps = nps; 3166 } 3167 3168 if (save_nps == nps) { 3169 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3170 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3171 return -EINVAL; 3172 for (i = 0; i < adev->umc.retire_unit; i++) { 3173 err_data->err_addr[i].address = bps->address; 3174 err_data->err_addr[i].mem_channel = bps->mem_channel; 3175 err_data->err_addr[i].bank = bps->bank; 3176 err_data->err_addr[i].err_type = bps->err_type; 3177 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3178 } 3179 } else { 3180 if (bps->address) { 3181 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3182 return -EINVAL; 3183 } else { 3184 /* for specific old eeprom data, mca address is not stored, 3185 * calc it from pa 3186 */ 3187 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3188 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3189 return -EINVAL; 3190 3191 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3192 return -EOPNOTSUPP; 3193 } 3194 } 3195 3196 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3197 adev->umc.retire_unit); 3198 } 3199 3200 /* it deal with vram only. */ 3201 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3202 struct eeprom_table_record *bps, int pages, bool from_rom) 3203 { 3204 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3205 struct ras_err_data err_data; 3206 struct amdgpu_ras_eeprom_control *control = 3207 &adev->psp.ras_context.ras->eeprom_control; 3208 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3209 int ret = 0; 3210 uint32_t i = 0; 3211 3212 if (!con || !con->eh_data || !bps || pages <= 0) 3213 return 0; 3214 3215 if (from_rom) { 3216 err_data.err_addr = 3217 kcalloc(adev->umc.retire_unit, 3218 sizeof(struct eeprom_table_record), GFP_KERNEL); 3219 if (!err_data.err_addr) { 3220 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3221 return -ENOMEM; 3222 } 3223 3224 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3225 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3226 } 3227 3228 mutex_lock(&con->recovery_lock); 3229 3230 if (from_rom) { 3231 /* there is no pa recs in V3, so skip pa recs processing */ 3232 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3233 !amdgpu_ras_smu_eeprom_supported(adev)) { 3234 for (i = 0; i < pages; i++) { 3235 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3236 if ((bps[i].address == bps[i + 1].address) && 3237 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3238 /* deal with retire_unit records a time */ 3239 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3240 &bps[i], &err_data, nps); 3241 if (ret) 3242 con->bad_page_num -= adev->umc.retire_unit; 3243 i += (adev->umc.retire_unit - 1); 3244 } else { 3245 break; 3246 } 3247 } else { 3248 break; 3249 } 3250 } 3251 } 3252 for (; i < pages; i++) { 3253 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3254 &bps[i], &err_data, nps); 3255 if (ret) 3256 con->bad_page_num -= adev->umc.retire_unit; 3257 } 3258 3259 con->eh_data->count_saved = con->eh_data->count; 3260 } else { 3261 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3262 } 3263 3264 if (from_rom) 3265 kfree(err_data.err_addr); 3266 mutex_unlock(&con->recovery_lock); 3267 3268 return ret; 3269 } 3270 3271 /* 3272 * write error record array to eeprom, the function should be 3273 * protected by recovery_lock 3274 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3275 */ 3276 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3277 unsigned long *new_cnt) 3278 { 3279 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3280 struct ras_err_handler_data *data; 3281 struct amdgpu_ras_eeprom_control *control; 3282 int save_count, unit_num, i; 3283 3284 if (!con || !con->eh_data) { 3285 if (new_cnt) 3286 *new_cnt = 0; 3287 3288 return 0; 3289 } 3290 3291 if (!con->eeprom_control.is_eeprom_valid) { 3292 dev_warn(adev->dev, 3293 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3294 if (new_cnt) 3295 *new_cnt = 0; 3296 3297 return 0; 3298 } 3299 3300 mutex_lock(&con->recovery_lock); 3301 control = &con->eeprom_control; 3302 data = con->eh_data; 3303 if (amdgpu_ras_smu_eeprom_supported(adev)) 3304 unit_num = control->ras_num_recs - 3305 control->ras_num_recs_old; 3306 else 3307 unit_num = data->count / adev->umc.retire_unit - 3308 control->ras_num_recs; 3309 3310 save_count = con->bad_page_num - control->ras_num_bad_pages; 3311 mutex_unlock(&con->recovery_lock); 3312 3313 if (new_cnt) 3314 *new_cnt = unit_num; 3315 3316 /* only new entries are saved */ 3317 if (unit_num && save_count) { 3318 /*old asics only save pa to eeprom like before*/ 3319 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3320 if (amdgpu_ras_eeprom_append(control, 3321 &data->bps[data->count_saved], unit_num)) { 3322 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3323 return -EIO; 3324 } 3325 } else { 3326 for (i = 0; i < unit_num; i++) { 3327 if (amdgpu_ras_eeprom_append(control, 3328 &data->bps[data->count_saved + 3329 i * adev->umc.retire_unit], 1)) { 3330 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3331 return -EIO; 3332 } 3333 } 3334 } 3335 3336 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3337 data->count_saved = data->count; 3338 } 3339 3340 return 0; 3341 } 3342 3343 /* 3344 * read error record array in eeprom and reserve enough space for 3345 * storing new bad pages 3346 */ 3347 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3348 { 3349 struct amdgpu_ras_eeprom_control *control = 3350 &adev->psp.ras_context.ras->eeprom_control; 3351 struct eeprom_table_record *bps; 3352 int ret, i = 0; 3353 3354 /* no bad page record, skip eeprom access */ 3355 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3356 return 0; 3357 3358 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3359 if (!bps) 3360 return -ENOMEM; 3361 3362 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3363 if (ret) { 3364 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3365 } else { 3366 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3367 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3368 as pa recs, so add verion check to avoid it. 3369 */ 3370 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3371 !amdgpu_ras_smu_eeprom_supported(adev)) { 3372 for (i = 0; i < control->ras_num_recs; i++) { 3373 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3374 if ((bps[i].address == bps[i + 1].address) && 3375 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3376 control->ras_num_pa_recs += adev->umc.retire_unit; 3377 i += (adev->umc.retire_unit - 1); 3378 } else { 3379 control->ras_num_mca_recs += 3380 (control->ras_num_recs - i); 3381 break; 3382 } 3383 } else { 3384 control->ras_num_mca_recs += (control->ras_num_recs - i); 3385 break; 3386 } 3387 } 3388 } else { 3389 control->ras_num_mca_recs = control->ras_num_recs; 3390 } 3391 } 3392 3393 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3394 if (ret) 3395 goto out; 3396 3397 ret = amdgpu_ras_eeprom_check(control); 3398 if (ret) 3399 goto out; 3400 3401 /* HW not usable */ 3402 if (amdgpu_ras_is_rma(adev)) 3403 ret = -EHWPOISON; 3404 } 3405 3406 out: 3407 kfree(bps); 3408 return ret; 3409 } 3410 3411 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3412 uint64_t addr) 3413 { 3414 struct ras_err_handler_data *data = con->eh_data; 3415 struct amdgpu_device *adev = con->adev; 3416 int i; 3417 3418 if ((addr >= adev->gmc.mc_vram_size && 3419 adev->gmc.mc_vram_size) || 3420 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3421 return -EINVAL; 3422 3423 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3424 for (i = 0; i < data->count; i++) 3425 if (addr == data->bps[i].retired_page) 3426 return 1; 3427 3428 return 0; 3429 } 3430 3431 /* 3432 * check if an address belongs to bad page 3433 * 3434 * Note: this check is only for umc block 3435 */ 3436 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3437 uint64_t addr) 3438 { 3439 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3440 int ret = 0; 3441 3442 if (!con || !con->eh_data) 3443 return ret; 3444 3445 mutex_lock(&con->recovery_lock); 3446 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3447 mutex_unlock(&con->recovery_lock); 3448 return ret; 3449 } 3450 3451 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3452 uint32_t max_count) 3453 { 3454 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3455 3456 /* 3457 * amdgpu_bad_page_threshold is used to config 3458 * the threshold for the number of bad pages. 3459 * -1: Threshold is set to default value 3460 * Driver will issue a warning message when threshold is reached 3461 * and continue runtime services. 3462 * 0: Disable bad page retirement 3463 * Driver will not retire bad pages 3464 * which is intended for debugging purpose. 3465 * -2: Threshold is determined by a formula 3466 * that assumes 1 bad page per 100M of local memory. 3467 * Driver will continue runtime services when threhold is reached. 3468 * 0 < threshold < max number of bad page records in EEPROM, 3469 * A user-defined threshold is set 3470 * Driver will halt runtime services when this custom threshold is reached. 3471 */ 3472 if (amdgpu_bad_page_threshold == -2) { 3473 u64 val = adev->gmc.mc_vram_size; 3474 3475 do_div(val, RAS_BAD_PAGE_COVER); 3476 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3477 max_count); 3478 } else if (amdgpu_bad_page_threshold == -1) { 3479 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3480 } else { 3481 con->bad_page_cnt_threshold = min_t(int, max_count, 3482 amdgpu_bad_page_threshold); 3483 } 3484 } 3485 3486 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3487 enum amdgpu_ras_block block, uint16_t pasid, 3488 pasid_notify pasid_fn, void *data, uint32_t reset) 3489 { 3490 int ret = 0; 3491 struct ras_poison_msg poison_msg; 3492 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3493 3494 memset(&poison_msg, 0, sizeof(poison_msg)); 3495 poison_msg.block = block; 3496 poison_msg.pasid = pasid; 3497 poison_msg.reset = reset; 3498 poison_msg.pasid_fn = pasid_fn; 3499 poison_msg.data = data; 3500 3501 ret = kfifo_put(&con->poison_fifo, poison_msg); 3502 if (!ret) { 3503 dev_err(adev->dev, "Poison message fifo is full!\n"); 3504 return -ENOSPC; 3505 } 3506 3507 return 0; 3508 } 3509 3510 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3511 struct ras_poison_msg *poison_msg) 3512 { 3513 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3514 3515 return kfifo_get(&con->poison_fifo, poison_msg); 3516 } 3517 3518 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3519 { 3520 mutex_init(&ecc_log->lock); 3521 3522 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3523 ecc_log->de_queried_count = 0; 3524 ecc_log->consumption_q_count = 0; 3525 } 3526 3527 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3528 { 3529 struct radix_tree_iter iter; 3530 void __rcu **slot; 3531 struct ras_ecc_err *ecc_err; 3532 3533 mutex_lock(&ecc_log->lock); 3534 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3535 ecc_err = radix_tree_deref_slot(slot); 3536 kfree(ecc_err->err_pages.pfn); 3537 kfree(ecc_err); 3538 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3539 } 3540 mutex_unlock(&ecc_log->lock); 3541 3542 mutex_destroy(&ecc_log->lock); 3543 ecc_log->de_queried_count = 0; 3544 ecc_log->consumption_q_count = 0; 3545 } 3546 3547 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3548 uint32_t delayed_ms) 3549 { 3550 int ret; 3551 3552 mutex_lock(&con->umc_ecc_log.lock); 3553 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3554 UMC_ECC_NEW_DETECTED_TAG); 3555 mutex_unlock(&con->umc_ecc_log.lock); 3556 3557 if (ret) 3558 schedule_delayed_work(&con->page_retirement_dwork, 3559 msecs_to_jiffies(delayed_ms)); 3560 3561 return ret ? true : false; 3562 } 3563 3564 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3565 { 3566 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3567 page_retirement_dwork.work); 3568 struct amdgpu_device *adev = con->adev; 3569 struct ras_err_data err_data; 3570 3571 /* If gpu reset is ongoing, delay retiring the bad pages */ 3572 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3573 amdgpu_ras_schedule_retirement_dwork(con, 3574 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3575 return; 3576 } 3577 3578 amdgpu_ras_error_data_init(&err_data); 3579 3580 amdgpu_umc_handle_bad_pages(adev, &err_data); 3581 3582 amdgpu_ras_error_data_fini(&err_data); 3583 3584 amdgpu_ras_schedule_retirement_dwork(con, 3585 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3586 } 3587 3588 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3589 uint32_t poison_creation_count) 3590 { 3591 int ret = 0; 3592 struct ras_ecc_log_info *ecc_log; 3593 struct ras_query_if info; 3594 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3595 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3596 u64 de_queried_count; 3597 u64 consumption_q_count; 3598 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3599 3600 memset(&info, 0, sizeof(info)); 3601 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3602 3603 ecc_log = &ras->umc_ecc_log; 3604 ecc_log->de_queried_count = 0; 3605 ecc_log->consumption_q_count = 0; 3606 3607 do { 3608 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3609 if (ret) 3610 return ret; 3611 3612 de_queried_count = ecc_log->de_queried_count; 3613 consumption_q_count = ecc_log->consumption_q_count; 3614 3615 if (de_queried_count && consumption_q_count) 3616 break; 3617 3618 msleep(100); 3619 } while (--timeout); 3620 3621 if (de_queried_count) 3622 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3623 3624 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3625 amdgpu_ras_reset_gpu(adev); 3626 3627 return 0; 3628 } 3629 3630 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3631 { 3632 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3633 struct ras_poison_msg msg; 3634 int ret; 3635 3636 do { 3637 ret = kfifo_get(&con->poison_fifo, &msg); 3638 } while (ret); 3639 } 3640 3641 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3642 uint32_t msg_count, uint32_t *gpu_reset) 3643 { 3644 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3645 uint32_t reset_flags = 0, reset = 0; 3646 struct ras_poison_msg msg; 3647 int ret, i; 3648 3649 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3650 3651 for (i = 0; i < msg_count; i++) { 3652 ret = amdgpu_ras_get_poison_req(adev, &msg); 3653 if (!ret) 3654 continue; 3655 3656 if (msg.pasid_fn) 3657 msg.pasid_fn(adev, msg.pasid, msg.data); 3658 3659 reset_flags |= msg.reset; 3660 } 3661 3662 /* 3663 * Try to ensure poison creation handler is completed first 3664 * to set rma if bad page exceed threshold. 3665 */ 3666 flush_delayed_work(&con->page_retirement_dwork); 3667 3668 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3669 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3670 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3671 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3672 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3673 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3674 else 3675 reset = reset_flags; 3676 3677 con->gpu_reset_flags |= reset; 3678 amdgpu_ras_reset_gpu(adev); 3679 3680 *gpu_reset = reset; 3681 3682 /* Wait for gpu recovery to complete */ 3683 flush_work(&con->recovery_work); 3684 } 3685 3686 return 0; 3687 } 3688 3689 static int amdgpu_ras_page_retirement_thread(void *param) 3690 { 3691 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3692 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3693 uint32_t poison_creation_count, msg_count; 3694 uint32_t gpu_reset; 3695 int ret; 3696 3697 while (!kthread_should_stop()) { 3698 3699 wait_event_interruptible(con->page_retirement_wq, 3700 kthread_should_stop() || 3701 atomic_read(&con->page_retirement_req_cnt)); 3702 3703 if (kthread_should_stop()) 3704 break; 3705 3706 mutex_lock(&con->poison_lock); 3707 gpu_reset = 0; 3708 3709 do { 3710 poison_creation_count = atomic_read(&con->poison_creation_count); 3711 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3712 if (ret == -EIO) 3713 break; 3714 3715 if (poison_creation_count) { 3716 atomic_sub(poison_creation_count, &con->poison_creation_count); 3717 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3718 } 3719 } while (atomic_read(&con->poison_creation_count) && 3720 !atomic_read(&con->poison_consumption_count)); 3721 3722 if (ret != -EIO) { 3723 msg_count = kfifo_len(&con->poison_fifo); 3724 if (msg_count) { 3725 ret = amdgpu_ras_poison_consumption_handler(adev, 3726 msg_count, &gpu_reset); 3727 if ((ret != -EIO) && 3728 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3729 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3730 } 3731 } 3732 3733 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3734 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3735 /* Clear poison creation request */ 3736 atomic_set(&con->poison_creation_count, 0); 3737 atomic_set(&con->poison_consumption_count, 0); 3738 3739 /* Clear poison fifo */ 3740 amdgpu_ras_clear_poison_fifo(adev); 3741 3742 /* Clear all poison requests */ 3743 atomic_set(&con->page_retirement_req_cnt, 0); 3744 3745 if (ret == -EIO) { 3746 /* Wait for mode-1 reset to complete */ 3747 down_read(&adev->reset_domain->sem); 3748 up_read(&adev->reset_domain->sem); 3749 } 3750 3751 /* Wake up work to save bad pages to eeprom */ 3752 schedule_delayed_work(&con->page_retirement_dwork, 0); 3753 } else if (gpu_reset) { 3754 /* gpu just completed mode-2 reset or other reset */ 3755 /* Clear poison consumption messages cached in fifo */ 3756 msg_count = kfifo_len(&con->poison_fifo); 3757 if (msg_count) { 3758 amdgpu_ras_clear_poison_fifo(adev); 3759 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3760 } 3761 3762 atomic_set(&con->poison_consumption_count, 0); 3763 3764 /* Wake up work to save bad pages to eeprom */ 3765 schedule_delayed_work(&con->page_retirement_dwork, 0); 3766 } 3767 mutex_unlock(&con->poison_lock); 3768 } 3769 3770 return 0; 3771 } 3772 3773 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3774 { 3775 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3776 struct amdgpu_ras_eeprom_control *control; 3777 int ret; 3778 3779 if (!con || amdgpu_sriov_vf(adev)) 3780 return 0; 3781 3782 if (amdgpu_uniras_enabled(adev)) 3783 return 0; 3784 3785 control = &con->eeprom_control; 3786 con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev); 3787 3788 ret = amdgpu_ras_eeprom_init(control); 3789 control->is_eeprom_valid = !ret; 3790 3791 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3792 control->ras_num_pa_recs = control->ras_num_recs; 3793 3794 if (adev->umc.ras && 3795 adev->umc.ras->get_retire_flip_bits) 3796 adev->umc.ras->get_retire_flip_bits(adev); 3797 3798 if (control->ras_num_recs && control->is_eeprom_valid) { 3799 ret = amdgpu_ras_load_bad_pages(adev); 3800 if (ret) { 3801 control->is_eeprom_valid = false; 3802 return 0; 3803 } 3804 3805 amdgpu_dpm_send_hbm_bad_pages_num( 3806 adev, control->ras_num_bad_pages); 3807 3808 if (con->update_channel_flag == true) { 3809 amdgpu_dpm_send_hbm_bad_channel_flag( 3810 adev, control->bad_channel_bitmap); 3811 con->update_channel_flag = false; 3812 } 3813 3814 /* The format action is only applied to new ASICs */ 3815 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3816 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3817 if (!amdgpu_ras_eeprom_reset_table(control)) 3818 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3819 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3820 } 3821 3822 return 0; 3823 } 3824 3825 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3826 { 3827 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3828 struct ras_err_handler_data **data; 3829 u32 max_eeprom_records_count = 0; 3830 int ret; 3831 3832 if (!con || amdgpu_sriov_vf(adev)) 3833 return 0; 3834 3835 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3836 * supports RAS and debugfs is enabled, but when 3837 * adev->ras_enabled is unset, i.e. when "ras_enable" 3838 * module parameter is set to 0. 3839 */ 3840 con->adev = adev; 3841 3842 if (!adev->ras_enabled) 3843 return 0; 3844 3845 data = &con->eh_data; 3846 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3847 if (!*data) { 3848 ret = -ENOMEM; 3849 goto out; 3850 } 3851 3852 mutex_init(&con->recovery_lock); 3853 mutex_init(&con->poison_lock); 3854 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3855 atomic_set(&con->in_recovery, 0); 3856 atomic_set(&con->rma_in_recovery, 0); 3857 con->eeprom_control.bad_channel_bitmap = 0; 3858 3859 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3860 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3861 3862 if (init_bp_info) { 3863 ret = amdgpu_ras_init_badpage_info(adev); 3864 if (ret) 3865 goto free; 3866 } 3867 3868 mutex_init(&con->page_rsv_lock); 3869 INIT_KFIFO(con->poison_fifo); 3870 mutex_init(&con->page_retirement_lock); 3871 init_waitqueue_head(&con->page_retirement_wq); 3872 atomic_set(&con->page_retirement_req_cnt, 0); 3873 atomic_set(&con->poison_creation_count, 0); 3874 atomic_set(&con->poison_consumption_count, 0); 3875 con->page_retirement_thread = 3876 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3877 if (IS_ERR(con->page_retirement_thread)) { 3878 con->page_retirement_thread = NULL; 3879 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3880 } 3881 3882 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3883 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3884 #ifdef CONFIG_X86_MCE_AMD 3885 if ((adev->asic_type == CHIP_ALDEBARAN) && 3886 (adev->gmc.xgmi.connected_to_cpu)) 3887 amdgpu_register_bad_pages_mca_notifier(adev); 3888 #endif 3889 return 0; 3890 3891 free: 3892 kfree((*data)->bps); 3893 kfree(*data); 3894 con->eh_data = NULL; 3895 out: 3896 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3897 3898 /* 3899 * Except error threshold exceeding case, other failure cases in this 3900 * function would not fail amdgpu driver init. 3901 */ 3902 if (!amdgpu_ras_is_rma(adev)) 3903 ret = 0; 3904 else 3905 ret = -EINVAL; 3906 3907 return ret; 3908 } 3909 3910 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3911 { 3912 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3913 struct ras_err_handler_data *data = con->eh_data; 3914 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3915 bool ret; 3916 3917 /* recovery_init failed to init it, fini is useless */ 3918 if (!data) 3919 return 0; 3920 3921 /* Save all cached bad pages to eeprom */ 3922 do { 3923 flush_delayed_work(&con->page_retirement_dwork); 3924 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3925 } while (ret && max_flush_timeout--); 3926 3927 if (con->page_retirement_thread) 3928 kthread_stop(con->page_retirement_thread); 3929 3930 atomic_set(&con->page_retirement_req_cnt, 0); 3931 atomic_set(&con->poison_creation_count, 0); 3932 3933 mutex_destroy(&con->page_rsv_lock); 3934 3935 cancel_work_sync(&con->recovery_work); 3936 3937 cancel_delayed_work_sync(&con->page_retirement_dwork); 3938 3939 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3940 3941 mutex_lock(&con->recovery_lock); 3942 con->eh_data = NULL; 3943 kfree(data->bps); 3944 kfree(data); 3945 mutex_unlock(&con->recovery_lock); 3946 3947 amdgpu_ras_critical_region_init(adev); 3948 3949 return 0; 3950 } 3951 /* recovery end */ 3952 3953 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3954 { 3955 if (amdgpu_sriov_vf(adev)) { 3956 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3957 case IP_VERSION(13, 0, 2): 3958 case IP_VERSION(13, 0, 6): 3959 case IP_VERSION(13, 0, 12): 3960 case IP_VERSION(13, 0, 14): 3961 return true; 3962 default: 3963 return false; 3964 } 3965 } 3966 3967 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3968 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3969 case IP_VERSION(13, 0, 0): 3970 case IP_VERSION(13, 0, 6): 3971 case IP_VERSION(13, 0, 10): 3972 case IP_VERSION(13, 0, 12): 3973 case IP_VERSION(13, 0, 14): 3974 case IP_VERSION(14, 0, 3): 3975 return true; 3976 default: 3977 return false; 3978 } 3979 } 3980 3981 return adev->asic_type == CHIP_VEGA10 || 3982 adev->asic_type == CHIP_VEGA20 || 3983 adev->asic_type == CHIP_ARCTURUS || 3984 adev->asic_type == CHIP_ALDEBARAN || 3985 adev->asic_type == CHIP_SIENNA_CICHLID; 3986 } 3987 3988 /* 3989 * this is workaround for vega20 workstation sku, 3990 * force enable gfx ras, ignore vbios gfx ras flag 3991 * due to GC EDC can not write 3992 */ 3993 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3994 { 3995 struct atom_context *ctx = adev->mode_info.atom_context; 3996 3997 if (!ctx) 3998 return; 3999 4000 if (strnstr(ctx->vbios_pn, "D16406", 4001 sizeof(ctx->vbios_pn)) || 4002 strnstr(ctx->vbios_pn, "D36002", 4003 sizeof(ctx->vbios_pn))) 4004 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 4005 } 4006 4007 /* Query ras capablity via atomfirmware interface */ 4008 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 4009 { 4010 /* mem_ecc cap */ 4011 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 4012 dev_info(adev->dev, "MEM ECC is active.\n"); 4013 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 4014 1 << AMDGPU_RAS_BLOCK__DF); 4015 } else { 4016 dev_info(adev->dev, "MEM ECC is not presented.\n"); 4017 } 4018 4019 /* sram_ecc cap */ 4020 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4021 dev_info(adev->dev, "SRAM ECC is active.\n"); 4022 if (!amdgpu_sriov_vf(adev)) 4023 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4024 1 << AMDGPU_RAS_BLOCK__DF); 4025 else 4026 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4027 1 << AMDGPU_RAS_BLOCK__SDMA | 4028 1 << AMDGPU_RAS_BLOCK__GFX); 4029 4030 /* 4031 * VCN/JPEG RAS can be supported on both bare metal and 4032 * SRIOV environment 4033 */ 4034 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4035 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4036 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4037 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4038 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4039 1 << AMDGPU_RAS_BLOCK__JPEG); 4040 else 4041 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4042 1 << AMDGPU_RAS_BLOCK__JPEG); 4043 4044 /* 4045 * XGMI RAS is not supported if xgmi num physical nodes 4046 * is zero 4047 */ 4048 if (!adev->gmc.xgmi.num_physical_nodes) 4049 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4050 } else { 4051 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4052 } 4053 } 4054 4055 /* Query poison mode from umc/df IP callbacks */ 4056 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4057 { 4058 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4059 bool df_poison, umc_poison; 4060 4061 /* poison setting is useless on SRIOV guest */ 4062 if (amdgpu_sriov_vf(adev) || !con) 4063 return; 4064 4065 /* Init poison supported flag, the default value is false */ 4066 if (adev->gmc.xgmi.connected_to_cpu || 4067 adev->gmc.is_app_apu) { 4068 /* enabled by default when GPU is connected to CPU */ 4069 con->poison_supported = true; 4070 } else if (adev->df.funcs && 4071 adev->df.funcs->query_ras_poison_mode && 4072 adev->umc.ras && 4073 adev->umc.ras->query_ras_poison_mode) { 4074 df_poison = 4075 adev->df.funcs->query_ras_poison_mode(adev); 4076 umc_poison = 4077 adev->umc.ras->query_ras_poison_mode(adev); 4078 4079 /* Only poison is set in both DF and UMC, we can support it */ 4080 if (df_poison && umc_poison) 4081 con->poison_supported = true; 4082 else if (df_poison != umc_poison) 4083 dev_warn(adev->dev, 4084 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4085 df_poison, umc_poison); 4086 } 4087 } 4088 4089 /* 4090 * check hardware's ras ability which will be saved in hw_supported. 4091 * if hardware does not support ras, we can skip some ras initializtion and 4092 * forbid some ras operations from IP. 4093 * if software itself, say boot parameter, limit the ras ability. We still 4094 * need allow IP do some limited operations, like disable. In such case, 4095 * we have to initialize ras as normal. but need check if operation is 4096 * allowed or not in each function. 4097 */ 4098 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4099 { 4100 adev->ras_hw_enabled = adev->ras_enabled = 0; 4101 4102 if (!amdgpu_ras_asic_supported(adev)) 4103 return; 4104 4105 if (amdgpu_sriov_vf(adev)) { 4106 if (amdgpu_virt_get_ras_capability(adev)) 4107 goto init_ras_enabled_flag; 4108 } 4109 4110 /* query ras capability from psp */ 4111 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4112 goto init_ras_enabled_flag; 4113 4114 /* query ras capablity from bios */ 4115 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4116 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4117 } else { 4118 /* driver only manages a few IP blocks RAS feature 4119 * when GPU is connected cpu through XGMI */ 4120 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4121 1 << AMDGPU_RAS_BLOCK__SDMA | 4122 1 << AMDGPU_RAS_BLOCK__MMHUB); 4123 } 4124 4125 /* apply asic specific settings (vega20 only for now) */ 4126 amdgpu_ras_get_quirks(adev); 4127 4128 /* query poison mode from umc/df ip callback */ 4129 amdgpu_ras_query_poison_mode(adev); 4130 4131 init_ras_enabled_flag: 4132 /* hw_supported needs to be aligned with RAS block mask. */ 4133 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4134 4135 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4136 adev->ras_hw_enabled & amdgpu_ras_mask; 4137 4138 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4139 if (!amdgpu_sriov_vf(adev)) { 4140 adev->aca.is_enabled = 4141 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4142 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4143 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4144 } 4145 4146 /* bad page feature is not applicable to specific app platform */ 4147 if (adev->gmc.is_app_apu && 4148 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4149 amdgpu_bad_page_threshold = 0; 4150 } 4151 4152 static void amdgpu_ras_counte_dw(struct work_struct *work) 4153 { 4154 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4155 ras_counte_delay_work.work); 4156 struct amdgpu_device *adev = con->adev; 4157 struct drm_device *dev = adev_to_drm(adev); 4158 unsigned long ce_count, ue_count; 4159 int res; 4160 4161 res = pm_runtime_get_sync(dev->dev); 4162 if (res < 0) 4163 goto Out; 4164 4165 /* Cache new values. 4166 */ 4167 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4168 atomic_set(&con->ras_ce_count, ce_count); 4169 atomic_set(&con->ras_ue_count, ue_count); 4170 } 4171 4172 Out: 4173 pm_runtime_put_autosuspend(dev->dev); 4174 } 4175 4176 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4177 { 4178 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4179 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4180 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4181 AMDGPU_RAS_ERROR__PARITY; 4182 } 4183 4184 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4185 { 4186 struct ras_event_state *event_state; 4187 int i; 4188 4189 memset(mgr, 0, sizeof(*mgr)); 4190 atomic64_set(&mgr->seqno, 0); 4191 4192 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4193 event_state = &mgr->event_state[i]; 4194 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4195 atomic64_set(&event_state->count, 0); 4196 } 4197 } 4198 4199 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4200 { 4201 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4202 struct amdgpu_hive_info *hive; 4203 4204 if (!ras) 4205 return; 4206 4207 hive = amdgpu_get_xgmi_hive(adev); 4208 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4209 4210 /* init event manager with node 0 on xgmi system */ 4211 if (!amdgpu_reset_in_recovery(adev)) { 4212 if (!hive || adev->gmc.xgmi.node_id == 0) 4213 ras_event_mgr_init(ras->event_mgr); 4214 } 4215 4216 if (hive) 4217 amdgpu_put_xgmi_hive(hive); 4218 } 4219 4220 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4221 { 4222 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4223 4224 if (!con || (adev->flags & AMD_IS_APU)) 4225 return; 4226 4227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4228 case IP_VERSION(13, 0, 2): 4229 case IP_VERSION(13, 0, 6): 4230 case IP_VERSION(13, 0, 12): 4231 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4232 break; 4233 case IP_VERSION(13, 0, 14): 4234 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4235 break; 4236 default: 4237 break; 4238 } 4239 } 4240 4241 int amdgpu_ras_init(struct amdgpu_device *adev) 4242 { 4243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4244 int r; 4245 4246 if (con) 4247 return 0; 4248 4249 con = kzalloc(sizeof(*con) + 4250 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4251 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4252 GFP_KERNEL); 4253 if (!con) 4254 return -ENOMEM; 4255 4256 con->adev = adev; 4257 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4258 atomic_set(&con->ras_ce_count, 0); 4259 atomic_set(&con->ras_ue_count, 0); 4260 4261 con->objs = (struct ras_manager *)(con + 1); 4262 4263 amdgpu_ras_set_context(adev, con); 4264 4265 amdgpu_ras_check_supported(adev); 4266 4267 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4268 /* set gfx block ras context feature for VEGA20 Gaming 4269 * send ras disable cmd to ras ta during ras late init. 4270 */ 4271 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4272 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4273 4274 return 0; 4275 } 4276 4277 r = 0; 4278 goto release_con; 4279 } 4280 4281 con->update_channel_flag = false; 4282 con->features = 0; 4283 con->schema = 0; 4284 INIT_LIST_HEAD(&con->head); 4285 /* Might need get this flag from vbios. */ 4286 con->flags = RAS_DEFAULT_FLAGS; 4287 4288 /* initialize nbio ras function ahead of any other 4289 * ras functions so hardware fatal error interrupt 4290 * can be enabled as early as possible */ 4291 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4292 case IP_VERSION(7, 4, 0): 4293 case IP_VERSION(7, 4, 1): 4294 case IP_VERSION(7, 4, 4): 4295 if (!adev->gmc.xgmi.connected_to_cpu) 4296 adev->nbio.ras = &nbio_v7_4_ras; 4297 break; 4298 case IP_VERSION(4, 3, 0): 4299 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4300 /* unlike other generation of nbio ras, 4301 * nbio v4_3 only support fatal error interrupt 4302 * to inform software that DF is freezed due to 4303 * system fatal error event. driver should not 4304 * enable nbio ras in such case. Instead, 4305 * check DF RAS */ 4306 adev->nbio.ras = &nbio_v4_3_ras; 4307 break; 4308 case IP_VERSION(6, 3, 1): 4309 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4310 /* unlike other generation of nbio ras, 4311 * nbif v6_3_1 only support fatal error interrupt 4312 * to inform software that DF is freezed due to 4313 * system fatal error event. driver should not 4314 * enable nbio ras in such case. Instead, 4315 * check DF RAS 4316 */ 4317 adev->nbio.ras = &nbif_v6_3_1_ras; 4318 break; 4319 case IP_VERSION(7, 9, 0): 4320 case IP_VERSION(7, 9, 1): 4321 if (!adev->gmc.is_app_apu) 4322 adev->nbio.ras = &nbio_v7_9_ras; 4323 break; 4324 default: 4325 /* nbio ras is not available */ 4326 break; 4327 } 4328 4329 /* nbio ras block needs to be enabled ahead of other ras blocks 4330 * to handle fatal error */ 4331 r = amdgpu_nbio_ras_sw_init(adev); 4332 if (r) 4333 return r; 4334 4335 if (adev->nbio.ras && 4336 adev->nbio.ras->init_ras_controller_interrupt) { 4337 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4338 if (r) 4339 goto release_con; 4340 } 4341 4342 if (adev->nbio.ras && 4343 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4344 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4345 if (r) 4346 goto release_con; 4347 } 4348 4349 /* Packed socket_id to ras feature mask bits[31:29] */ 4350 if (adev->smuio.funcs && 4351 adev->smuio.funcs->get_socket_id) 4352 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4353 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4354 4355 /* Get RAS schema for particular SOC */ 4356 con->schema = amdgpu_get_ras_schema(adev); 4357 4358 amdgpu_ras_init_reserved_vram_size(adev); 4359 4360 if (amdgpu_ras_fs_init(adev)) { 4361 r = -EINVAL; 4362 goto release_con; 4363 } 4364 4365 if (amdgpu_ras_aca_is_supported(adev)) { 4366 if (amdgpu_aca_is_enabled(adev)) 4367 r = amdgpu_aca_init(adev); 4368 else 4369 r = amdgpu_mca_init(adev); 4370 if (r) 4371 goto release_con; 4372 } 4373 4374 con->init_task_pid = task_pid_nr(current); 4375 get_task_comm(con->init_task_comm, current); 4376 4377 mutex_init(&con->critical_region_lock); 4378 INIT_LIST_HEAD(&con->critical_region_head); 4379 4380 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4381 "hardware ability[%x] ras_mask[%x]\n", 4382 adev->ras_hw_enabled, adev->ras_enabled); 4383 4384 return 0; 4385 release_con: 4386 amdgpu_ras_set_context(adev, NULL); 4387 kfree(con); 4388 4389 return r; 4390 } 4391 4392 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4393 { 4394 if (adev->gmc.xgmi.connected_to_cpu || 4395 adev->gmc.is_app_apu) 4396 return 1; 4397 return 0; 4398 } 4399 4400 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4401 struct ras_common_if *ras_block) 4402 { 4403 struct ras_query_if info = { 4404 .head = *ras_block, 4405 }; 4406 4407 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4408 return 0; 4409 4410 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4411 DRM_WARN("RAS init harvest failure"); 4412 4413 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4414 DRM_WARN("RAS init harvest reset failure"); 4415 4416 return 0; 4417 } 4418 4419 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4420 { 4421 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4422 4423 if (!con) 4424 return false; 4425 4426 return con->poison_supported; 4427 } 4428 4429 /* helper function to handle common stuff in ip late init phase */ 4430 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4431 struct ras_common_if *ras_block) 4432 { 4433 struct amdgpu_ras_block_object *ras_obj = NULL; 4434 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4435 struct ras_query_if *query_info; 4436 unsigned long ue_count, ce_count; 4437 int r; 4438 4439 /* disable RAS feature per IP block if it is not supported */ 4440 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4441 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4442 return 0; 4443 } 4444 4445 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4446 if (r) { 4447 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4448 /* in resume phase, if fail to enable ras, 4449 * clean up all ras fs nodes, and disable ras */ 4450 goto cleanup; 4451 } else 4452 return r; 4453 } 4454 4455 /* check for errors on warm reset edc persisant supported ASIC */ 4456 amdgpu_persistent_edc_harvesting(adev, ras_block); 4457 4458 /* in resume phase, no need to create ras fs node */ 4459 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4460 return 0; 4461 4462 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4463 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4464 (ras_obj->hw_ops->query_poison_status || 4465 ras_obj->hw_ops->handle_poison_consumption))) { 4466 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4467 if (r) 4468 goto cleanup; 4469 } 4470 4471 if (ras_obj->hw_ops && 4472 (ras_obj->hw_ops->query_ras_error_count || 4473 ras_obj->hw_ops->query_ras_error_status)) { 4474 r = amdgpu_ras_sysfs_create(adev, ras_block); 4475 if (r) 4476 goto interrupt; 4477 4478 /* Those are the cached values at init. 4479 */ 4480 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4481 if (!query_info) 4482 return -ENOMEM; 4483 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4484 4485 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4486 atomic_set(&con->ras_ce_count, ce_count); 4487 atomic_set(&con->ras_ue_count, ue_count); 4488 } 4489 4490 kfree(query_info); 4491 } 4492 4493 return 0; 4494 4495 interrupt: 4496 if (ras_obj->ras_cb) 4497 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4498 cleanup: 4499 amdgpu_ras_feature_enable(adev, ras_block, 0); 4500 return r; 4501 } 4502 4503 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4504 struct ras_common_if *ras_block) 4505 { 4506 return amdgpu_ras_block_late_init(adev, ras_block); 4507 } 4508 4509 /* helper function to remove ras fs node and interrupt handler */ 4510 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4511 struct ras_common_if *ras_block) 4512 { 4513 struct amdgpu_ras_block_object *ras_obj; 4514 if (!ras_block) 4515 return; 4516 4517 amdgpu_ras_sysfs_remove(adev, ras_block); 4518 4519 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4520 if (ras_obj->ras_cb) 4521 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4522 } 4523 4524 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4525 struct ras_common_if *ras_block) 4526 { 4527 return amdgpu_ras_block_late_fini(adev, ras_block); 4528 } 4529 4530 /* do some init work after IP late init as dependence. 4531 * and it runs in resume/gpu reset/booting up cases. 4532 */ 4533 void amdgpu_ras_resume(struct amdgpu_device *adev) 4534 { 4535 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4536 struct ras_manager *obj, *tmp; 4537 4538 if (!adev->ras_enabled || !con) { 4539 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4540 amdgpu_release_ras_context(adev); 4541 4542 return; 4543 } 4544 4545 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4546 /* Set up all other IPs which are not implemented. There is a 4547 * tricky thing that IP's actual ras error type should be 4548 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4549 * ERROR_NONE make sense anyway. 4550 */ 4551 amdgpu_ras_enable_all_features(adev, 1); 4552 4553 /* We enable ras on all hw_supported block, but as boot 4554 * parameter might disable some of them and one or more IP has 4555 * not implemented yet. So we disable them on behalf. 4556 */ 4557 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4558 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4559 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4560 /* there should be no any reference. */ 4561 WARN_ON(alive_obj(obj)); 4562 } 4563 } 4564 } 4565 } 4566 4567 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4568 { 4569 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4570 4571 if (!adev->ras_enabled || !con) 4572 return; 4573 4574 amdgpu_ras_disable_all_features(adev, 0); 4575 /* Make sure all ras objects are disabled. */ 4576 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4577 amdgpu_ras_disable_all_features(adev, 1); 4578 } 4579 4580 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4581 { 4582 struct amdgpu_ras_block_list *node, *tmp; 4583 struct amdgpu_ras_block_object *obj; 4584 int r; 4585 4586 amdgpu_ras_event_mgr_init(adev); 4587 4588 if (amdgpu_ras_aca_is_supported(adev)) { 4589 if (amdgpu_reset_in_recovery(adev)) { 4590 if (amdgpu_aca_is_enabled(adev)) 4591 r = amdgpu_aca_reset(adev); 4592 else 4593 r = amdgpu_mca_reset(adev); 4594 if (r) 4595 return r; 4596 } 4597 4598 if (!amdgpu_sriov_vf(adev)) { 4599 if (amdgpu_aca_is_enabled(adev)) 4600 amdgpu_ras_set_aca_debug_mode(adev, false); 4601 else 4602 amdgpu_ras_set_mca_debug_mode(adev, false); 4603 } 4604 } 4605 4606 /* Guest side doesn't need init ras feature */ 4607 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4608 return 0; 4609 4610 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4611 obj = node->ras_obj; 4612 if (!obj) { 4613 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4614 continue; 4615 } 4616 4617 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4618 continue; 4619 4620 if (obj->ras_late_init) { 4621 r = obj->ras_late_init(adev, &obj->ras_comm); 4622 if (r) { 4623 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4624 obj->ras_comm.name, r); 4625 return r; 4626 } 4627 } else 4628 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4629 } 4630 4631 return 0; 4632 } 4633 4634 /* do some fini work before IP fini as dependence */ 4635 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4636 { 4637 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4638 4639 if (!adev->ras_enabled || !con) 4640 return 0; 4641 4642 4643 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4644 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4645 amdgpu_ras_disable_all_features(adev, 0); 4646 amdgpu_ras_recovery_fini(adev); 4647 return 0; 4648 } 4649 4650 int amdgpu_ras_fini(struct amdgpu_device *adev) 4651 { 4652 struct amdgpu_ras_block_list *ras_node, *tmp; 4653 struct amdgpu_ras_block_object *obj = NULL; 4654 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4655 4656 if (!adev->ras_enabled || !con) 4657 return 0; 4658 4659 amdgpu_ras_critical_region_fini(adev); 4660 mutex_destroy(&con->critical_region_lock); 4661 4662 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4663 if (ras_node->ras_obj) { 4664 obj = ras_node->ras_obj; 4665 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4666 obj->ras_fini) 4667 obj->ras_fini(adev, &obj->ras_comm); 4668 else 4669 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4670 } 4671 4672 /* Clear ras blocks from ras_list and free ras block list node */ 4673 list_del(&ras_node->node); 4674 kfree(ras_node); 4675 } 4676 4677 amdgpu_ras_fs_fini(adev); 4678 amdgpu_ras_interrupt_remove_all(adev); 4679 4680 if (amdgpu_ras_aca_is_supported(adev)) { 4681 if (amdgpu_aca_is_enabled(adev)) 4682 amdgpu_aca_fini(adev); 4683 else 4684 amdgpu_mca_fini(adev); 4685 } 4686 4687 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4688 4689 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4690 amdgpu_ras_disable_all_features(adev, 0); 4691 4692 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4693 4694 amdgpu_ras_set_context(adev, NULL); 4695 kfree(con); 4696 4697 return 0; 4698 } 4699 4700 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4701 { 4702 struct amdgpu_ras *ras; 4703 4704 ras = amdgpu_ras_get_context(adev); 4705 if (!ras) 4706 return false; 4707 4708 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4709 } 4710 4711 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4712 { 4713 struct amdgpu_ras *ras; 4714 4715 ras = amdgpu_ras_get_context(adev); 4716 if (ras) { 4717 if (status) 4718 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4719 else 4720 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4721 } 4722 } 4723 4724 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4725 { 4726 struct amdgpu_ras *ras; 4727 4728 ras = amdgpu_ras_get_context(adev); 4729 if (ras) { 4730 ras->ras_err_state = 0; 4731 ras->gpu_reset_flags = 0; 4732 } 4733 } 4734 4735 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4736 enum amdgpu_ras_block block) 4737 { 4738 struct amdgpu_ras *ras; 4739 4740 ras = amdgpu_ras_get_context(adev); 4741 if (ras) 4742 set_bit(block, &ras->ras_err_state); 4743 } 4744 4745 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4746 { 4747 struct amdgpu_ras *ras; 4748 4749 ras = amdgpu_ras_get_context(adev); 4750 if (ras) { 4751 if (block == AMDGPU_RAS_BLOCK__ANY) 4752 return (ras->ras_err_state != 0); 4753 else 4754 return test_bit(block, &ras->ras_err_state) || 4755 test_bit(AMDGPU_RAS_BLOCK__LAST, 4756 &ras->ras_err_state); 4757 } 4758 4759 return false; 4760 } 4761 4762 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4763 { 4764 struct amdgpu_ras *ras; 4765 4766 ras = amdgpu_ras_get_context(adev); 4767 if (!ras) 4768 return NULL; 4769 4770 return ras->event_mgr; 4771 } 4772 4773 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4774 const void *caller) 4775 { 4776 struct ras_event_manager *event_mgr; 4777 struct ras_event_state *event_state; 4778 int ret = 0; 4779 4780 if (amdgpu_uniras_enabled(adev)) 4781 return 0; 4782 4783 if (type >= RAS_EVENT_TYPE_COUNT) { 4784 ret = -EINVAL; 4785 goto out; 4786 } 4787 4788 event_mgr = __get_ras_event_mgr(adev); 4789 if (!event_mgr) { 4790 ret = -EINVAL; 4791 goto out; 4792 } 4793 4794 event_state = &event_mgr->event_state[type]; 4795 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4796 atomic64_inc(&event_state->count); 4797 4798 out: 4799 if (ret && caller) 4800 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4801 (int)type, caller, ret); 4802 4803 return ret; 4804 } 4805 4806 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4807 { 4808 struct ras_event_manager *event_mgr; 4809 u64 id; 4810 4811 if (type >= RAS_EVENT_TYPE_COUNT) 4812 return RAS_EVENT_INVALID_ID; 4813 4814 switch (type) { 4815 case RAS_EVENT_TYPE_FATAL: 4816 case RAS_EVENT_TYPE_POISON_CREATION: 4817 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4818 event_mgr = __get_ras_event_mgr(adev); 4819 if (!event_mgr) 4820 return RAS_EVENT_INVALID_ID; 4821 4822 id = event_mgr->event_state[type].last_seqno; 4823 break; 4824 case RAS_EVENT_TYPE_INVALID: 4825 default: 4826 id = RAS_EVENT_INVALID_ID; 4827 break; 4828 } 4829 4830 return id; 4831 } 4832 4833 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4834 { 4835 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4836 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4837 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4838 u64 event_id = RAS_EVENT_INVALID_ID; 4839 4840 if (amdgpu_uniras_enabled(adev)) 4841 return 0; 4842 4843 if (!amdgpu_ras_mark_ras_event(adev, type)) 4844 event_id = amdgpu_ras_acquire_event_id(adev, type); 4845 4846 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4847 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4848 4849 amdgpu_ras_set_fed(adev, true); 4850 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4851 amdgpu_ras_reset_gpu(adev); 4852 } 4853 4854 return -EBUSY; 4855 } 4856 4857 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4858 { 4859 if (adev->asic_type == CHIP_VEGA20 && 4860 adev->pm.fw_version <= 0x283400) { 4861 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4862 amdgpu_ras_intr_triggered(); 4863 } 4864 4865 return false; 4866 } 4867 4868 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4869 { 4870 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4871 4872 if (!con) 4873 return; 4874 4875 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4876 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4877 amdgpu_ras_set_context(adev, NULL); 4878 kfree(con); 4879 } 4880 } 4881 4882 #ifdef CONFIG_X86_MCE_AMD 4883 static struct amdgpu_device *find_adev(uint32_t node_id) 4884 { 4885 int i; 4886 struct amdgpu_device *adev = NULL; 4887 4888 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4889 adev = mce_adev_list.devs[i]; 4890 4891 if (adev && adev->gmc.xgmi.connected_to_cpu && 4892 adev->gmc.xgmi.physical_node_id == node_id) 4893 break; 4894 adev = NULL; 4895 } 4896 4897 return adev; 4898 } 4899 4900 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4901 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4902 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4903 #define GPU_ID_OFFSET 8 4904 4905 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4906 unsigned long val, void *data) 4907 { 4908 struct mce *m = (struct mce *)data; 4909 struct amdgpu_device *adev = NULL; 4910 uint32_t gpu_id = 0; 4911 uint32_t umc_inst = 0, ch_inst = 0; 4912 4913 /* 4914 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4915 * and error occurred in DramECC (Extended error code = 0) then only 4916 * process the error, else bail out. 4917 */ 4918 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4919 (XEC(m->status, 0x3f) == 0x0))) 4920 return NOTIFY_DONE; 4921 4922 /* 4923 * If it is correctable error, return. 4924 */ 4925 if (mce_is_correctable(m)) 4926 return NOTIFY_OK; 4927 4928 /* 4929 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4930 */ 4931 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4932 4933 adev = find_adev(gpu_id); 4934 if (!adev) { 4935 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4936 gpu_id); 4937 return NOTIFY_DONE; 4938 } 4939 4940 /* 4941 * If it is uncorrectable error, then find out UMC instance and 4942 * channel index. 4943 */ 4944 umc_inst = GET_UMC_INST(m->ipid); 4945 ch_inst = GET_CHAN_INDEX(m->ipid); 4946 4947 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4948 umc_inst, ch_inst); 4949 4950 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4951 return NOTIFY_OK; 4952 else 4953 return NOTIFY_DONE; 4954 } 4955 4956 static struct notifier_block amdgpu_bad_page_nb = { 4957 .notifier_call = amdgpu_bad_page_notifier, 4958 .priority = MCE_PRIO_UC, 4959 }; 4960 4961 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4962 { 4963 /* 4964 * Add the adev to the mce_adev_list. 4965 * During mode2 reset, amdgpu device is temporarily 4966 * removed from the mgpu_info list which can cause 4967 * page retirement to fail. 4968 * Use this list instead of mgpu_info to find the amdgpu 4969 * device on which the UMC error was reported. 4970 */ 4971 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4972 4973 /* 4974 * Register the x86 notifier only once 4975 * with MCE subsystem. 4976 */ 4977 if (notifier_registered == false) { 4978 mce_register_decode_chain(&amdgpu_bad_page_nb); 4979 notifier_registered = true; 4980 } 4981 } 4982 #endif 4983 4984 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4985 { 4986 if (!adev) 4987 return NULL; 4988 4989 return adev->psp.ras_context.ras; 4990 } 4991 4992 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4993 { 4994 if (!adev) 4995 return -EINVAL; 4996 4997 adev->psp.ras_context.ras = ras_con; 4998 return 0; 4999 } 5000 5001 /* check if ras is supported on block, say, sdma, gfx */ 5002 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 5003 unsigned int block) 5004 { 5005 int ret = 0; 5006 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5007 5008 if (block >= AMDGPU_RAS_BLOCK_COUNT) 5009 return 0; 5010 5011 ret = ras && (adev->ras_enabled & (1 << block)); 5012 5013 /* For the special asic with mem ecc enabled but sram ecc 5014 * not enabled, even if the ras block is not supported on 5015 * .ras_enabled, if the asic supports poison mode and the 5016 * ras block has ras configuration, it can be considered 5017 * that the ras block supports ras function. 5018 */ 5019 if (!ret && 5020 (block == AMDGPU_RAS_BLOCK__GFX || 5021 block == AMDGPU_RAS_BLOCK__SDMA || 5022 block == AMDGPU_RAS_BLOCK__VCN || 5023 block == AMDGPU_RAS_BLOCK__JPEG) && 5024 (amdgpu_ras_mask & (1 << block)) && 5025 amdgpu_ras_is_poison_mode_supported(adev) && 5026 amdgpu_ras_get_ras_block(adev, block, 0)) 5027 ret = 1; 5028 5029 return ret; 5030 } 5031 5032 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5033 { 5034 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5035 5036 /* mode1 is the only selection for RMA status */ 5037 if (amdgpu_ras_is_rma(adev)) { 5038 ras->gpu_reset_flags = 0; 5039 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5040 } 5041 5042 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5043 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5044 int hive_ras_recovery = 0; 5045 5046 if (hive) { 5047 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5048 amdgpu_put_xgmi_hive(hive); 5049 } 5050 /* In the case of multiple GPUs, after a GPU has started 5051 * resetting all GPUs on hive, other GPUs do not need to 5052 * trigger GPU reset again. 5053 */ 5054 if (!hive_ras_recovery) 5055 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5056 else 5057 atomic_set(&ras->in_recovery, 0); 5058 } else { 5059 flush_work(&ras->recovery_work); 5060 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5061 } 5062 5063 return 0; 5064 } 5065 5066 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5067 { 5068 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5069 int ret = 0; 5070 5071 if (con) { 5072 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5073 if (!ret) 5074 con->is_aca_debug_mode = enable; 5075 } 5076 5077 return ret; 5078 } 5079 5080 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5081 { 5082 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5083 int ret = 0; 5084 5085 if (con) { 5086 if (amdgpu_aca_is_enabled(adev)) 5087 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5088 else 5089 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5090 if (!ret) 5091 con->is_aca_debug_mode = enable; 5092 } 5093 5094 return ret; 5095 } 5096 5097 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5098 { 5099 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5100 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5101 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5102 5103 if (!con) 5104 return false; 5105 5106 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5107 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5108 return con->is_aca_debug_mode; 5109 else 5110 return true; 5111 } 5112 5113 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5114 unsigned int *error_query_mode) 5115 { 5116 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5117 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5118 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5119 5120 if (!con) { 5121 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5122 return false; 5123 } 5124 5125 if (amdgpu_sriov_vf(adev)) { 5126 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5127 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5128 *error_query_mode = 5129 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5130 } else { 5131 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5132 } 5133 5134 return true; 5135 } 5136 5137 /* Register each ip ras block into amdgpu ras */ 5138 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5139 struct amdgpu_ras_block_object *ras_block_obj) 5140 { 5141 struct amdgpu_ras_block_list *ras_node; 5142 if (!adev || !ras_block_obj) 5143 return -EINVAL; 5144 5145 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5146 if (!ras_node) 5147 return -ENOMEM; 5148 5149 INIT_LIST_HEAD(&ras_node->node); 5150 ras_node->ras_obj = ras_block_obj; 5151 list_add_tail(&ras_node->node, &adev->ras_list); 5152 5153 return 0; 5154 } 5155 5156 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5157 { 5158 if (!err_type_name) 5159 return; 5160 5161 switch (err_type) { 5162 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5163 sprintf(err_type_name, "correctable"); 5164 break; 5165 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5166 sprintf(err_type_name, "uncorrectable"); 5167 break; 5168 default: 5169 sprintf(err_type_name, "unknown"); 5170 break; 5171 } 5172 } 5173 5174 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5175 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5176 uint32_t instance, 5177 uint32_t *memory_id) 5178 { 5179 uint32_t err_status_lo_data, err_status_lo_offset; 5180 5181 if (!reg_entry) 5182 return false; 5183 5184 err_status_lo_offset = 5185 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5186 reg_entry->seg_lo, reg_entry->reg_lo); 5187 err_status_lo_data = RREG32(err_status_lo_offset); 5188 5189 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5190 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5191 return false; 5192 5193 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5194 5195 return true; 5196 } 5197 5198 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5199 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5200 uint32_t instance, 5201 unsigned long *err_cnt) 5202 { 5203 uint32_t err_status_hi_data, err_status_hi_offset; 5204 5205 if (!reg_entry) 5206 return false; 5207 5208 err_status_hi_offset = 5209 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5210 reg_entry->seg_hi, reg_entry->reg_hi); 5211 err_status_hi_data = RREG32(err_status_hi_offset); 5212 5213 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5214 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5215 /* keep the check here in case we need to refer to the result later */ 5216 dev_dbg(adev->dev, "Invalid err_info field\n"); 5217 5218 /* read err count */ 5219 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5220 5221 return true; 5222 } 5223 5224 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5225 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5226 uint32_t reg_list_size, 5227 const struct amdgpu_ras_memory_id_entry *mem_list, 5228 uint32_t mem_list_size, 5229 uint32_t instance, 5230 uint32_t err_type, 5231 unsigned long *err_count) 5232 { 5233 uint32_t memory_id; 5234 unsigned long err_cnt; 5235 char err_type_name[16]; 5236 uint32_t i, j; 5237 5238 for (i = 0; i < reg_list_size; i++) { 5239 /* query memory_id from err_status_lo */ 5240 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5241 instance, &memory_id)) 5242 continue; 5243 5244 /* query err_cnt from err_status_hi */ 5245 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5246 instance, &err_cnt) || 5247 !err_cnt) 5248 continue; 5249 5250 *err_count += err_cnt; 5251 5252 /* log the errors */ 5253 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5254 if (!mem_list) { 5255 /* memory_list is not supported */ 5256 dev_info(adev->dev, 5257 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5258 err_cnt, err_type_name, 5259 reg_list[i].block_name, 5260 instance, memory_id); 5261 } else { 5262 for (j = 0; j < mem_list_size; j++) { 5263 if (memory_id == mem_list[j].memory_id) { 5264 dev_info(adev->dev, 5265 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5266 err_cnt, err_type_name, 5267 reg_list[i].block_name, 5268 instance, mem_list[j].name); 5269 break; 5270 } 5271 } 5272 } 5273 } 5274 } 5275 5276 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5277 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5278 uint32_t reg_list_size, 5279 uint32_t instance) 5280 { 5281 uint32_t err_status_lo_offset, err_status_hi_offset; 5282 uint32_t i; 5283 5284 for (i = 0; i < reg_list_size; i++) { 5285 err_status_lo_offset = 5286 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5287 reg_list[i].seg_lo, reg_list[i].reg_lo); 5288 err_status_hi_offset = 5289 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5290 reg_list[i].seg_hi, reg_list[i].reg_hi); 5291 WREG32(err_status_lo_offset, 0); 5292 WREG32(err_status_hi_offset, 0); 5293 } 5294 } 5295 5296 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5297 { 5298 memset(err_data, 0, sizeof(*err_data)); 5299 5300 INIT_LIST_HEAD(&err_data->err_node_list); 5301 5302 return 0; 5303 } 5304 5305 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5306 { 5307 if (!err_node) 5308 return; 5309 5310 list_del(&err_node->node); 5311 kvfree(err_node); 5312 } 5313 5314 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5315 { 5316 struct ras_err_node *err_node, *tmp; 5317 5318 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5319 amdgpu_ras_error_node_release(err_node); 5320 } 5321 5322 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5323 struct amdgpu_smuio_mcm_config_info *mcm_info) 5324 { 5325 struct ras_err_node *err_node; 5326 struct amdgpu_smuio_mcm_config_info *ref_id; 5327 5328 if (!err_data || !mcm_info) 5329 return NULL; 5330 5331 for_each_ras_error(err_node, err_data) { 5332 ref_id = &err_node->err_info.mcm_info; 5333 5334 if (mcm_info->socket_id == ref_id->socket_id && 5335 mcm_info->die_id == ref_id->die_id) 5336 return err_node; 5337 } 5338 5339 return NULL; 5340 } 5341 5342 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5343 { 5344 struct ras_err_node *err_node; 5345 5346 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5347 if (!err_node) 5348 return NULL; 5349 5350 INIT_LIST_HEAD(&err_node->node); 5351 5352 return err_node; 5353 } 5354 5355 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5356 { 5357 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5358 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5359 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5360 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5361 5362 if (unlikely(infoa->socket_id != infob->socket_id)) 5363 return infoa->socket_id - infob->socket_id; 5364 else 5365 return infoa->die_id - infob->die_id; 5366 5367 return 0; 5368 } 5369 5370 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5371 struct amdgpu_smuio_mcm_config_info *mcm_info) 5372 { 5373 struct ras_err_node *err_node; 5374 5375 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5376 if (err_node) 5377 return &err_node->err_info; 5378 5379 err_node = amdgpu_ras_error_node_new(); 5380 if (!err_node) 5381 return NULL; 5382 5383 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5384 5385 err_data->err_list_count++; 5386 list_add_tail(&err_node->node, &err_data->err_node_list); 5387 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5388 5389 return &err_node->err_info; 5390 } 5391 5392 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5393 struct amdgpu_smuio_mcm_config_info *mcm_info, 5394 u64 count) 5395 { 5396 struct ras_err_info *err_info; 5397 5398 if (!err_data || !mcm_info) 5399 return -EINVAL; 5400 5401 if (!count) 5402 return 0; 5403 5404 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5405 if (!err_info) 5406 return -EINVAL; 5407 5408 err_info->ue_count += count; 5409 err_data->ue_count += count; 5410 5411 return 0; 5412 } 5413 5414 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5415 struct amdgpu_smuio_mcm_config_info *mcm_info, 5416 u64 count) 5417 { 5418 struct ras_err_info *err_info; 5419 5420 if (!err_data || !mcm_info) 5421 return -EINVAL; 5422 5423 if (!count) 5424 return 0; 5425 5426 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5427 if (!err_info) 5428 return -EINVAL; 5429 5430 err_info->ce_count += count; 5431 err_data->ce_count += count; 5432 5433 return 0; 5434 } 5435 5436 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5437 struct amdgpu_smuio_mcm_config_info *mcm_info, 5438 u64 count) 5439 { 5440 struct ras_err_info *err_info; 5441 5442 if (!err_data || !mcm_info) 5443 return -EINVAL; 5444 5445 if (!count) 5446 return 0; 5447 5448 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5449 if (!err_info) 5450 return -EINVAL; 5451 5452 err_info->de_count += count; 5453 err_data->de_count += count; 5454 5455 return 0; 5456 } 5457 5458 #define mmMP0_SMN_C2PMSG_92 0x1609C 5459 #define mmMP0_SMN_C2PMSG_126 0x160BE 5460 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5461 u32 instance) 5462 { 5463 u32 socket_id, aid_id, hbm_id; 5464 u32 fw_status; 5465 u32 boot_error; 5466 u64 reg_addr; 5467 5468 /* The pattern for smn addressing in other SOC could be different from 5469 * the one for aqua_vanjaram. We should revisit the code if the pattern 5470 * is changed. In such case, replace the aqua_vanjaram implementation 5471 * with more common helper */ 5472 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5473 aqua_vanjaram_encode_ext_smn_addressing(instance); 5474 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5475 5476 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5477 aqua_vanjaram_encode_ext_smn_addressing(instance); 5478 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5479 5480 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5481 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5482 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5483 5484 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5485 dev_info(adev->dev, 5486 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5487 socket_id, aid_id, hbm_id, fw_status); 5488 5489 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5490 dev_info(adev->dev, 5491 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5492 socket_id, aid_id, fw_status); 5493 5494 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5495 dev_info(adev->dev, 5496 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5497 socket_id, aid_id, fw_status); 5498 5499 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5500 dev_info(adev->dev, 5501 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5502 socket_id, aid_id, fw_status); 5503 5504 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5505 dev_info(adev->dev, 5506 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5507 socket_id, aid_id, fw_status); 5508 5509 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5510 dev_info(adev->dev, 5511 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5512 socket_id, aid_id, fw_status); 5513 5514 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5515 dev_info(adev->dev, 5516 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5517 socket_id, aid_id, hbm_id, fw_status); 5518 5519 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5520 dev_info(adev->dev, 5521 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5522 socket_id, aid_id, hbm_id, fw_status); 5523 5524 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5525 dev_info(adev->dev, 5526 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5527 socket_id, aid_id, fw_status); 5528 5529 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5530 dev_info(adev->dev, 5531 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5532 socket_id, aid_id, fw_status); 5533 } 5534 5535 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5536 u32 instance) 5537 { 5538 u64 reg_addr; 5539 u32 reg_data; 5540 int retry_loop; 5541 5542 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5543 aqua_vanjaram_encode_ext_smn_addressing(instance); 5544 5545 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5546 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5547 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5548 return false; 5549 else 5550 msleep(1); 5551 } 5552 5553 return true; 5554 } 5555 5556 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5557 { 5558 u32 i; 5559 5560 for (i = 0; i < num_instances; i++) { 5561 if (amdgpu_ras_boot_error_detected(adev, i)) 5562 amdgpu_ras_boot_time_error_reporting(adev, i); 5563 } 5564 } 5565 5566 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5567 { 5568 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5569 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5570 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5571 int ret = 0; 5572 5573 if (amdgpu_ras_check_critical_address(adev, start)) 5574 return 0; 5575 5576 mutex_lock(&con->page_rsv_lock); 5577 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5578 if (ret == -ENOENT) 5579 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5580 mutex_unlock(&con->page_rsv_lock); 5581 5582 return ret; 5583 } 5584 5585 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5586 const char *fmt, ...) 5587 { 5588 struct va_format vaf; 5589 va_list args; 5590 5591 va_start(args, fmt); 5592 vaf.fmt = fmt; 5593 vaf.va = &args; 5594 5595 if (RAS_EVENT_ID_IS_VALID(event_id)) 5596 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5597 else 5598 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5599 5600 va_end(args); 5601 } 5602 5603 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5604 { 5605 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5606 5607 if (amdgpu_uniras_enabled(adev)) 5608 return amdgpu_ras_mgr_is_rma(adev); 5609 5610 if (!con) 5611 return false; 5612 5613 return con->is_rma; 5614 } 5615 5616 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5617 struct amdgpu_bo *bo) 5618 { 5619 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5620 struct amdgpu_vram_mgr_resource *vres; 5621 struct ras_critical_region *region; 5622 struct drm_buddy_block *block; 5623 int ret = 0; 5624 5625 if (!bo || !bo->tbo.resource) 5626 return -EINVAL; 5627 5628 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5629 5630 mutex_lock(&con->critical_region_lock); 5631 5632 /* Check if the bo had been recorded */ 5633 list_for_each_entry(region, &con->critical_region_head, node) 5634 if (region->bo == bo) 5635 goto out; 5636 5637 /* Record new critical amdgpu bo */ 5638 list_for_each_entry(block, &vres->blocks, link) { 5639 region = kzalloc(sizeof(*region), GFP_KERNEL); 5640 if (!region) { 5641 ret = -ENOMEM; 5642 goto out; 5643 } 5644 region->bo = bo; 5645 region->start = amdgpu_vram_mgr_block_start(block); 5646 region->size = amdgpu_vram_mgr_block_size(block); 5647 list_add_tail(®ion->node, &con->critical_region_head); 5648 } 5649 5650 out: 5651 mutex_unlock(&con->critical_region_lock); 5652 5653 return ret; 5654 } 5655 5656 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5657 { 5658 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5659 } 5660 5661 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5662 { 5663 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5664 struct ras_critical_region *region, *tmp; 5665 5666 mutex_lock(&con->critical_region_lock); 5667 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5668 list_del(®ion->node); 5669 kfree(region); 5670 } 5671 mutex_unlock(&con->critical_region_lock); 5672 } 5673 5674 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5675 { 5676 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5677 struct ras_critical_region *region; 5678 bool ret = false; 5679 5680 mutex_lock(&con->critical_region_lock); 5681 list_for_each_entry(region, &con->critical_region_head, node) { 5682 if ((region->start <= addr) && 5683 (addr < (region->start + region->size))) { 5684 ret = true; 5685 break; 5686 } 5687 } 5688 mutex_unlock(&con->critical_region_lock); 5689 5690 return ret; 5691 } 5692 5693 void amdgpu_ras_pre_reset(struct amdgpu_device *adev, 5694 struct list_head *device_list) 5695 { 5696 struct amdgpu_device *tmp_adev = NULL; 5697 5698 list_for_each_entry(tmp_adev, device_list, reset_list) { 5699 if (amdgpu_uniras_enabled(tmp_adev)) 5700 amdgpu_ras_mgr_pre_reset(tmp_adev); 5701 } 5702 } 5703 5704 void amdgpu_ras_post_reset(struct amdgpu_device *adev, 5705 struct list_head *device_list) 5706 { 5707 struct amdgpu_device *tmp_adev = NULL; 5708 5709 list_for_each_entry(tmp_adev, device_list, reset_list) { 5710 if (amdgpu_uniras_enabled(tmp_adev)) 5711 amdgpu_ras_mgr_post_reset(tmp_adev); 5712 } 5713 } 5714