1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 616 617 /** 618 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 619 * 620 * Some boards contain an EEPROM which is used to persistently store a list of 621 * bad pages which experiences ECC errors in vram. This interface provides 622 * a way to reset the EEPROM, e.g., after testing error injection. 623 * 624 * Usage: 625 * 626 * .. code-block:: bash 627 * 628 * echo 1 > ../ras/ras_eeprom_reset 629 * 630 * will reset EEPROM table to 0 entries. 631 * 632 */ 633 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 634 const char __user *buf, 635 size_t size, loff_t *pos) 636 { 637 struct amdgpu_device *adev = 638 (struct amdgpu_device *)file_inode(f)->i_private; 639 int ret; 640 641 if (amdgpu_uniras_enabled(adev)) { 642 ret = amdgpu_uniras_clear_badpages_info(adev); 643 return ret ? ret : size; 644 } 645 646 ret = amdgpu_ras_eeprom_reset_table( 647 &(amdgpu_ras_get_context(adev)->eeprom_control)); 648 649 if (!ret) { 650 /* Something was written to EEPROM. 651 */ 652 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 653 return size; 654 } else { 655 return ret; 656 } 657 } 658 659 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_ctrl_write, 663 .llseek = default_llseek 664 }; 665 666 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 667 .owner = THIS_MODULE, 668 .read = NULL, 669 .write = amdgpu_ras_debugfs_eeprom_write, 670 .llseek = default_llseek 671 }; 672 673 /** 674 * DOC: AMDGPU RAS sysfs Error Count Interface 675 * 676 * It allows the user to read the error count for each IP block on the gpu through 677 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 678 * 679 * It outputs the multiple lines which report the uncorrected (ue) and corrected 680 * (ce) error counts. 681 * 682 * The format of one line is below, 683 * 684 * [ce|ue]: count 685 * 686 * Example: 687 * 688 * .. code-block:: bash 689 * 690 * ue: 0 691 * ce: 1 692 * 693 */ 694 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 695 struct device_attribute *attr, char *buf) 696 { 697 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 698 struct ras_query_if info = { 699 .head = obj->head, 700 }; 701 702 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 703 return sysfs_emit(buf, "Query currently inaccessible\n"); 704 705 if (amdgpu_ras_query_error_status(obj->adev, &info)) 706 return -EINVAL; 707 708 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 709 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 710 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 711 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 712 } 713 714 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 715 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 716 "ce", info.ce_count, "de", info.de_count); 717 else 718 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 719 "ce", info.ce_count); 720 } 721 722 /* obj begin */ 723 724 #define get_obj(obj) do { (obj)->use++; } while (0) 725 #define alive_obj(obj) ((obj)->use) 726 727 static inline void put_obj(struct ras_manager *obj) 728 { 729 if (obj && (--obj->use == 0)) { 730 list_del(&obj->node); 731 amdgpu_ras_error_data_fini(&obj->err_data); 732 } 733 734 if (obj && (obj->use < 0)) 735 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 736 } 737 738 /* make one obj and return it. */ 739 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 740 struct ras_common_if *head) 741 { 742 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 743 struct ras_manager *obj; 744 745 if (!adev->ras_enabled || !con) 746 return NULL; 747 748 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 749 return NULL; 750 751 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 752 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 753 return NULL; 754 755 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 756 } else 757 obj = &con->objs[head->block]; 758 759 /* already exist. return obj? */ 760 if (alive_obj(obj)) 761 return NULL; 762 763 if (amdgpu_ras_error_data_init(&obj->err_data)) 764 return NULL; 765 766 obj->head = *head; 767 obj->adev = adev; 768 list_add(&obj->node, &con->head); 769 get_obj(obj); 770 771 return obj; 772 } 773 774 /* return an obj equal to head, or the first when head is NULL */ 775 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 776 struct ras_common_if *head) 777 { 778 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 779 struct ras_manager *obj; 780 int i; 781 782 if (!adev->ras_enabled || !con) 783 return NULL; 784 785 if (head) { 786 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 787 return NULL; 788 789 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 790 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 791 return NULL; 792 793 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 794 } else 795 obj = &con->objs[head->block]; 796 797 if (alive_obj(obj)) 798 return obj; 799 } else { 800 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 801 obj = &con->objs[i]; 802 if (alive_obj(obj)) 803 return obj; 804 } 805 } 806 807 return NULL; 808 } 809 /* obj end */ 810 811 /* feature ctl begin */ 812 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 813 struct ras_common_if *head) 814 { 815 return adev->ras_hw_enabled & BIT(head->block); 816 } 817 818 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 819 struct ras_common_if *head) 820 { 821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 822 823 return con->features & BIT(head->block); 824 } 825 826 /* 827 * if obj is not created, then create one. 828 * set feature enable flag. 829 */ 830 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 831 struct ras_common_if *head, int enable) 832 { 833 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 834 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 835 836 /* If hardware does not support ras, then do not create obj. 837 * But if hardware support ras, we can create the obj. 838 * Ras framework checks con->hw_supported to see if it need do 839 * corresponding initialization. 840 * IP checks con->support to see if it need disable ras. 841 */ 842 if (!amdgpu_ras_is_feature_allowed(adev, head)) 843 return 0; 844 845 if (enable) { 846 if (!obj) { 847 obj = amdgpu_ras_create_obj(adev, head); 848 if (!obj) 849 return -EINVAL; 850 } else { 851 /* In case we create obj somewhere else */ 852 get_obj(obj); 853 } 854 con->features |= BIT(head->block); 855 } else { 856 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 857 con->features &= ~BIT(head->block); 858 put_obj(obj); 859 } 860 } 861 862 return 0; 863 } 864 865 /* wrapper of psp_ras_enable_features */ 866 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 867 struct ras_common_if *head, bool enable) 868 { 869 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 870 union ta_ras_cmd_input *info; 871 int ret; 872 873 if (!con) 874 return -EINVAL; 875 876 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 877 /* For gfx ip, regardless of feature support status, */ 878 /* Force issue enable or disable ras feature commands */ 879 if (head->block != AMDGPU_RAS_BLOCK__GFX && 880 !amdgpu_ras_is_feature_allowed(adev, head)) 881 return 0; 882 883 /* Only enable gfx ras feature from host side */ 884 if (head->block == AMDGPU_RAS_BLOCK__GFX && 885 !amdgpu_sriov_vf(adev) && 886 !amdgpu_ras_intr_triggered()) { 887 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 888 if (!info) 889 return -ENOMEM; 890 891 if (!enable) { 892 info->disable_features = (struct ta_ras_disable_features_input) { 893 .block_id = amdgpu_ras_block_to_ta(head->block), 894 .error_type = amdgpu_ras_error_to_ta(head->type), 895 }; 896 } else { 897 info->enable_features = (struct ta_ras_enable_features_input) { 898 .block_id = amdgpu_ras_block_to_ta(head->block), 899 .error_type = amdgpu_ras_error_to_ta(head->type), 900 }; 901 } 902 903 ret = psp_ras_enable_features(&adev->psp, info, enable); 904 if (ret) { 905 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 906 enable ? "enable":"disable", 907 get_ras_block_str(head), 908 amdgpu_ras_is_poison_mode_supported(adev), ret); 909 kfree(info); 910 return ret; 911 } 912 913 kfree(info); 914 } 915 916 /* setup the obj */ 917 __amdgpu_ras_feature_enable(adev, head, enable); 918 919 return 0; 920 } 921 922 /* Only used in device probe stage and called only once. */ 923 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 924 struct ras_common_if *head, bool enable) 925 { 926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 927 int ret; 928 929 if (!con) 930 return -EINVAL; 931 932 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 933 if (enable) { 934 /* There is no harm to issue a ras TA cmd regardless of 935 * the currecnt ras state. 936 * If current state == target state, it will do nothing 937 * But sometimes it requests driver to reset and repost 938 * with error code -EAGAIN. 939 */ 940 ret = amdgpu_ras_feature_enable(adev, head, 1); 941 /* With old ras TA, we might fail to enable ras. 942 * Log it and just setup the object. 943 * TODO need remove this WA in the future. 944 */ 945 if (ret == -EINVAL) { 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (!ret) 948 dev_info(adev->dev, 949 "RAS INFO: %s setup object\n", 950 get_ras_block_str(head)); 951 } 952 } else { 953 /* setup the object then issue a ras TA disable cmd.*/ 954 ret = __amdgpu_ras_feature_enable(adev, head, 1); 955 if (ret) 956 return ret; 957 958 /* gfx block ras disable cmd must send to ras-ta */ 959 if (head->block == AMDGPU_RAS_BLOCK__GFX) 960 con->features |= BIT(head->block); 961 962 ret = amdgpu_ras_feature_enable(adev, head, 0); 963 964 /* clean gfx block ras features flag */ 965 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 966 con->features &= ~BIT(head->block); 967 } 968 } else 969 ret = amdgpu_ras_feature_enable(adev, head, enable); 970 971 return ret; 972 } 973 974 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 975 bool bypass) 976 { 977 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 978 struct ras_manager *obj, *tmp; 979 980 list_for_each_entry_safe(obj, tmp, &con->head, node) { 981 /* bypass psp. 982 * aka just release the obj and corresponding flags 983 */ 984 if (bypass) { 985 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 986 break; 987 } else { 988 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 989 break; 990 } 991 } 992 993 return con->features; 994 } 995 996 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 997 bool bypass) 998 { 999 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1000 int i; 1001 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1002 1003 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1004 struct ras_common_if head = { 1005 .block = i, 1006 .type = default_ras_type, 1007 .sub_block_index = 0, 1008 }; 1009 1010 if (i == AMDGPU_RAS_BLOCK__MCA) 1011 continue; 1012 1013 if (bypass) { 1014 /* 1015 * bypass psp. vbios enable ras for us. 1016 * so just create the obj 1017 */ 1018 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1019 break; 1020 } else { 1021 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1022 break; 1023 } 1024 } 1025 1026 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1027 struct ras_common_if head = { 1028 .block = AMDGPU_RAS_BLOCK__MCA, 1029 .type = default_ras_type, 1030 .sub_block_index = i, 1031 }; 1032 1033 if (bypass) { 1034 /* 1035 * bypass psp. vbios enable ras for us. 1036 * so just create the obj 1037 */ 1038 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1039 break; 1040 } else { 1041 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1042 break; 1043 } 1044 } 1045 1046 return con->features; 1047 } 1048 /* feature ctl end */ 1049 1050 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1051 enum amdgpu_ras_block block) 1052 { 1053 if (!block_obj) 1054 return -EINVAL; 1055 1056 if (block_obj->ras_comm.block == block) 1057 return 0; 1058 1059 return -EINVAL; 1060 } 1061 1062 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1063 enum amdgpu_ras_block block, uint32_t sub_block_index) 1064 { 1065 struct amdgpu_ras_block_list *node, *tmp; 1066 struct amdgpu_ras_block_object *obj; 1067 1068 if (block >= AMDGPU_RAS_BLOCK__LAST) 1069 return NULL; 1070 1071 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1072 if (!node->ras_obj) { 1073 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1074 continue; 1075 } 1076 1077 obj = node->ras_obj; 1078 if (obj->ras_block_match) { 1079 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1080 return obj; 1081 } else { 1082 if (amdgpu_ras_block_match_default(obj, block) == 0) 1083 return obj; 1084 } 1085 } 1086 1087 return NULL; 1088 } 1089 1090 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1091 { 1092 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1093 int ret = 0; 1094 1095 /* 1096 * choosing right query method according to 1097 * whether smu support query error information 1098 */ 1099 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1100 if (ret == -EOPNOTSUPP) { 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1104 1105 /* umc query_ras_error_address is also responsible for clearing 1106 * error status 1107 */ 1108 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1109 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1110 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1111 } else if (!ret) { 1112 if (adev->umc.ras && 1113 adev->umc.ras->ecc_info_query_ras_error_count) 1114 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1115 1116 if (adev->umc.ras && 1117 adev->umc.ras->ecc_info_query_ras_error_address) 1118 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1119 } 1120 } 1121 1122 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1123 struct ras_manager *ras_mgr, 1124 struct ras_err_data *err_data, 1125 struct ras_query_context *qctx, 1126 const char *blk_name, 1127 bool is_ue, 1128 bool is_de) 1129 { 1130 struct amdgpu_smuio_mcm_config_info *mcm_info; 1131 struct ras_err_node *err_node; 1132 struct ras_err_info *err_info; 1133 u64 event_id = qctx->evid.event_id; 1134 1135 if (is_ue) { 1136 for_each_ras_error(err_node, err_data) { 1137 err_info = &err_node->err_info; 1138 mcm_info = &err_info->mcm_info; 1139 if (err_info->ue_count) { 1140 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1141 "%lld new uncorrectable hardware errors detected in %s block\n", 1142 mcm_info->socket_id, 1143 mcm_info->die_id, 1144 err_info->ue_count, 1145 blk_name); 1146 } 1147 } 1148 1149 for_each_ras_error(err_node, &ras_mgr->err_data) { 1150 err_info = &err_node->err_info; 1151 mcm_info = &err_info->mcm_info; 1152 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1153 "%lld uncorrectable hardware errors detected in total in %s block\n", 1154 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1155 } 1156 1157 } else { 1158 if (is_de) { 1159 for_each_ras_error(err_node, err_data) { 1160 err_info = &err_node->err_info; 1161 mcm_info = &err_info->mcm_info; 1162 if (err_info->de_count) { 1163 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1164 "%lld new deferred hardware errors detected in %s block\n", 1165 mcm_info->socket_id, 1166 mcm_info->die_id, 1167 err_info->de_count, 1168 blk_name); 1169 } 1170 } 1171 1172 for_each_ras_error(err_node, &ras_mgr->err_data) { 1173 err_info = &err_node->err_info; 1174 mcm_info = &err_info->mcm_info; 1175 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1176 "%lld deferred hardware errors detected in total in %s block\n", 1177 mcm_info->socket_id, mcm_info->die_id, 1178 err_info->de_count, blk_name); 1179 } 1180 } else { 1181 if (adev->debug_disable_ce_logs) 1182 return; 1183 1184 for_each_ras_error(err_node, err_data) { 1185 err_info = &err_node->err_info; 1186 mcm_info = &err_info->mcm_info; 1187 if (err_info->ce_count) { 1188 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1189 "%lld new correctable hardware errors detected in %s block\n", 1190 mcm_info->socket_id, 1191 mcm_info->die_id, 1192 err_info->ce_count, 1193 blk_name); 1194 } 1195 } 1196 1197 for_each_ras_error(err_node, &ras_mgr->err_data) { 1198 err_info = &err_node->err_info; 1199 mcm_info = &err_info->mcm_info; 1200 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1201 "%lld correctable hardware errors detected in total in %s block\n", 1202 mcm_info->socket_id, mcm_info->die_id, 1203 err_info->ce_count, blk_name); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline bool err_data_has_source_info(struct ras_err_data *data) 1210 { 1211 return !list_empty(&data->err_node_list); 1212 } 1213 1214 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1215 struct ras_query_if *query_if, 1216 struct ras_err_data *err_data, 1217 struct ras_query_context *qctx) 1218 { 1219 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1220 const char *blk_name = get_ras_block_str(&query_if->head); 1221 u64 event_id = qctx->evid.event_id; 1222 1223 if (err_data->ce_count) { 1224 if (err_data_has_source_info(err_data)) { 1225 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1226 blk_name, false, false); 1227 } else if (!adev->aid_mask && 1228 adev->smuio.funcs && 1229 adev->smuio.funcs->get_socket_id && 1230 adev->smuio.funcs->get_die_id) { 1231 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1232 "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 adev->smuio.funcs->get_socket_id(adev), 1235 adev->smuio.funcs->get_die_id(adev), 1236 ras_mgr->err_data.ce_count, 1237 blk_name); 1238 } else { 1239 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1240 "detected in %s block\n", 1241 ras_mgr->err_data.ce_count, 1242 blk_name); 1243 } 1244 } 1245 1246 if (err_data->ue_count) { 1247 if (err_data_has_source_info(err_data)) { 1248 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1249 blk_name, true, false); 1250 } else if (!adev->aid_mask && 1251 adev->smuio.funcs && 1252 adev->smuio.funcs->get_socket_id && 1253 adev->smuio.funcs->get_die_id) { 1254 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1255 "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 adev->smuio.funcs->get_socket_id(adev), 1258 adev->smuio.funcs->get_die_id(adev), 1259 ras_mgr->err_data.ue_count, 1260 blk_name); 1261 } else { 1262 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1263 "detected in %s block\n", 1264 ras_mgr->err_data.ue_count, 1265 blk_name); 1266 } 1267 } 1268 1269 if (err_data->de_count) { 1270 if (err_data_has_source_info(err_data)) { 1271 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1272 blk_name, false, true); 1273 } else if (!adev->aid_mask && 1274 adev->smuio.funcs && 1275 adev->smuio.funcs->get_socket_id && 1276 adev->smuio.funcs->get_die_id) { 1277 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1278 "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 adev->smuio.funcs->get_socket_id(adev), 1281 adev->smuio.funcs->get_die_id(adev), 1282 ras_mgr->err_data.de_count, 1283 blk_name); 1284 } else { 1285 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1286 "detected in %s block\n", 1287 ras_mgr->err_data.de_count, 1288 blk_name); 1289 } 1290 } 1291 } 1292 1293 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1294 struct ras_query_if *query_if, 1295 struct ras_err_data *err_data, 1296 struct ras_query_context *qctx) 1297 { 1298 unsigned long new_ue, new_ce, new_de; 1299 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1300 const char *blk_name = get_ras_block_str(&query_if->head); 1301 u64 event_id = qctx->evid.event_id; 1302 1303 new_ce = err_data->ce_count - obj->err_data.ce_count; 1304 new_ue = err_data->ue_count - obj->err_data.ue_count; 1305 new_de = err_data->de_count - obj->err_data.de_count; 1306 1307 if (new_ce) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1309 "detected in %s block\n", 1310 new_ce, 1311 blk_name); 1312 } 1313 1314 if (new_ue) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1316 "detected in %s block\n", 1317 new_ue, 1318 blk_name); 1319 } 1320 1321 if (new_de) { 1322 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1323 "detected in %s block\n", 1324 new_de, 1325 blk_name); 1326 } 1327 } 1328 1329 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1330 { 1331 struct ras_err_node *err_node; 1332 struct ras_err_info *err_info; 1333 1334 if (err_data_has_source_info(err_data)) { 1335 for_each_ras_error(err_node, err_data) { 1336 err_info = &err_node->err_info; 1337 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1338 &err_info->mcm_info, err_info->de_count); 1339 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->ce_count); 1341 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ue_count); 1343 } 1344 } else { 1345 /* for legacy asic path which doesn't has error source info */ 1346 obj->err_data.ue_count += err_data->ue_count; 1347 obj->err_data.ce_count += err_data->ce_count; 1348 obj->err_data.de_count += err_data->de_count; 1349 } 1350 } 1351 1352 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1353 struct ras_err_data *err_data) 1354 { 1355 /* Host reports absolute counts */ 1356 obj->err_data.ue_count = err_data->ue_count; 1357 obj->err_data.ce_count = err_data->ce_count; 1358 obj->err_data.de_count = err_data->de_count; 1359 } 1360 1361 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1362 { 1363 struct ras_common_if head; 1364 1365 memset(&head, 0, sizeof(head)); 1366 head.block = blk; 1367 1368 return amdgpu_ras_find_obj(adev, &head); 1369 } 1370 1371 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1372 const struct aca_info *aca_info, void *data) 1373 { 1374 struct ras_manager *obj; 1375 1376 /* in resume phase, no need to create aca fs node */ 1377 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1378 return 0; 1379 1380 obj = get_ras_manager(adev, blk); 1381 if (!obj) 1382 return -EINVAL; 1383 1384 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1385 } 1386 1387 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1388 { 1389 struct ras_manager *obj; 1390 1391 obj = get_ras_manager(adev, blk); 1392 if (!obj) 1393 return -EINVAL; 1394 1395 amdgpu_aca_remove_handle(&obj->aca_handle); 1396 1397 return 0; 1398 } 1399 1400 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1401 enum aca_error_type type, struct ras_err_data *err_data, 1402 struct ras_query_context *qctx) 1403 { 1404 struct ras_manager *obj; 1405 1406 obj = get_ras_manager(adev, blk); 1407 if (!obj) 1408 return -EINVAL; 1409 1410 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1411 } 1412 1413 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1414 struct aca_handle *handle, char *buf, void *data) 1415 { 1416 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1417 struct ras_query_if info = { 1418 .head = obj->head, 1419 }; 1420 1421 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1422 return sysfs_emit(buf, "Query currently inaccessible\n"); 1423 1424 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1425 return -EINVAL; 1426 1427 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1428 "ce", info.ce_count, "de", info.de_count); 1429 } 1430 1431 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1432 struct ras_query_if *info, 1433 struct ras_err_data *err_data, 1434 struct ras_query_context *qctx, 1435 unsigned int error_query_mode) 1436 { 1437 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1438 struct amdgpu_ras_block_object *block_obj = NULL; 1439 int ret; 1440 1441 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1442 return -EINVAL; 1443 1444 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1445 return -EINVAL; 1446 1447 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1448 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1449 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1450 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1451 amdgpu_ras_get_ecc_info(adev, err_data); 1452 } else { 1453 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1454 if (!block_obj || !block_obj->hw_ops) { 1455 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1456 get_ras_block_str(&info->head)); 1457 return -EINVAL; 1458 } 1459 1460 if (block_obj->hw_ops->query_ras_error_count) 1461 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1462 1463 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1464 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1465 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1466 if (block_obj->hw_ops->query_ras_error_status) 1467 block_obj->hw_ops->query_ras_error_status(adev); 1468 } 1469 } 1470 } else { 1471 if (amdgpu_aca_is_enabled(adev)) { 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 1476 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1477 if (ret) 1478 return ret; 1479 1480 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1481 if (ret) 1482 return ret; 1483 } else { 1484 /* FIXME: add code to check return value later */ 1485 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1486 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1487 } 1488 } 1489 1490 return 0; 1491 } 1492 1493 /* query/inject/cure begin */ 1494 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1495 struct ras_query_if *info, 1496 enum ras_event_type type) 1497 { 1498 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1499 struct ras_err_data err_data; 1500 struct ras_query_context qctx; 1501 unsigned int error_query_mode; 1502 int ret; 1503 1504 if (!obj) 1505 return -EINVAL; 1506 1507 ret = amdgpu_ras_error_data_init(&err_data); 1508 if (ret) 1509 return ret; 1510 1511 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1512 return -EINVAL; 1513 1514 memset(&qctx, 0, sizeof(qctx)); 1515 qctx.evid.type = type; 1516 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1517 1518 if (!down_read_trylock(&adev->reset_domain->sem)) { 1519 ret = -EIO; 1520 goto out_fini_err_data; 1521 } 1522 1523 ret = amdgpu_ras_query_error_status_helper(adev, info, 1524 &err_data, 1525 &qctx, 1526 error_query_mode); 1527 up_read(&adev->reset_domain->sem); 1528 if (ret) 1529 goto out_fini_err_data; 1530 1531 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1532 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1533 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1534 } else { 1535 /* Host provides absolute error counts. First generate the report 1536 * using the previous VF internal count against new host count. 1537 * Then Update VF internal count. 1538 */ 1539 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1540 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1541 } 1542 1543 info->ue_count = obj->err_data.ue_count; 1544 info->ce_count = obj->err_data.ce_count; 1545 info->de_count = obj->err_data.de_count; 1546 1547 out_fini_err_data: 1548 amdgpu_ras_error_data_fini(&err_data); 1549 1550 return ret; 1551 } 1552 1553 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1554 { 1555 struct ras_cmd_dev_handle req = {0}; 1556 int ret; 1557 1558 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1559 &req, sizeof(req), NULL, 0); 1560 if (ret) { 1561 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1562 return ret; 1563 } 1564 1565 return 0; 1566 } 1567 1568 static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1569 struct ras_query_if *info) 1570 { 1571 struct ras_cmd_block_ecc_info_req req = {0}; 1572 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1573 int ret; 1574 1575 if (!info) 1576 return -EINVAL; 1577 1578 req.block_id = info->head.block; 1579 req.subblock_id = info->head.sub_block_index; 1580 1581 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1582 &req, sizeof(req), &rsp, sizeof(rsp)); 1583 if (!ret) { 1584 info->ce_count = rsp.ce_count; 1585 info->ue_count = rsp.ue_count; 1586 info->de_count = rsp.de_count; 1587 } 1588 1589 return ret; 1590 } 1591 1592 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1593 { 1594 if (amdgpu_uniras_enabled(adev)) 1595 return amdgpu_uniras_query_block_ecc(adev, info); 1596 else 1597 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1598 } 1599 1600 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1601 enum amdgpu_ras_block block) 1602 { 1603 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1604 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1605 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1606 1607 if (!block_obj || !block_obj->hw_ops) { 1608 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1609 ras_block_str(block)); 1610 return -EOPNOTSUPP; 1611 } 1612 1613 if (!amdgpu_ras_is_supported(adev, block) || 1614 !amdgpu_ras_get_aca_debug_mode(adev)) 1615 return -EOPNOTSUPP; 1616 1617 if (amdgpu_sriov_vf(adev)) 1618 return -EOPNOTSUPP; 1619 1620 /* skip ras error reset in gpu reset */ 1621 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1622 ((smu_funcs && smu_funcs->set_debug_mode) || 1623 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1624 return -EOPNOTSUPP; 1625 1626 if (block_obj->hw_ops->reset_ras_error_count) 1627 block_obj->hw_ops->reset_ras_error_count(adev); 1628 1629 return 0; 1630 } 1631 1632 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1633 enum amdgpu_ras_block block) 1634 { 1635 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1636 1637 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1638 return 0; 1639 1640 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1641 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1642 if (block_obj->hw_ops->reset_ras_error_status) 1643 block_obj->hw_ops->reset_ras_error_status(adev); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1650 struct ras_inject_if *info) 1651 { 1652 struct ras_cmd_inject_error_req inject_req; 1653 struct ras_cmd_inject_error_rsp rsp; 1654 1655 if (!info) 1656 return -EINVAL; 1657 1658 memset(&inject_req, 0, sizeof(inject_req)); 1659 inject_req.block_id = info->head.block; 1660 inject_req.subblock_id = info->head.sub_block_index; 1661 inject_req.address = info->address; 1662 inject_req.error_type = info->head.type; 1663 inject_req.instance_mask = info->instance_mask; 1664 inject_req.method = info->value; 1665 1666 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1667 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1668 } 1669 1670 /* wrapper of psp_ras_trigger_error */ 1671 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1672 struct ras_inject_if *info) 1673 { 1674 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1675 struct ta_ras_trigger_error_input block_info = { 1676 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1677 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1678 .sub_block_index = info->head.sub_block_index, 1679 .address = info->address, 1680 .value = info->value, 1681 }; 1682 int ret = -EINVAL; 1683 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1684 info->head.block, 1685 info->head.sub_block_index); 1686 1687 if (amdgpu_uniras_enabled(adev)) 1688 return amdgpu_uniras_error_inject(adev, info); 1689 1690 /* inject on guest isn't allowed, return success directly */ 1691 if (amdgpu_sriov_vf(adev)) 1692 return 0; 1693 1694 if (!obj) 1695 return -EINVAL; 1696 1697 if (!block_obj || !block_obj->hw_ops) { 1698 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1699 get_ras_block_str(&info->head)); 1700 return -EINVAL; 1701 } 1702 1703 /* Calculate XGMI relative offset */ 1704 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1705 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1706 block_info.address = 1707 amdgpu_xgmi_get_relative_phy_addr(adev, 1708 block_info.address); 1709 } 1710 1711 if (block_obj->hw_ops->ras_error_inject) { 1712 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1713 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1714 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1715 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1716 info->instance_mask); 1717 } else { 1718 /* default path */ 1719 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1720 } 1721 1722 if (ret) 1723 dev_err(adev->dev, "ras inject %s failed %d\n", 1724 get_ras_block_str(&info->head), ret); 1725 1726 return ret; 1727 } 1728 1729 /** 1730 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1731 * @adev: pointer to AMD GPU device 1732 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1733 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1734 * @query_info: pointer to ras_query_if 1735 * 1736 * Return 0 for query success or do nothing, otherwise return an error 1737 * on failures 1738 */ 1739 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1740 unsigned long *ce_count, 1741 unsigned long *ue_count, 1742 struct ras_query_if *query_info) 1743 { 1744 int ret; 1745 1746 if (!query_info) 1747 /* do nothing if query_info is not specified */ 1748 return 0; 1749 1750 ret = amdgpu_ras_query_error_status(adev, query_info); 1751 if (ret) 1752 return ret; 1753 1754 *ce_count += query_info->ce_count; 1755 *ue_count += query_info->ue_count; 1756 1757 /* some hardware/IP supports read to clear 1758 * no need to explictly reset the err status after the query call */ 1759 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1760 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1761 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1762 dev_warn(adev->dev, 1763 "Failed to reset error counter and error status\n"); 1764 } 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1771 * @adev: pointer to AMD GPU device 1772 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1773 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1774 * errors. 1775 * @query_info: pointer to ras_query_if if the query request is only for 1776 * specific ip block; if info is NULL, then the qurey request is for 1777 * all the ip blocks that support query ras error counters/status 1778 * 1779 * If set, @ce_count or @ue_count, count and return the corresponding 1780 * error counts in those integer pointers. Return 0 if the device 1781 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1782 */ 1783 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1784 unsigned long *ce_count, 1785 unsigned long *ue_count, 1786 struct ras_query_if *query_info) 1787 { 1788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1789 struct ras_manager *obj; 1790 unsigned long ce, ue; 1791 int ret; 1792 1793 if (!adev->ras_enabled || !con) 1794 return -EOPNOTSUPP; 1795 1796 /* Don't count since no reporting. 1797 */ 1798 if (!ce_count && !ue_count) 1799 return 0; 1800 1801 ce = 0; 1802 ue = 0; 1803 if (!query_info) { 1804 /* query all the ip blocks that support ras query interface */ 1805 list_for_each_entry(obj, &con->head, node) { 1806 struct ras_query_if info = { 1807 .head = obj->head, 1808 }; 1809 1810 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1811 } 1812 } else { 1813 /* query specific ip block */ 1814 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1815 } 1816 1817 if (ret) 1818 return ret; 1819 1820 if (ce_count) 1821 *ce_count = ce; 1822 1823 if (ue_count) 1824 *ue_count = ue; 1825 1826 return 0; 1827 } 1828 /* query/inject/cure end */ 1829 1830 1831 /* sysfs begin */ 1832 1833 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1834 struct ras_badpage *bps, uint32_t count, uint32_t start); 1835 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837 1838 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1839 { 1840 switch (flags) { 1841 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1842 return "R"; 1843 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1844 return "P"; 1845 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1846 default: 1847 return "F"; 1848 } 1849 } 1850 1851 /** 1852 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1853 * 1854 * It allows user to read the bad pages of vram on the gpu through 1855 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1856 * 1857 * It outputs multiple lines, and each line stands for one gpu page. 1858 * 1859 * The format of one line is below, 1860 * gpu pfn : gpu page size : flags 1861 * 1862 * gpu pfn and gpu page size are printed in hex format. 1863 * flags can be one of below character, 1864 * 1865 * R: reserved, this gpu page is reserved and not able to use. 1866 * 1867 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1868 * in next window of page_reserve. 1869 * 1870 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1871 * 1872 * Examples: 1873 * 1874 * .. code-block:: bash 1875 * 1876 * 0x00000001 : 0x00001000 : R 1877 * 0x00000002 : 0x00001000 : P 1878 * 1879 */ 1880 1881 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1882 struct kobject *kobj, const struct bin_attribute *attr, 1883 char *buf, loff_t ppos, size_t count) 1884 { 1885 struct amdgpu_ras *con = 1886 container_of(attr, struct amdgpu_ras, badpages_attr); 1887 struct amdgpu_device *adev = con->adev; 1888 const unsigned int element_size = 1889 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1890 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1891 unsigned int end = div64_ul(ppos + count - 1, element_size); 1892 ssize_t s = 0; 1893 struct ras_badpage *bps = NULL; 1894 int bps_count = 0, i, status; 1895 uint64_t address; 1896 1897 memset(buf, 0, count); 1898 1899 bps_count = end - start; 1900 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1901 if (!bps) 1902 return 0; 1903 1904 memset(bps, 0, sizeof(*bps) * bps_count); 1905 1906 if (amdgpu_uniras_enabled(adev)) 1907 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1908 else 1909 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1910 1911 if (bps_count <= 0) { 1912 kfree(bps); 1913 return 0; 1914 } 1915 1916 for (i = 0; i < bps_count; i++) { 1917 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1918 if (amdgpu_ras_check_critical_address(adev, address)) 1919 continue; 1920 1921 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1922 1923 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1924 address); 1925 if (status == -EBUSY) 1926 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1927 else if (status == -ENOENT) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1929 else 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1931 1932 s += scnprintf(&buf[s], element_size + 1, 1933 "0x%08x : 0x%08x : %1s\n", 1934 bps[i].bp, 1935 bps[i].size, 1936 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1937 } 1938 1939 kfree(bps); 1940 1941 return s; 1942 } 1943 1944 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1945 struct device_attribute *attr, char *buf) 1946 { 1947 struct amdgpu_ras *con = 1948 container_of(attr, struct amdgpu_ras, features_attr); 1949 1950 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1951 } 1952 1953 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1954 u32 *minor, u32 *rev) 1955 { 1956 int i; 1957 1958 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1959 return false; 1960 1961 for (i = 0; i < adev->num_ip_blocks; i++) { 1962 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1963 *major = adev->ip_blocks[i].version->major; 1964 *minor = adev->ip_blocks[i].version->minor; 1965 *rev = adev->ip_blocks[i].version->rev; 1966 return true; 1967 } 1968 } 1969 1970 return false; 1971 } 1972 1973 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1974 struct device_attribute *attr, char *buf) 1975 { 1976 struct amdgpu_ras *con = 1977 container_of(attr, struct amdgpu_ras, version_attr); 1978 u32 major, minor, rev; 1979 ssize_t size = 0; 1980 1981 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1982 con->eeprom_control.tbl_hdr.version); 1983 1984 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1985 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1986 major, minor, rev); 1987 1988 return size; 1989 } 1990 1991 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1992 struct device_attribute *attr, char *buf) 1993 { 1994 struct amdgpu_ras *con = 1995 container_of(attr, struct amdgpu_ras, schema_attr); 1996 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1997 } 1998 1999 static struct { 2000 enum ras_event_type type; 2001 const char *name; 2002 } dump_event[] = { 2003 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2004 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2005 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2006 }; 2007 2008 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2009 struct device_attribute *attr, char *buf) 2010 { 2011 struct amdgpu_ras *con = 2012 container_of(attr, struct amdgpu_ras, event_state_attr); 2013 struct ras_event_manager *event_mgr = con->event_mgr; 2014 struct ras_event_state *event_state; 2015 int i, size = 0; 2016 2017 if (!event_mgr) 2018 return -EINVAL; 2019 2020 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2021 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2022 event_state = &event_mgr->event_state[dump_event[i].type]; 2023 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2024 dump_event[i].name, 2025 atomic64_read(&event_state->count), 2026 event_state->last_seqno); 2027 } 2028 2029 return (ssize_t)size; 2030 } 2031 2032 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2033 { 2034 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2035 2036 if (adev->dev->kobj.sd) 2037 sysfs_remove_file_from_group(&adev->dev->kobj, 2038 &con->badpages_attr.attr, 2039 RAS_FS_NAME); 2040 } 2041 2042 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2043 { 2044 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2045 struct attribute *attrs[] = { 2046 &con->features_attr.attr, 2047 &con->version_attr.attr, 2048 &con->schema_attr.attr, 2049 &con->event_state_attr.attr, 2050 NULL 2051 }; 2052 struct attribute_group group = { 2053 .name = RAS_FS_NAME, 2054 .attrs = attrs, 2055 }; 2056 2057 if (adev->dev->kobj.sd) 2058 sysfs_remove_group(&adev->dev->kobj, &group); 2059 2060 return 0; 2061 } 2062 2063 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2064 struct ras_common_if *head) 2065 { 2066 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2067 2068 if (amdgpu_aca_is_enabled(adev)) 2069 return 0; 2070 2071 if (!obj || obj->attr_inuse) 2072 return -EINVAL; 2073 2074 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2075 return 0; 2076 2077 get_obj(obj); 2078 2079 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2080 "%s_err_count", head->name); 2081 2082 obj->sysfs_attr = (struct device_attribute){ 2083 .attr = { 2084 .name = obj->fs_data.sysfs_name, 2085 .mode = S_IRUGO, 2086 }, 2087 .show = amdgpu_ras_sysfs_read, 2088 }; 2089 sysfs_attr_init(&obj->sysfs_attr.attr); 2090 2091 if (sysfs_add_file_to_group(&adev->dev->kobj, 2092 &obj->sysfs_attr.attr, 2093 RAS_FS_NAME)) { 2094 put_obj(obj); 2095 return -EINVAL; 2096 } 2097 2098 obj->attr_inuse = 1; 2099 2100 return 0; 2101 } 2102 2103 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2104 struct ras_common_if *head) 2105 { 2106 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2107 2108 if (amdgpu_aca_is_enabled(adev)) 2109 return 0; 2110 2111 if (!obj || !obj->attr_inuse) 2112 return -EINVAL; 2113 2114 if (adev->dev->kobj.sd) 2115 sysfs_remove_file_from_group(&adev->dev->kobj, 2116 &obj->sysfs_attr.attr, 2117 RAS_FS_NAME); 2118 obj->attr_inuse = 0; 2119 put_obj(obj); 2120 2121 return 0; 2122 } 2123 2124 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2125 { 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2127 struct ras_manager *obj, *tmp; 2128 2129 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2130 amdgpu_ras_sysfs_remove(adev, &obj->head); 2131 } 2132 2133 if (amdgpu_bad_page_threshold != 0) 2134 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2135 2136 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2137 2138 return 0; 2139 } 2140 /* sysfs end */ 2141 2142 /** 2143 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2144 * 2145 * Normally when there is an uncorrectable error, the driver will reset 2146 * the GPU to recover. However, in the event of an unrecoverable error, 2147 * the driver provides an interface to reboot the system automatically 2148 * in that event. 2149 * 2150 * The following file in debugfs provides that interface: 2151 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2152 * 2153 * Usage: 2154 * 2155 * .. code-block:: bash 2156 * 2157 * echo true > .../ras/auto_reboot 2158 * 2159 */ 2160 /* debugfs begin */ 2161 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2162 { 2163 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2164 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2165 struct drm_minor *minor = adev_to_drm(adev)->primary; 2166 struct dentry *dir; 2167 2168 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2169 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2170 &amdgpu_ras_debugfs_ctrl_ops); 2171 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_eeprom_ops); 2173 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2174 &con->bad_page_cnt_threshold); 2175 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2176 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2177 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2178 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2179 &amdgpu_ras_debugfs_eeprom_size_ops); 2180 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2181 S_IRUGO, dir, adev, 2182 &amdgpu_ras_debugfs_eeprom_table_ops); 2183 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2184 2185 /* 2186 * After one uncorrectable error happens, usually GPU recovery will 2187 * be scheduled. But due to the known problem in GPU recovery failing 2188 * to bring GPU back, below interface provides one direct way to 2189 * user to reboot system automatically in such case within 2190 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2191 * will never be called. 2192 */ 2193 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2194 2195 /* 2196 * User could set this not to clean up hardware's error count register 2197 * of RAS IPs during ras recovery. 2198 */ 2199 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2200 &con->disable_ras_err_cnt_harvest); 2201 return dir; 2202 } 2203 2204 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2205 struct ras_fs_if *head, 2206 struct dentry *dir) 2207 { 2208 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2209 2210 if (!obj || !dir) 2211 return; 2212 2213 get_obj(obj); 2214 2215 memcpy(obj->fs_data.debugfs_name, 2216 head->debugfs_name, 2217 sizeof(obj->fs_data.debugfs_name)); 2218 2219 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2220 obj, &amdgpu_ras_debugfs_ops); 2221 } 2222 2223 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2224 { 2225 bool ret; 2226 2227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2228 case IP_VERSION(13, 0, 6): 2229 case IP_VERSION(13, 0, 12): 2230 case IP_VERSION(13, 0, 14): 2231 ret = true; 2232 break; 2233 default: 2234 ret = false; 2235 break; 2236 } 2237 2238 return ret; 2239 } 2240 2241 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2242 { 2243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2244 struct dentry *dir; 2245 struct ras_manager *obj; 2246 struct ras_fs_if fs_info; 2247 2248 /* 2249 * it won't be called in resume path, no need to check 2250 * suspend and gpu reset status 2251 */ 2252 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2253 return; 2254 2255 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2256 2257 list_for_each_entry(obj, &con->head, node) { 2258 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2259 (obj->attr_inuse == 1)) { 2260 sprintf(fs_info.debugfs_name, "%s_err_inject", 2261 get_ras_block_str(&obj->head)); 2262 fs_info.head = obj->head; 2263 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2264 } 2265 } 2266 2267 if (amdgpu_ras_aca_is_supported(adev)) { 2268 if (amdgpu_aca_is_enabled(adev)) 2269 amdgpu_aca_smu_debugfs_init(adev, dir); 2270 else 2271 amdgpu_mca_smu_debugfs_init(adev, dir); 2272 } 2273 } 2274 2275 /* debugfs end */ 2276 2277 /* ras fs */ 2278 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2279 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2280 static DEVICE_ATTR(features, S_IRUGO, 2281 amdgpu_ras_sysfs_features_read, NULL); 2282 static DEVICE_ATTR(version, 0444, 2283 amdgpu_ras_sysfs_version_show, NULL); 2284 static DEVICE_ATTR(schema, 0444, 2285 amdgpu_ras_sysfs_schema_show, NULL); 2286 static DEVICE_ATTR(event_state, 0444, 2287 amdgpu_ras_sysfs_event_state_show, NULL); 2288 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2289 { 2290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2291 struct attribute_group group = { 2292 .name = RAS_FS_NAME, 2293 }; 2294 struct attribute *attrs[] = { 2295 &con->features_attr.attr, 2296 &con->version_attr.attr, 2297 &con->schema_attr.attr, 2298 &con->event_state_attr.attr, 2299 NULL 2300 }; 2301 const struct bin_attribute *bin_attrs[] = { 2302 NULL, 2303 NULL, 2304 }; 2305 int r; 2306 2307 group.attrs = attrs; 2308 2309 /* add features entry */ 2310 con->features_attr = dev_attr_features; 2311 sysfs_attr_init(attrs[0]); 2312 2313 /* add version entry */ 2314 con->version_attr = dev_attr_version; 2315 sysfs_attr_init(attrs[1]); 2316 2317 /* add schema entry */ 2318 con->schema_attr = dev_attr_schema; 2319 sysfs_attr_init(attrs[2]); 2320 2321 /* add event_state entry */ 2322 con->event_state_attr = dev_attr_event_state; 2323 sysfs_attr_init(attrs[3]); 2324 2325 if (amdgpu_bad_page_threshold != 0) { 2326 /* add bad_page_features entry */ 2327 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2328 sysfs_bin_attr_init(&con->badpages_attr); 2329 bin_attrs[0] = &con->badpages_attr; 2330 group.bin_attrs = bin_attrs; 2331 } 2332 2333 r = sysfs_create_group(&adev->dev->kobj, &group); 2334 if (r) 2335 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2336 2337 return 0; 2338 } 2339 2340 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2341 { 2342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2343 struct ras_manager *con_obj, *ip_obj, *tmp; 2344 2345 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2346 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2347 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2348 if (ip_obj) 2349 put_obj(ip_obj); 2350 } 2351 } 2352 2353 amdgpu_ras_sysfs_remove_all(adev); 2354 return 0; 2355 } 2356 /* ras fs end */ 2357 2358 /* ih begin */ 2359 2360 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2361 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2362 * register to check whether the interrupt is triggered or not, and properly 2363 * ack the interrupt if it is there 2364 */ 2365 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2366 { 2367 /* Fatal error events are handled on host side */ 2368 if (amdgpu_sriov_vf(adev)) 2369 return; 2370 /* 2371 * If the current interrupt is caused by a non-fatal RAS error, skip 2372 * check for fatal error. For fatal errors, FED status of all devices 2373 * in XGMI hive gets set when the first device gets fatal error 2374 * interrupt. The error gets propagated to other devices as well, so 2375 * make sure to ack the interrupt regardless of FED status. 2376 */ 2377 if (!amdgpu_ras_get_fed_status(adev) && 2378 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2379 return; 2380 2381 if (amdgpu_uniras_enabled(adev)) { 2382 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2383 return; 2384 } 2385 2386 if (adev->nbio.ras && 2387 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2388 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2389 2390 if (adev->nbio.ras && 2391 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2392 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2393 } 2394 2395 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2396 struct amdgpu_iv_entry *entry) 2397 { 2398 bool poison_stat = false; 2399 struct amdgpu_device *adev = obj->adev; 2400 struct amdgpu_ras_block_object *block_obj = 2401 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2402 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2403 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2404 u64 event_id; 2405 int ret; 2406 2407 if (!block_obj || !con) 2408 return; 2409 2410 ret = amdgpu_ras_mark_ras_event(adev, type); 2411 if (ret) 2412 return; 2413 2414 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2415 /* both query_poison_status and handle_poison_consumption are optional, 2416 * but at least one of them should be implemented if we need poison 2417 * consumption handler 2418 */ 2419 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2420 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2421 if (!poison_stat) { 2422 /* Not poison consumption interrupt, no need to handle it */ 2423 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2424 block_obj->ras_comm.name); 2425 2426 return; 2427 } 2428 } 2429 2430 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2431 2432 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2433 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2434 2435 /* gpu reset is fallback for failed and default cases. 2436 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2437 */ 2438 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2439 event_id = amdgpu_ras_acquire_event_id(adev, type); 2440 RAS_EVENT_LOG(adev, event_id, 2441 "GPU reset for %s RAS poison consumption is issued!\n", 2442 block_obj->ras_comm.name); 2443 amdgpu_ras_reset_gpu(adev); 2444 } 2445 2446 if (!poison_stat) 2447 amdgpu_gfx_poison_consumption_handler(adev, entry); 2448 } 2449 2450 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2451 struct amdgpu_iv_entry *entry) 2452 { 2453 struct amdgpu_device *adev = obj->adev; 2454 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2455 u64 event_id; 2456 int ret; 2457 2458 ret = amdgpu_ras_mark_ras_event(adev, type); 2459 if (ret) 2460 return; 2461 2462 event_id = amdgpu_ras_acquire_event_id(adev, type); 2463 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2464 2465 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2466 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2467 2468 atomic_inc(&con->page_retirement_req_cnt); 2469 atomic_inc(&con->poison_creation_count); 2470 2471 wake_up(&con->page_retirement_wq); 2472 } 2473 } 2474 2475 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2476 struct amdgpu_iv_entry *entry) 2477 { 2478 struct ras_ih_data *data = &obj->ih_data; 2479 struct ras_err_data err_data; 2480 int ret; 2481 2482 if (!data->cb) 2483 return; 2484 2485 ret = amdgpu_ras_error_data_init(&err_data); 2486 if (ret) 2487 return; 2488 2489 /* Let IP handle its data, maybe we need get the output 2490 * from the callback to update the error type/count, etc 2491 */ 2492 amdgpu_ras_set_fed(obj->adev, true); 2493 ret = data->cb(obj->adev, &err_data, entry); 2494 /* ue will trigger an interrupt, and in that case 2495 * we need do a reset to recovery the whole system. 2496 * But leave IP do that recovery, here we just dispatch 2497 * the error. 2498 */ 2499 if (ret == AMDGPU_RAS_SUCCESS) { 2500 /* these counts could be left as 0 if 2501 * some blocks do not count error number 2502 */ 2503 obj->err_data.ue_count += err_data.ue_count; 2504 obj->err_data.ce_count += err_data.ce_count; 2505 obj->err_data.de_count += err_data.de_count; 2506 } 2507 2508 amdgpu_ras_error_data_fini(&err_data); 2509 } 2510 2511 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2512 { 2513 struct ras_ih_data *data = &obj->ih_data; 2514 struct amdgpu_iv_entry entry; 2515 2516 while (data->rptr != data->wptr) { 2517 rmb(); 2518 memcpy(&entry, &data->ring[data->rptr], 2519 data->element_size); 2520 2521 wmb(); 2522 data->rptr = (data->aligned_element_size + 2523 data->rptr) % data->ring_size; 2524 2525 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2526 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2527 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2528 else 2529 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2530 } else { 2531 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2532 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2533 else 2534 dev_warn(obj->adev->dev, 2535 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2536 } 2537 } 2538 } 2539 2540 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2541 { 2542 struct ras_ih_data *data = 2543 container_of(work, struct ras_ih_data, ih_work); 2544 struct ras_manager *obj = 2545 container_of(data, struct ras_manager, ih_data); 2546 2547 amdgpu_ras_interrupt_handler(obj); 2548 } 2549 2550 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2551 struct ras_dispatch_if *info) 2552 { 2553 struct ras_manager *obj; 2554 struct ras_ih_data *data; 2555 2556 if (amdgpu_uniras_enabled(adev)) { 2557 struct ras_ih_info ih_info; 2558 2559 memset(&ih_info, 0, sizeof(ih_info)); 2560 ih_info.block = info->head.block; 2561 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2562 2563 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2564 } 2565 2566 obj = amdgpu_ras_find_obj(adev, &info->head); 2567 if (!obj) 2568 return -EINVAL; 2569 2570 data = &obj->ih_data; 2571 2572 if (data->inuse == 0) 2573 return 0; 2574 2575 /* Might be overflow... */ 2576 memcpy(&data->ring[data->wptr], info->entry, 2577 data->element_size); 2578 2579 wmb(); 2580 data->wptr = (data->aligned_element_size + 2581 data->wptr) % data->ring_size; 2582 2583 schedule_work(&data->ih_work); 2584 2585 return 0; 2586 } 2587 2588 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2589 struct ras_common_if *head) 2590 { 2591 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2592 struct ras_ih_data *data; 2593 2594 if (!obj) 2595 return -EINVAL; 2596 2597 data = &obj->ih_data; 2598 if (data->inuse == 0) 2599 return 0; 2600 2601 cancel_work_sync(&data->ih_work); 2602 2603 kfree(data->ring); 2604 memset(data, 0, sizeof(*data)); 2605 put_obj(obj); 2606 2607 return 0; 2608 } 2609 2610 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2611 struct ras_common_if *head) 2612 { 2613 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2614 struct ras_ih_data *data; 2615 struct amdgpu_ras_block_object *ras_obj; 2616 2617 if (!obj) { 2618 /* in case we registe the IH before enable ras feature */ 2619 obj = amdgpu_ras_create_obj(adev, head); 2620 if (!obj) 2621 return -EINVAL; 2622 } else 2623 get_obj(obj); 2624 2625 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2626 2627 data = &obj->ih_data; 2628 /* add the callback.etc */ 2629 *data = (struct ras_ih_data) { 2630 .inuse = 0, 2631 .cb = ras_obj->ras_cb, 2632 .element_size = sizeof(struct amdgpu_iv_entry), 2633 .rptr = 0, 2634 .wptr = 0, 2635 }; 2636 2637 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2638 2639 data->aligned_element_size = ALIGN(data->element_size, 8); 2640 /* the ring can store 64 iv entries. */ 2641 data->ring_size = 64 * data->aligned_element_size; 2642 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2643 if (!data->ring) { 2644 put_obj(obj); 2645 return -ENOMEM; 2646 } 2647 2648 /* IH is ready */ 2649 data->inuse = 1; 2650 2651 return 0; 2652 } 2653 2654 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2655 { 2656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2657 struct ras_manager *obj, *tmp; 2658 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2660 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2661 } 2662 2663 return 0; 2664 } 2665 /* ih end */ 2666 2667 /* traversal all IPs except NBIO to query error counter */ 2668 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2669 { 2670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2671 struct ras_manager *obj; 2672 2673 if (!adev->ras_enabled || !con) 2674 return; 2675 2676 list_for_each_entry(obj, &con->head, node) { 2677 struct ras_query_if info = { 2678 .head = obj->head, 2679 }; 2680 2681 /* 2682 * PCIE_BIF IP has one different isr by ras controller 2683 * interrupt, the specific ras counter query will be 2684 * done in that isr. So skip such block from common 2685 * sync flood interrupt isr calling. 2686 */ 2687 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2688 continue; 2689 2690 /* 2691 * this is a workaround for aldebaran, skip send msg to 2692 * smu to get ecc_info table due to smu handle get ecc 2693 * info table failed temporarily. 2694 * should be removed until smu fix handle ecc_info table. 2695 */ 2696 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2697 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2698 IP_VERSION(13, 0, 2))) 2699 continue; 2700 2701 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2702 2703 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2704 IP_VERSION(11, 0, 2) && 2705 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 4) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(13, 0, 0)) { 2709 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2710 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2711 } 2712 } 2713 } 2714 2715 /* Parse RdRspStatus and WrRspStatus */ 2716 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2717 struct ras_query_if *info) 2718 { 2719 struct amdgpu_ras_block_object *block_obj; 2720 /* 2721 * Only two block need to query read/write 2722 * RspStatus at current state 2723 */ 2724 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2725 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2726 return; 2727 2728 block_obj = amdgpu_ras_get_ras_block(adev, 2729 info->head.block, 2730 info->head.sub_block_index); 2731 2732 if (!block_obj || !block_obj->hw_ops) { 2733 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2734 get_ras_block_str(&info->head)); 2735 return; 2736 } 2737 2738 if (block_obj->hw_ops->query_ras_error_status) 2739 block_obj->hw_ops->query_ras_error_status(adev); 2740 2741 } 2742 2743 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2744 { 2745 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2746 struct ras_manager *obj; 2747 2748 if (!adev->ras_enabled || !con) 2749 return; 2750 2751 list_for_each_entry(obj, &con->head, node) { 2752 struct ras_query_if info = { 2753 .head = obj->head, 2754 }; 2755 2756 amdgpu_ras_error_status_query(adev, &info); 2757 } 2758 } 2759 2760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2761 struct ras_badpage *bps, uint32_t count, uint32_t start) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_err_handler_data *data; 2765 int r = 0; 2766 uint32_t i; 2767 2768 if (!con || !con->eh_data || !bps || !count) 2769 return -EINVAL; 2770 2771 mutex_lock(&con->recovery_lock); 2772 data = con->eh_data; 2773 if (start < data->count) { 2774 for (i = start; i < data->count; i++) { 2775 if (!data->bps[i].ts) 2776 continue; 2777 2778 bps[r].bp = data->bps[i].retired_page; 2779 r++; 2780 if (r >= count) 2781 break; 2782 } 2783 } 2784 mutex_unlock(&con->recovery_lock); 2785 2786 return r; 2787 } 2788 2789 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2790 struct ras_badpage *bps, uint32_t count, uint32_t start) 2791 { 2792 struct ras_cmd_bad_pages_info_req cmd_input; 2793 struct ras_cmd_bad_pages_info_rsp *output; 2794 uint32_t group, start_group, end_group; 2795 uint32_t pos, pos_in_group; 2796 int r = 0, i; 2797 2798 if (!bps || !count) 2799 return -EINVAL; 2800 2801 output = kmalloc(sizeof(*output), GFP_KERNEL); 2802 if (!output) 2803 return -ENOMEM; 2804 2805 memset(&cmd_input, 0, sizeof(cmd_input)); 2806 2807 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2808 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2809 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2810 2811 pos = start; 2812 for (group = start_group; group < end_group; group++) { 2813 memset(output, 0, sizeof(*output)); 2814 cmd_input.group_index = group; 2815 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2816 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2817 goto out; 2818 2819 if (pos >= output->bp_total_cnt) 2820 goto out; 2821 2822 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2823 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2824 if (!output->records[i].ts) 2825 continue; 2826 2827 bps[r].bp = output->records[i].retired_page; 2828 r++; 2829 if (r >= count) 2830 goto out; 2831 } 2832 } 2833 2834 out: 2835 kfree(output); 2836 return r; 2837 } 2838 2839 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2840 struct amdgpu_hive_info *hive, bool status) 2841 { 2842 struct amdgpu_device *tmp_adev; 2843 2844 if (hive) { 2845 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2846 amdgpu_ras_set_fed(tmp_adev, status); 2847 } else { 2848 amdgpu_ras_set_fed(adev, status); 2849 } 2850 } 2851 2852 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2853 { 2854 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2856 int hive_ras_recovery = 0; 2857 2858 if (hive) { 2859 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2860 amdgpu_put_xgmi_hive(hive); 2861 } 2862 2863 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2864 return true; 2865 2866 return false; 2867 } 2868 2869 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2870 { 2871 if (amdgpu_ras_intr_triggered()) 2872 return RAS_EVENT_TYPE_FATAL; 2873 else 2874 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2875 } 2876 2877 static void amdgpu_ras_do_recovery(struct work_struct *work) 2878 { 2879 struct amdgpu_ras *ras = 2880 container_of(work, struct amdgpu_ras, recovery_work); 2881 struct amdgpu_device *remote_adev = NULL; 2882 struct amdgpu_device *adev = ras->adev; 2883 struct list_head device_list, *device_list_handle = NULL; 2884 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2885 unsigned int error_query_mode; 2886 enum ras_event_type type; 2887 2888 if (hive) { 2889 atomic_set(&hive->ras_recovery, 1); 2890 2891 /* If any device which is part of the hive received RAS fatal 2892 * error interrupt, set fatal error status on all. This 2893 * condition will need a recovery, and flag will be cleared 2894 * as part of recovery. 2895 */ 2896 list_for_each_entry(remote_adev, &hive->device_list, 2897 gmc.xgmi.head) 2898 if (amdgpu_ras_get_fed_status(remote_adev)) { 2899 amdgpu_ras_set_fed_all(adev, hive, true); 2900 break; 2901 } 2902 } 2903 if (!ras->disable_ras_err_cnt_harvest) { 2904 2905 /* Build list of devices to query RAS related errors */ 2906 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2907 device_list_handle = &hive->device_list; 2908 } else { 2909 INIT_LIST_HEAD(&device_list); 2910 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2911 device_list_handle = &device_list; 2912 } 2913 2914 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2915 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2916 /* wait 500ms to ensure pmfw polling mca bank info done */ 2917 msleep(500); 2918 } 2919 } 2920 2921 type = amdgpu_ras_get_fatal_error_event(adev); 2922 list_for_each_entry(remote_adev, 2923 device_list_handle, gmc.xgmi.head) { 2924 if (amdgpu_uniras_enabled(remote_adev)) { 2925 amdgpu_ras_mgr_update_ras_ecc(remote_adev); 2926 } else { 2927 amdgpu_ras_query_err_status(remote_adev); 2928 amdgpu_ras_log_on_err_counter(remote_adev, type); 2929 } 2930 } 2931 2932 } 2933 2934 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2935 struct amdgpu_reset_context reset_context; 2936 memset(&reset_context, 0, sizeof(reset_context)); 2937 2938 reset_context.method = AMD_RESET_METHOD_NONE; 2939 reset_context.reset_req_dev = adev; 2940 reset_context.src = AMDGPU_RESET_SRC_RAS; 2941 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2942 2943 /* Perform full reset in fatal error mode */ 2944 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2945 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2946 else { 2947 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2948 2949 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2950 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2951 reset_context.method = AMD_RESET_METHOD_MODE2; 2952 } 2953 2954 /* Fatal error occurs in poison mode, mode1 reset is used to 2955 * recover gpu. 2956 */ 2957 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2958 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2959 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2960 2961 psp_fatal_error_recovery_quirk(&adev->psp); 2962 } 2963 } 2964 2965 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2966 } 2967 atomic_set(&ras->in_recovery, 0); 2968 if (hive) { 2969 atomic_set(&hive->ras_recovery, 0); 2970 amdgpu_put_xgmi_hive(hive); 2971 } 2972 } 2973 2974 /* alloc/realloc bps array */ 2975 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2976 struct ras_err_handler_data *data, int pages) 2977 { 2978 unsigned int old_space = data->count + data->space_left; 2979 unsigned int new_space = old_space + pages; 2980 unsigned int align_space = ALIGN(new_space, 512); 2981 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2982 2983 if (!bps) { 2984 return -ENOMEM; 2985 } 2986 2987 if (data->bps) { 2988 memcpy(bps, data->bps, 2989 data->count * sizeof(*data->bps)); 2990 kfree(data->bps); 2991 } 2992 2993 data->bps = bps; 2994 data->space_left += align_space - old_space; 2995 return 0; 2996 } 2997 2998 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2999 struct eeprom_table_record *bps, 3000 struct ras_err_data *err_data) 3001 { 3002 struct ta_ras_query_address_input addr_in; 3003 uint32_t socket = 0; 3004 int ret = 0; 3005 3006 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3007 socket = adev->smuio.funcs->get_socket_id(adev); 3008 3009 /* reinit err_data */ 3010 err_data->err_addr_cnt = 0; 3011 err_data->err_addr_len = adev->umc.retire_unit; 3012 3013 memset(&addr_in, 0, sizeof(addr_in)); 3014 addr_in.ma.err_addr = bps->address; 3015 addr_in.ma.socket_id = socket; 3016 addr_in.ma.ch_inst = bps->mem_channel; 3017 /* tell RAS TA the node instance is not used */ 3018 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3019 3020 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3021 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3022 &addr_in, NULL, false); 3023 3024 return ret; 3025 } 3026 3027 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3028 struct eeprom_table_record *bps, 3029 struct ras_err_data *err_data) 3030 { 3031 struct ta_ras_query_address_input addr_in; 3032 uint32_t die_id, socket = 0; 3033 3034 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3035 socket = adev->smuio.funcs->get_socket_id(adev); 3036 3037 /* although die id is gotten from PA in nps1 mode, the id is 3038 * fitable for any nps mode 3039 */ 3040 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3041 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3042 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3043 else 3044 return -EINVAL; 3045 3046 /* reinit err_data */ 3047 err_data->err_addr_cnt = 0; 3048 err_data->err_addr_len = adev->umc.retire_unit; 3049 3050 memset(&addr_in, 0, sizeof(addr_in)); 3051 addr_in.ma.err_addr = bps->address; 3052 addr_in.ma.ch_inst = bps->mem_channel; 3053 addr_in.ma.umc_inst = bps->mcumc_id; 3054 addr_in.ma.node_inst = die_id; 3055 addr_in.ma.socket_id = socket; 3056 3057 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3058 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3059 &addr_in, NULL, false); 3060 else 3061 return -EINVAL; 3062 } 3063 3064 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3065 struct eeprom_table_record *bps, int count) 3066 { 3067 int j; 3068 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3069 struct ras_err_handler_data *data = con->eh_data; 3070 3071 for (j = 0; j < count; j++) { 3072 if (amdgpu_ras_check_bad_page_unlock(con, 3073 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3074 data->count++; 3075 data->space_left--; 3076 continue; 3077 } 3078 3079 if (!data->space_left && 3080 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3081 return -ENOMEM; 3082 } 3083 3084 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3085 3086 memcpy(&data->bps[data->count], &(bps[j]), 3087 sizeof(struct eeprom_table_record)); 3088 data->count++; 3089 data->space_left--; 3090 con->bad_page_num++; 3091 } 3092 3093 return 0; 3094 } 3095 3096 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3097 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3098 enum amdgpu_memory_partition nps) 3099 { 3100 int i = 0; 3101 enum amdgpu_memory_partition save_nps; 3102 3103 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3104 3105 /*old asics just have pa in eeprom*/ 3106 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3107 memcpy(err_data->err_addr, bps, 3108 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3109 goto out; 3110 } 3111 3112 for (i = 0; i < adev->umc.retire_unit; i++) 3113 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3114 3115 if (save_nps) { 3116 if (save_nps == nps) { 3117 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3118 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3119 return -EINVAL; 3120 for (i = 0; i < adev->umc.retire_unit; i++) { 3121 err_data->err_addr[i].address = bps[0].address; 3122 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3123 err_data->err_addr[i].bank = bps[0].bank; 3124 err_data->err_addr[i].err_type = bps[0].err_type; 3125 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3126 } 3127 } else { 3128 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3129 return -EINVAL; 3130 } 3131 } else { 3132 if (bps[0].address == 0) { 3133 /* for specific old eeprom data, mca address is not stored, 3134 * calc it from pa 3135 */ 3136 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3137 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3138 return -EINVAL; 3139 } 3140 3141 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3142 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3143 memcpy(err_data->err_addr, bps, 3144 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3145 else 3146 return -EOPNOTSUPP; 3147 } 3148 } 3149 3150 out: 3151 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3152 } 3153 3154 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3155 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3156 enum amdgpu_memory_partition nps) 3157 { 3158 int i = 0; 3159 enum amdgpu_memory_partition save_nps; 3160 3161 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3162 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3163 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3164 } else { 3165 save_nps = nps; 3166 } 3167 3168 if (save_nps == nps) { 3169 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3170 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3171 return -EINVAL; 3172 for (i = 0; i < adev->umc.retire_unit; i++) { 3173 err_data->err_addr[i].address = bps->address; 3174 err_data->err_addr[i].mem_channel = bps->mem_channel; 3175 err_data->err_addr[i].bank = bps->bank; 3176 err_data->err_addr[i].err_type = bps->err_type; 3177 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3178 } 3179 } else { 3180 if (bps->address) { 3181 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3182 return -EINVAL; 3183 } else { 3184 /* for specific old eeprom data, mca address is not stored, 3185 * calc it from pa 3186 */ 3187 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3188 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3189 return -EINVAL; 3190 3191 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3192 return -EOPNOTSUPP; 3193 } 3194 } 3195 3196 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3197 adev->umc.retire_unit); 3198 } 3199 3200 /* it deal with vram only. */ 3201 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3202 struct eeprom_table_record *bps, int pages, bool from_rom) 3203 { 3204 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3205 struct ras_err_data err_data; 3206 struct amdgpu_ras_eeprom_control *control = 3207 &adev->psp.ras_context.ras->eeprom_control; 3208 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3209 int ret = 0; 3210 uint32_t i = 0; 3211 3212 if (!con || !con->eh_data || !bps || pages <= 0) 3213 return 0; 3214 3215 if (from_rom) { 3216 err_data.err_addr = 3217 kcalloc(adev->umc.retire_unit, 3218 sizeof(struct eeprom_table_record), GFP_KERNEL); 3219 if (!err_data.err_addr) { 3220 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3221 return -ENOMEM; 3222 } 3223 3224 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3225 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3226 } 3227 3228 mutex_lock(&con->recovery_lock); 3229 3230 if (from_rom) { 3231 /* there is no pa recs in V3, so skip pa recs processing */ 3232 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3233 !amdgpu_ras_smu_eeprom_supported(adev)) { 3234 for (i = 0; i < pages; i++) { 3235 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3236 if ((bps[i].address == bps[i + 1].address) && 3237 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3238 /* deal with retire_unit records a time */ 3239 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3240 &bps[i], &err_data, nps); 3241 if (ret) 3242 con->bad_page_num -= adev->umc.retire_unit; 3243 i += (adev->umc.retire_unit - 1); 3244 } else { 3245 break; 3246 } 3247 } else { 3248 break; 3249 } 3250 } 3251 } 3252 for (; i < pages; i++) { 3253 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3254 &bps[i], &err_data, nps); 3255 if (ret) 3256 con->bad_page_num -= adev->umc.retire_unit; 3257 } 3258 3259 con->eh_data->count_saved = con->eh_data->count; 3260 } else { 3261 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3262 } 3263 3264 if (from_rom) 3265 kfree(err_data.err_addr); 3266 mutex_unlock(&con->recovery_lock); 3267 3268 return ret; 3269 } 3270 3271 /* 3272 * write error record array to eeprom, the function should be 3273 * protected by recovery_lock 3274 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3275 */ 3276 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3277 unsigned long *new_cnt) 3278 { 3279 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3280 struct ras_err_handler_data *data; 3281 struct amdgpu_ras_eeprom_control *control; 3282 int save_count, unit_num, i; 3283 3284 if (!con || !con->eh_data) { 3285 if (new_cnt) 3286 *new_cnt = 0; 3287 3288 return 0; 3289 } 3290 3291 if (!con->eeprom_control.is_eeprom_valid) { 3292 dev_warn(adev->dev, 3293 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3294 if (new_cnt) 3295 *new_cnt = 0; 3296 3297 return 0; 3298 } 3299 3300 mutex_lock(&con->recovery_lock); 3301 control = &con->eeprom_control; 3302 data = con->eh_data; 3303 unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; 3304 save_count = con->bad_page_num - control->ras_num_bad_pages; 3305 mutex_unlock(&con->recovery_lock); 3306 3307 if (new_cnt) 3308 *new_cnt = unit_num; 3309 3310 /* only new entries are saved */ 3311 if (unit_num && save_count) { 3312 /*old asics only save pa to eeprom like before*/ 3313 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3314 if (amdgpu_ras_eeprom_append(control, 3315 &data->bps[data->count_saved], unit_num)) { 3316 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3317 return -EIO; 3318 } 3319 } else { 3320 for (i = 0; i < unit_num; i++) { 3321 if (amdgpu_ras_eeprom_append(control, 3322 &data->bps[data->count_saved + 3323 i * adev->umc.retire_unit], 1)) { 3324 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3325 return -EIO; 3326 } 3327 } 3328 } 3329 3330 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3331 data->count_saved = data->count; 3332 } 3333 3334 return 0; 3335 } 3336 3337 /* 3338 * read error record array in eeprom and reserve enough space for 3339 * storing new bad pages 3340 */ 3341 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3342 { 3343 struct amdgpu_ras_eeprom_control *control = 3344 &adev->psp.ras_context.ras->eeprom_control; 3345 struct eeprom_table_record *bps; 3346 int ret, i = 0; 3347 3348 /* no bad page record, skip eeprom access */ 3349 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3350 return 0; 3351 3352 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3353 if (!bps) 3354 return -ENOMEM; 3355 3356 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3357 if (ret) { 3358 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3359 } else { 3360 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3361 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3362 as pa recs, so add verion check to avoid it. 3363 */ 3364 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3365 !amdgpu_ras_smu_eeprom_supported(adev)) { 3366 for (i = 0; i < control->ras_num_recs; i++) { 3367 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3368 if ((bps[i].address == bps[i + 1].address) && 3369 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3370 control->ras_num_pa_recs += adev->umc.retire_unit; 3371 i += (adev->umc.retire_unit - 1); 3372 } else { 3373 control->ras_num_mca_recs += 3374 (control->ras_num_recs - i); 3375 break; 3376 } 3377 } else { 3378 control->ras_num_mca_recs += (control->ras_num_recs - i); 3379 break; 3380 } 3381 } 3382 } else { 3383 control->ras_num_mca_recs = control->ras_num_recs; 3384 } 3385 } 3386 3387 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3388 if (ret) 3389 goto out; 3390 3391 ret = amdgpu_ras_eeprom_check(control); 3392 if (ret) 3393 goto out; 3394 3395 /* HW not usable */ 3396 if (amdgpu_ras_is_rma(adev)) 3397 ret = -EHWPOISON; 3398 } 3399 3400 out: 3401 kfree(bps); 3402 return ret; 3403 } 3404 3405 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3406 uint64_t addr) 3407 { 3408 struct ras_err_handler_data *data = con->eh_data; 3409 struct amdgpu_device *adev = con->adev; 3410 int i; 3411 3412 if ((addr >= adev->gmc.mc_vram_size && 3413 adev->gmc.mc_vram_size) || 3414 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3415 return -EINVAL; 3416 3417 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3418 for (i = 0; i < data->count; i++) 3419 if (addr == data->bps[i].retired_page) 3420 return 1; 3421 3422 return 0; 3423 } 3424 3425 /* 3426 * check if an address belongs to bad page 3427 * 3428 * Note: this check is only for umc block 3429 */ 3430 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3431 uint64_t addr) 3432 { 3433 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3434 int ret = 0; 3435 3436 if (!con || !con->eh_data) 3437 return ret; 3438 3439 mutex_lock(&con->recovery_lock); 3440 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3441 mutex_unlock(&con->recovery_lock); 3442 return ret; 3443 } 3444 3445 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3446 uint32_t max_count) 3447 { 3448 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3449 3450 /* 3451 * amdgpu_bad_page_threshold is used to config 3452 * the threshold for the number of bad pages. 3453 * -1: Threshold is set to default value 3454 * Driver will issue a warning message when threshold is reached 3455 * and continue runtime services. 3456 * 0: Disable bad page retirement 3457 * Driver will not retire bad pages 3458 * which is intended for debugging purpose. 3459 * -2: Threshold is determined by a formula 3460 * that assumes 1 bad page per 100M of local memory. 3461 * Driver will continue runtime services when threhold is reached. 3462 * 0 < threshold < max number of bad page records in EEPROM, 3463 * A user-defined threshold is set 3464 * Driver will halt runtime services when this custom threshold is reached. 3465 */ 3466 if (amdgpu_bad_page_threshold == -2) { 3467 u64 val = adev->gmc.mc_vram_size; 3468 3469 do_div(val, RAS_BAD_PAGE_COVER); 3470 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3471 max_count); 3472 } else if (amdgpu_bad_page_threshold == -1) { 3473 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3474 } else { 3475 con->bad_page_cnt_threshold = min_t(int, max_count, 3476 amdgpu_bad_page_threshold); 3477 } 3478 } 3479 3480 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3481 enum amdgpu_ras_block block, uint16_t pasid, 3482 pasid_notify pasid_fn, void *data, uint32_t reset) 3483 { 3484 int ret = 0; 3485 struct ras_poison_msg poison_msg; 3486 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3487 3488 memset(&poison_msg, 0, sizeof(poison_msg)); 3489 poison_msg.block = block; 3490 poison_msg.pasid = pasid; 3491 poison_msg.reset = reset; 3492 poison_msg.pasid_fn = pasid_fn; 3493 poison_msg.data = data; 3494 3495 ret = kfifo_put(&con->poison_fifo, poison_msg); 3496 if (!ret) { 3497 dev_err(adev->dev, "Poison message fifo is full!\n"); 3498 return -ENOSPC; 3499 } 3500 3501 return 0; 3502 } 3503 3504 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3505 struct ras_poison_msg *poison_msg) 3506 { 3507 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3508 3509 return kfifo_get(&con->poison_fifo, poison_msg); 3510 } 3511 3512 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3513 { 3514 mutex_init(&ecc_log->lock); 3515 3516 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3517 ecc_log->de_queried_count = 0; 3518 ecc_log->consumption_q_count = 0; 3519 } 3520 3521 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3522 { 3523 struct radix_tree_iter iter; 3524 void __rcu **slot; 3525 struct ras_ecc_err *ecc_err; 3526 3527 mutex_lock(&ecc_log->lock); 3528 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3529 ecc_err = radix_tree_deref_slot(slot); 3530 kfree(ecc_err->err_pages.pfn); 3531 kfree(ecc_err); 3532 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3533 } 3534 mutex_unlock(&ecc_log->lock); 3535 3536 mutex_destroy(&ecc_log->lock); 3537 ecc_log->de_queried_count = 0; 3538 ecc_log->consumption_q_count = 0; 3539 } 3540 3541 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3542 uint32_t delayed_ms) 3543 { 3544 int ret; 3545 3546 mutex_lock(&con->umc_ecc_log.lock); 3547 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3548 UMC_ECC_NEW_DETECTED_TAG); 3549 mutex_unlock(&con->umc_ecc_log.lock); 3550 3551 if (ret) 3552 schedule_delayed_work(&con->page_retirement_dwork, 3553 msecs_to_jiffies(delayed_ms)); 3554 3555 return ret ? true : false; 3556 } 3557 3558 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3559 { 3560 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3561 page_retirement_dwork.work); 3562 struct amdgpu_device *adev = con->adev; 3563 struct ras_err_data err_data; 3564 3565 /* If gpu reset is ongoing, delay retiring the bad pages */ 3566 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3567 amdgpu_ras_schedule_retirement_dwork(con, 3568 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3569 return; 3570 } 3571 3572 amdgpu_ras_error_data_init(&err_data); 3573 3574 amdgpu_umc_handle_bad_pages(adev, &err_data); 3575 3576 amdgpu_ras_error_data_fini(&err_data); 3577 3578 amdgpu_ras_schedule_retirement_dwork(con, 3579 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3580 } 3581 3582 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3583 uint32_t poison_creation_count) 3584 { 3585 int ret = 0; 3586 struct ras_ecc_log_info *ecc_log; 3587 struct ras_query_if info; 3588 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3589 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3590 u64 de_queried_count; 3591 u64 consumption_q_count; 3592 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3593 3594 memset(&info, 0, sizeof(info)); 3595 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3596 3597 ecc_log = &ras->umc_ecc_log; 3598 ecc_log->de_queried_count = 0; 3599 ecc_log->consumption_q_count = 0; 3600 3601 do { 3602 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3603 if (ret) 3604 return ret; 3605 3606 de_queried_count = ecc_log->de_queried_count; 3607 consumption_q_count = ecc_log->consumption_q_count; 3608 3609 if (de_queried_count && consumption_q_count) 3610 break; 3611 3612 msleep(100); 3613 } while (--timeout); 3614 3615 if (de_queried_count) 3616 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3617 3618 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3619 amdgpu_ras_reset_gpu(adev); 3620 3621 return 0; 3622 } 3623 3624 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3625 { 3626 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3627 struct ras_poison_msg msg; 3628 int ret; 3629 3630 do { 3631 ret = kfifo_get(&con->poison_fifo, &msg); 3632 } while (ret); 3633 } 3634 3635 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3636 uint32_t msg_count, uint32_t *gpu_reset) 3637 { 3638 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3639 uint32_t reset_flags = 0, reset = 0; 3640 struct ras_poison_msg msg; 3641 int ret, i; 3642 3643 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3644 3645 for (i = 0; i < msg_count; i++) { 3646 ret = amdgpu_ras_get_poison_req(adev, &msg); 3647 if (!ret) 3648 continue; 3649 3650 if (msg.pasid_fn) 3651 msg.pasid_fn(adev, msg.pasid, msg.data); 3652 3653 reset_flags |= msg.reset; 3654 } 3655 3656 /* 3657 * Try to ensure poison creation handler is completed first 3658 * to set rma if bad page exceed threshold. 3659 */ 3660 flush_delayed_work(&con->page_retirement_dwork); 3661 3662 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3663 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3664 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3665 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3666 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3667 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3668 else 3669 reset = reset_flags; 3670 3671 con->gpu_reset_flags |= reset; 3672 amdgpu_ras_reset_gpu(adev); 3673 3674 *gpu_reset = reset; 3675 3676 /* Wait for gpu recovery to complete */ 3677 flush_work(&con->recovery_work); 3678 } 3679 3680 return 0; 3681 } 3682 3683 static int amdgpu_ras_page_retirement_thread(void *param) 3684 { 3685 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3686 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3687 uint32_t poison_creation_count, msg_count; 3688 uint32_t gpu_reset; 3689 int ret; 3690 3691 while (!kthread_should_stop()) { 3692 3693 wait_event_interruptible(con->page_retirement_wq, 3694 kthread_should_stop() || 3695 atomic_read(&con->page_retirement_req_cnt)); 3696 3697 if (kthread_should_stop()) 3698 break; 3699 3700 mutex_lock(&con->poison_lock); 3701 gpu_reset = 0; 3702 3703 do { 3704 poison_creation_count = atomic_read(&con->poison_creation_count); 3705 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3706 if (ret == -EIO) 3707 break; 3708 3709 if (poison_creation_count) { 3710 atomic_sub(poison_creation_count, &con->poison_creation_count); 3711 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3712 } 3713 } while (atomic_read(&con->poison_creation_count) && 3714 !atomic_read(&con->poison_consumption_count)); 3715 3716 if (ret != -EIO) { 3717 msg_count = kfifo_len(&con->poison_fifo); 3718 if (msg_count) { 3719 ret = amdgpu_ras_poison_consumption_handler(adev, 3720 msg_count, &gpu_reset); 3721 if ((ret != -EIO) && 3722 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3723 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3724 } 3725 } 3726 3727 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3728 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3729 /* Clear poison creation request */ 3730 atomic_set(&con->poison_creation_count, 0); 3731 atomic_set(&con->poison_consumption_count, 0); 3732 3733 /* Clear poison fifo */ 3734 amdgpu_ras_clear_poison_fifo(adev); 3735 3736 /* Clear all poison requests */ 3737 atomic_set(&con->page_retirement_req_cnt, 0); 3738 3739 if (ret == -EIO) { 3740 /* Wait for mode-1 reset to complete */ 3741 down_read(&adev->reset_domain->sem); 3742 up_read(&adev->reset_domain->sem); 3743 } 3744 3745 /* Wake up work to save bad pages to eeprom */ 3746 schedule_delayed_work(&con->page_retirement_dwork, 0); 3747 } else if (gpu_reset) { 3748 /* gpu just completed mode-2 reset or other reset */ 3749 /* Clear poison consumption messages cached in fifo */ 3750 msg_count = kfifo_len(&con->poison_fifo); 3751 if (msg_count) { 3752 amdgpu_ras_clear_poison_fifo(adev); 3753 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3754 } 3755 3756 atomic_set(&con->poison_consumption_count, 0); 3757 3758 /* Wake up work to save bad pages to eeprom */ 3759 schedule_delayed_work(&con->page_retirement_dwork, 0); 3760 } 3761 mutex_unlock(&con->poison_lock); 3762 } 3763 3764 return 0; 3765 } 3766 3767 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3768 { 3769 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3770 struct amdgpu_ras_eeprom_control *control; 3771 int ret; 3772 3773 if (!con || amdgpu_sriov_vf(adev)) 3774 return 0; 3775 3776 if (amdgpu_uniras_enabled(adev)) 3777 return 0; 3778 3779 control = &con->eeprom_control; 3780 con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev); 3781 3782 ret = amdgpu_ras_eeprom_init(control); 3783 control->is_eeprom_valid = !ret; 3784 3785 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3786 control->ras_num_pa_recs = control->ras_num_recs; 3787 3788 if (adev->umc.ras && 3789 adev->umc.ras->get_retire_flip_bits) 3790 adev->umc.ras->get_retire_flip_bits(adev); 3791 3792 if (control->ras_num_recs && control->is_eeprom_valid) { 3793 ret = amdgpu_ras_load_bad_pages(adev); 3794 if (ret) { 3795 control->is_eeprom_valid = false; 3796 return 0; 3797 } 3798 3799 amdgpu_dpm_send_hbm_bad_pages_num( 3800 adev, control->ras_num_bad_pages); 3801 3802 if (con->update_channel_flag == true) { 3803 amdgpu_dpm_send_hbm_bad_channel_flag( 3804 adev, control->bad_channel_bitmap); 3805 con->update_channel_flag = false; 3806 } 3807 3808 /* The format action is only applied to new ASICs */ 3809 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3810 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3811 if (!amdgpu_ras_eeprom_reset_table(control)) 3812 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3813 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3814 } 3815 3816 return 0; 3817 } 3818 3819 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3820 { 3821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3822 struct ras_err_handler_data **data; 3823 u32 max_eeprom_records_count = 0; 3824 int ret; 3825 3826 if (!con || amdgpu_sriov_vf(adev)) 3827 return 0; 3828 3829 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3830 * supports RAS and debugfs is enabled, but when 3831 * adev->ras_enabled is unset, i.e. when "ras_enable" 3832 * module parameter is set to 0. 3833 */ 3834 con->adev = adev; 3835 3836 if (!adev->ras_enabled) 3837 return 0; 3838 3839 data = &con->eh_data; 3840 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3841 if (!*data) { 3842 ret = -ENOMEM; 3843 goto out; 3844 } 3845 3846 mutex_init(&con->recovery_lock); 3847 mutex_init(&con->poison_lock); 3848 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3849 atomic_set(&con->in_recovery, 0); 3850 atomic_set(&con->rma_in_recovery, 0); 3851 con->eeprom_control.bad_channel_bitmap = 0; 3852 3853 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3854 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3855 3856 if (init_bp_info) { 3857 ret = amdgpu_ras_init_badpage_info(adev); 3858 if (ret) 3859 goto free; 3860 } 3861 3862 mutex_init(&con->page_rsv_lock); 3863 INIT_KFIFO(con->poison_fifo); 3864 mutex_init(&con->page_retirement_lock); 3865 init_waitqueue_head(&con->page_retirement_wq); 3866 atomic_set(&con->page_retirement_req_cnt, 0); 3867 atomic_set(&con->poison_creation_count, 0); 3868 atomic_set(&con->poison_consumption_count, 0); 3869 con->page_retirement_thread = 3870 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3871 if (IS_ERR(con->page_retirement_thread)) { 3872 con->page_retirement_thread = NULL; 3873 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3874 } 3875 3876 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3877 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3878 #ifdef CONFIG_X86_MCE_AMD 3879 if ((adev->asic_type == CHIP_ALDEBARAN) && 3880 (adev->gmc.xgmi.connected_to_cpu)) 3881 amdgpu_register_bad_pages_mca_notifier(adev); 3882 #endif 3883 return 0; 3884 3885 free: 3886 kfree((*data)->bps); 3887 kfree(*data); 3888 con->eh_data = NULL; 3889 out: 3890 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3891 3892 /* 3893 * Except error threshold exceeding case, other failure cases in this 3894 * function would not fail amdgpu driver init. 3895 */ 3896 if (!amdgpu_ras_is_rma(adev)) 3897 ret = 0; 3898 else 3899 ret = -EINVAL; 3900 3901 return ret; 3902 } 3903 3904 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3905 { 3906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3907 struct ras_err_handler_data *data = con->eh_data; 3908 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3909 bool ret; 3910 3911 /* recovery_init failed to init it, fini is useless */ 3912 if (!data) 3913 return 0; 3914 3915 /* Save all cached bad pages to eeprom */ 3916 do { 3917 flush_delayed_work(&con->page_retirement_dwork); 3918 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3919 } while (ret && max_flush_timeout--); 3920 3921 if (con->page_retirement_thread) 3922 kthread_stop(con->page_retirement_thread); 3923 3924 atomic_set(&con->page_retirement_req_cnt, 0); 3925 atomic_set(&con->poison_creation_count, 0); 3926 3927 mutex_destroy(&con->page_rsv_lock); 3928 3929 cancel_work_sync(&con->recovery_work); 3930 3931 cancel_delayed_work_sync(&con->page_retirement_dwork); 3932 3933 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3934 3935 mutex_lock(&con->recovery_lock); 3936 con->eh_data = NULL; 3937 kfree(data->bps); 3938 kfree(data); 3939 mutex_unlock(&con->recovery_lock); 3940 3941 amdgpu_ras_critical_region_init(adev); 3942 3943 return 0; 3944 } 3945 /* recovery end */ 3946 3947 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3948 { 3949 if (amdgpu_sriov_vf(adev)) { 3950 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3951 case IP_VERSION(13, 0, 2): 3952 case IP_VERSION(13, 0, 6): 3953 case IP_VERSION(13, 0, 12): 3954 case IP_VERSION(13, 0, 14): 3955 return true; 3956 default: 3957 return false; 3958 } 3959 } 3960 3961 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3962 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3963 case IP_VERSION(13, 0, 0): 3964 case IP_VERSION(13, 0, 6): 3965 case IP_VERSION(13, 0, 10): 3966 case IP_VERSION(13, 0, 12): 3967 case IP_VERSION(13, 0, 14): 3968 case IP_VERSION(14, 0, 3): 3969 return true; 3970 default: 3971 return false; 3972 } 3973 } 3974 3975 return adev->asic_type == CHIP_VEGA10 || 3976 adev->asic_type == CHIP_VEGA20 || 3977 adev->asic_type == CHIP_ARCTURUS || 3978 adev->asic_type == CHIP_ALDEBARAN || 3979 adev->asic_type == CHIP_SIENNA_CICHLID; 3980 } 3981 3982 /* 3983 * this is workaround for vega20 workstation sku, 3984 * force enable gfx ras, ignore vbios gfx ras flag 3985 * due to GC EDC can not write 3986 */ 3987 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3988 { 3989 struct atom_context *ctx = adev->mode_info.atom_context; 3990 3991 if (!ctx) 3992 return; 3993 3994 if (strnstr(ctx->vbios_pn, "D16406", 3995 sizeof(ctx->vbios_pn)) || 3996 strnstr(ctx->vbios_pn, "D36002", 3997 sizeof(ctx->vbios_pn))) 3998 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3999 } 4000 4001 /* Query ras capablity via atomfirmware interface */ 4002 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 4003 { 4004 /* mem_ecc cap */ 4005 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 4006 dev_info(adev->dev, "MEM ECC is active.\n"); 4007 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 4008 1 << AMDGPU_RAS_BLOCK__DF); 4009 } else { 4010 dev_info(adev->dev, "MEM ECC is not presented.\n"); 4011 } 4012 4013 /* sram_ecc cap */ 4014 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4015 dev_info(adev->dev, "SRAM ECC is active.\n"); 4016 if (!amdgpu_sriov_vf(adev)) 4017 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4018 1 << AMDGPU_RAS_BLOCK__DF); 4019 else 4020 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4021 1 << AMDGPU_RAS_BLOCK__SDMA | 4022 1 << AMDGPU_RAS_BLOCK__GFX); 4023 4024 /* 4025 * VCN/JPEG RAS can be supported on both bare metal and 4026 * SRIOV environment 4027 */ 4028 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4029 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4030 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4031 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4032 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4033 1 << AMDGPU_RAS_BLOCK__JPEG); 4034 else 4035 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4036 1 << AMDGPU_RAS_BLOCK__JPEG); 4037 4038 /* 4039 * XGMI RAS is not supported if xgmi num physical nodes 4040 * is zero 4041 */ 4042 if (!adev->gmc.xgmi.num_physical_nodes) 4043 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4044 } else { 4045 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4046 } 4047 } 4048 4049 /* Query poison mode from umc/df IP callbacks */ 4050 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4051 { 4052 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4053 bool df_poison, umc_poison; 4054 4055 /* poison setting is useless on SRIOV guest */ 4056 if (amdgpu_sriov_vf(adev) || !con) 4057 return; 4058 4059 /* Init poison supported flag, the default value is false */ 4060 if (adev->gmc.xgmi.connected_to_cpu || 4061 adev->gmc.is_app_apu) { 4062 /* enabled by default when GPU is connected to CPU */ 4063 con->poison_supported = true; 4064 } else if (adev->df.funcs && 4065 adev->df.funcs->query_ras_poison_mode && 4066 adev->umc.ras && 4067 adev->umc.ras->query_ras_poison_mode) { 4068 df_poison = 4069 adev->df.funcs->query_ras_poison_mode(adev); 4070 umc_poison = 4071 adev->umc.ras->query_ras_poison_mode(adev); 4072 4073 /* Only poison is set in both DF and UMC, we can support it */ 4074 if (df_poison && umc_poison) 4075 con->poison_supported = true; 4076 else if (df_poison != umc_poison) 4077 dev_warn(adev->dev, 4078 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4079 df_poison, umc_poison); 4080 } 4081 } 4082 4083 /* 4084 * check hardware's ras ability which will be saved in hw_supported. 4085 * if hardware does not support ras, we can skip some ras initializtion and 4086 * forbid some ras operations from IP. 4087 * if software itself, say boot parameter, limit the ras ability. We still 4088 * need allow IP do some limited operations, like disable. In such case, 4089 * we have to initialize ras as normal. but need check if operation is 4090 * allowed or not in each function. 4091 */ 4092 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4093 { 4094 adev->ras_hw_enabled = adev->ras_enabled = 0; 4095 4096 if (!amdgpu_ras_asic_supported(adev)) 4097 return; 4098 4099 if (amdgpu_sriov_vf(adev)) { 4100 if (amdgpu_virt_get_ras_capability(adev)) 4101 goto init_ras_enabled_flag; 4102 } 4103 4104 /* query ras capability from psp */ 4105 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4106 goto init_ras_enabled_flag; 4107 4108 /* query ras capablity from bios */ 4109 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4110 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4111 } else { 4112 /* driver only manages a few IP blocks RAS feature 4113 * when GPU is connected cpu through XGMI */ 4114 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4115 1 << AMDGPU_RAS_BLOCK__SDMA | 4116 1 << AMDGPU_RAS_BLOCK__MMHUB); 4117 } 4118 4119 /* apply asic specific settings (vega20 only for now) */ 4120 amdgpu_ras_get_quirks(adev); 4121 4122 /* query poison mode from umc/df ip callback */ 4123 amdgpu_ras_query_poison_mode(adev); 4124 4125 init_ras_enabled_flag: 4126 /* hw_supported needs to be aligned with RAS block mask. */ 4127 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4128 4129 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4130 adev->ras_hw_enabled & amdgpu_ras_mask; 4131 4132 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4133 if (!amdgpu_sriov_vf(adev)) { 4134 adev->aca.is_enabled = 4135 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4136 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4137 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4138 } 4139 4140 /* bad page feature is not applicable to specific app platform */ 4141 if (adev->gmc.is_app_apu && 4142 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4143 amdgpu_bad_page_threshold = 0; 4144 } 4145 4146 static void amdgpu_ras_counte_dw(struct work_struct *work) 4147 { 4148 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4149 ras_counte_delay_work.work); 4150 struct amdgpu_device *adev = con->adev; 4151 struct drm_device *dev = adev_to_drm(adev); 4152 unsigned long ce_count, ue_count; 4153 int res; 4154 4155 res = pm_runtime_get_sync(dev->dev); 4156 if (res < 0) 4157 goto Out; 4158 4159 /* Cache new values. 4160 */ 4161 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4162 atomic_set(&con->ras_ce_count, ce_count); 4163 atomic_set(&con->ras_ue_count, ue_count); 4164 } 4165 4166 Out: 4167 pm_runtime_put_autosuspend(dev->dev); 4168 } 4169 4170 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4171 { 4172 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4173 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4174 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4175 AMDGPU_RAS_ERROR__PARITY; 4176 } 4177 4178 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4179 { 4180 struct ras_event_state *event_state; 4181 int i; 4182 4183 memset(mgr, 0, sizeof(*mgr)); 4184 atomic64_set(&mgr->seqno, 0); 4185 4186 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4187 event_state = &mgr->event_state[i]; 4188 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4189 atomic64_set(&event_state->count, 0); 4190 } 4191 } 4192 4193 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4194 { 4195 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4196 struct amdgpu_hive_info *hive; 4197 4198 if (!ras) 4199 return; 4200 4201 hive = amdgpu_get_xgmi_hive(adev); 4202 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4203 4204 /* init event manager with node 0 on xgmi system */ 4205 if (!amdgpu_reset_in_recovery(adev)) { 4206 if (!hive || adev->gmc.xgmi.node_id == 0) 4207 ras_event_mgr_init(ras->event_mgr); 4208 } 4209 4210 if (hive) 4211 amdgpu_put_xgmi_hive(hive); 4212 } 4213 4214 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4215 { 4216 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4217 4218 if (!con || (adev->flags & AMD_IS_APU)) 4219 return; 4220 4221 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4222 case IP_VERSION(13, 0, 2): 4223 case IP_VERSION(13, 0, 6): 4224 case IP_VERSION(13, 0, 12): 4225 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4226 break; 4227 case IP_VERSION(13, 0, 14): 4228 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4229 break; 4230 default: 4231 break; 4232 } 4233 } 4234 4235 int amdgpu_ras_init(struct amdgpu_device *adev) 4236 { 4237 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4238 int r; 4239 4240 if (con) 4241 return 0; 4242 4243 con = kzalloc(sizeof(*con) + 4244 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4245 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4246 GFP_KERNEL); 4247 if (!con) 4248 return -ENOMEM; 4249 4250 con->adev = adev; 4251 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4252 atomic_set(&con->ras_ce_count, 0); 4253 atomic_set(&con->ras_ue_count, 0); 4254 4255 con->objs = (struct ras_manager *)(con + 1); 4256 4257 amdgpu_ras_set_context(adev, con); 4258 4259 amdgpu_ras_check_supported(adev); 4260 4261 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4262 /* set gfx block ras context feature for VEGA20 Gaming 4263 * send ras disable cmd to ras ta during ras late init. 4264 */ 4265 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4266 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4267 4268 return 0; 4269 } 4270 4271 r = 0; 4272 goto release_con; 4273 } 4274 4275 con->update_channel_flag = false; 4276 con->features = 0; 4277 con->schema = 0; 4278 INIT_LIST_HEAD(&con->head); 4279 /* Might need get this flag from vbios. */ 4280 con->flags = RAS_DEFAULT_FLAGS; 4281 4282 /* initialize nbio ras function ahead of any other 4283 * ras functions so hardware fatal error interrupt 4284 * can be enabled as early as possible */ 4285 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4286 case IP_VERSION(7, 4, 0): 4287 case IP_VERSION(7, 4, 1): 4288 case IP_VERSION(7, 4, 4): 4289 if (!adev->gmc.xgmi.connected_to_cpu) 4290 adev->nbio.ras = &nbio_v7_4_ras; 4291 break; 4292 case IP_VERSION(4, 3, 0): 4293 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4294 /* unlike other generation of nbio ras, 4295 * nbio v4_3 only support fatal error interrupt 4296 * to inform software that DF is freezed due to 4297 * system fatal error event. driver should not 4298 * enable nbio ras in such case. Instead, 4299 * check DF RAS */ 4300 adev->nbio.ras = &nbio_v4_3_ras; 4301 break; 4302 case IP_VERSION(6, 3, 1): 4303 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4304 /* unlike other generation of nbio ras, 4305 * nbif v6_3_1 only support fatal error interrupt 4306 * to inform software that DF is freezed due to 4307 * system fatal error event. driver should not 4308 * enable nbio ras in such case. Instead, 4309 * check DF RAS 4310 */ 4311 adev->nbio.ras = &nbif_v6_3_1_ras; 4312 break; 4313 case IP_VERSION(7, 9, 0): 4314 case IP_VERSION(7, 9, 1): 4315 if (!adev->gmc.is_app_apu) 4316 adev->nbio.ras = &nbio_v7_9_ras; 4317 break; 4318 default: 4319 /* nbio ras is not available */ 4320 break; 4321 } 4322 4323 /* nbio ras block needs to be enabled ahead of other ras blocks 4324 * to handle fatal error */ 4325 r = amdgpu_nbio_ras_sw_init(adev); 4326 if (r) 4327 return r; 4328 4329 if (adev->nbio.ras && 4330 adev->nbio.ras->init_ras_controller_interrupt) { 4331 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4332 if (r) 4333 goto release_con; 4334 } 4335 4336 if (adev->nbio.ras && 4337 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4338 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4339 if (r) 4340 goto release_con; 4341 } 4342 4343 /* Packed socket_id to ras feature mask bits[31:29] */ 4344 if (adev->smuio.funcs && 4345 adev->smuio.funcs->get_socket_id) 4346 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4347 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4348 4349 /* Get RAS schema for particular SOC */ 4350 con->schema = amdgpu_get_ras_schema(adev); 4351 4352 amdgpu_ras_init_reserved_vram_size(adev); 4353 4354 if (amdgpu_ras_fs_init(adev)) { 4355 r = -EINVAL; 4356 goto release_con; 4357 } 4358 4359 if (amdgpu_ras_aca_is_supported(adev)) { 4360 if (amdgpu_aca_is_enabled(adev)) 4361 r = amdgpu_aca_init(adev); 4362 else 4363 r = amdgpu_mca_init(adev); 4364 if (r) 4365 goto release_con; 4366 } 4367 4368 con->init_task_pid = task_pid_nr(current); 4369 get_task_comm(con->init_task_comm, current); 4370 4371 mutex_init(&con->critical_region_lock); 4372 INIT_LIST_HEAD(&con->critical_region_head); 4373 4374 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4375 "hardware ability[%x] ras_mask[%x]\n", 4376 adev->ras_hw_enabled, adev->ras_enabled); 4377 4378 return 0; 4379 release_con: 4380 amdgpu_ras_set_context(adev, NULL); 4381 kfree(con); 4382 4383 return r; 4384 } 4385 4386 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4387 { 4388 if (adev->gmc.xgmi.connected_to_cpu || 4389 adev->gmc.is_app_apu) 4390 return 1; 4391 return 0; 4392 } 4393 4394 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4395 struct ras_common_if *ras_block) 4396 { 4397 struct ras_query_if info = { 4398 .head = *ras_block, 4399 }; 4400 4401 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4402 return 0; 4403 4404 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4405 DRM_WARN("RAS init harvest failure"); 4406 4407 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4408 DRM_WARN("RAS init harvest reset failure"); 4409 4410 return 0; 4411 } 4412 4413 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4414 { 4415 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4416 4417 if (!con) 4418 return false; 4419 4420 return con->poison_supported; 4421 } 4422 4423 /* helper function to handle common stuff in ip late init phase */ 4424 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4425 struct ras_common_if *ras_block) 4426 { 4427 struct amdgpu_ras_block_object *ras_obj = NULL; 4428 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4429 struct ras_query_if *query_info; 4430 unsigned long ue_count, ce_count; 4431 int r; 4432 4433 /* disable RAS feature per IP block if it is not supported */ 4434 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4435 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4436 return 0; 4437 } 4438 4439 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4440 if (r) { 4441 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4442 /* in resume phase, if fail to enable ras, 4443 * clean up all ras fs nodes, and disable ras */ 4444 goto cleanup; 4445 } else 4446 return r; 4447 } 4448 4449 /* check for errors on warm reset edc persisant supported ASIC */ 4450 amdgpu_persistent_edc_harvesting(adev, ras_block); 4451 4452 /* in resume phase, no need to create ras fs node */ 4453 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4454 return 0; 4455 4456 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4457 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4458 (ras_obj->hw_ops->query_poison_status || 4459 ras_obj->hw_ops->handle_poison_consumption))) { 4460 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4461 if (r) 4462 goto cleanup; 4463 } 4464 4465 if (ras_obj->hw_ops && 4466 (ras_obj->hw_ops->query_ras_error_count || 4467 ras_obj->hw_ops->query_ras_error_status)) { 4468 r = amdgpu_ras_sysfs_create(adev, ras_block); 4469 if (r) 4470 goto interrupt; 4471 4472 /* Those are the cached values at init. 4473 */ 4474 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4475 if (!query_info) 4476 return -ENOMEM; 4477 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4478 4479 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4480 atomic_set(&con->ras_ce_count, ce_count); 4481 atomic_set(&con->ras_ue_count, ue_count); 4482 } 4483 4484 kfree(query_info); 4485 } 4486 4487 return 0; 4488 4489 interrupt: 4490 if (ras_obj->ras_cb) 4491 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4492 cleanup: 4493 amdgpu_ras_feature_enable(adev, ras_block, 0); 4494 return r; 4495 } 4496 4497 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4498 struct ras_common_if *ras_block) 4499 { 4500 return amdgpu_ras_block_late_init(adev, ras_block); 4501 } 4502 4503 /* helper function to remove ras fs node and interrupt handler */ 4504 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4505 struct ras_common_if *ras_block) 4506 { 4507 struct amdgpu_ras_block_object *ras_obj; 4508 if (!ras_block) 4509 return; 4510 4511 amdgpu_ras_sysfs_remove(adev, ras_block); 4512 4513 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4514 if (ras_obj->ras_cb) 4515 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4516 } 4517 4518 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4519 struct ras_common_if *ras_block) 4520 { 4521 return amdgpu_ras_block_late_fini(adev, ras_block); 4522 } 4523 4524 /* do some init work after IP late init as dependence. 4525 * and it runs in resume/gpu reset/booting up cases. 4526 */ 4527 void amdgpu_ras_resume(struct amdgpu_device *adev) 4528 { 4529 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4530 struct ras_manager *obj, *tmp; 4531 4532 if (!adev->ras_enabled || !con) { 4533 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4534 amdgpu_release_ras_context(adev); 4535 4536 return; 4537 } 4538 4539 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4540 /* Set up all other IPs which are not implemented. There is a 4541 * tricky thing that IP's actual ras error type should be 4542 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4543 * ERROR_NONE make sense anyway. 4544 */ 4545 amdgpu_ras_enable_all_features(adev, 1); 4546 4547 /* We enable ras on all hw_supported block, but as boot 4548 * parameter might disable some of them and one or more IP has 4549 * not implemented yet. So we disable them on behalf. 4550 */ 4551 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4552 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4553 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4554 /* there should be no any reference. */ 4555 WARN_ON(alive_obj(obj)); 4556 } 4557 } 4558 } 4559 } 4560 4561 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4562 { 4563 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4564 4565 if (!adev->ras_enabled || !con) 4566 return; 4567 4568 amdgpu_ras_disable_all_features(adev, 0); 4569 /* Make sure all ras objects are disabled. */ 4570 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4571 amdgpu_ras_disable_all_features(adev, 1); 4572 } 4573 4574 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4575 { 4576 struct amdgpu_ras_block_list *node, *tmp; 4577 struct amdgpu_ras_block_object *obj; 4578 int r; 4579 4580 amdgpu_ras_event_mgr_init(adev); 4581 4582 if (amdgpu_ras_aca_is_supported(adev)) { 4583 if (amdgpu_reset_in_recovery(adev)) { 4584 if (amdgpu_aca_is_enabled(adev)) 4585 r = amdgpu_aca_reset(adev); 4586 else 4587 r = amdgpu_mca_reset(adev); 4588 if (r) 4589 return r; 4590 } 4591 4592 if (!amdgpu_sriov_vf(adev)) { 4593 if (amdgpu_aca_is_enabled(adev)) 4594 amdgpu_ras_set_aca_debug_mode(adev, false); 4595 else 4596 amdgpu_ras_set_mca_debug_mode(adev, false); 4597 } 4598 } 4599 4600 /* Guest side doesn't need init ras feature */ 4601 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4602 return 0; 4603 4604 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4605 obj = node->ras_obj; 4606 if (!obj) { 4607 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4608 continue; 4609 } 4610 4611 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4612 continue; 4613 4614 if (obj->ras_late_init) { 4615 r = obj->ras_late_init(adev, &obj->ras_comm); 4616 if (r) { 4617 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4618 obj->ras_comm.name, r); 4619 return r; 4620 } 4621 } else 4622 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4623 } 4624 4625 return 0; 4626 } 4627 4628 /* do some fini work before IP fini as dependence */ 4629 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4630 { 4631 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4632 4633 if (!adev->ras_enabled || !con) 4634 return 0; 4635 4636 4637 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4638 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4639 amdgpu_ras_disable_all_features(adev, 0); 4640 amdgpu_ras_recovery_fini(adev); 4641 return 0; 4642 } 4643 4644 int amdgpu_ras_fini(struct amdgpu_device *adev) 4645 { 4646 struct amdgpu_ras_block_list *ras_node, *tmp; 4647 struct amdgpu_ras_block_object *obj = NULL; 4648 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4649 4650 if (!adev->ras_enabled || !con) 4651 return 0; 4652 4653 amdgpu_ras_critical_region_fini(adev); 4654 mutex_destroy(&con->critical_region_lock); 4655 4656 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4657 if (ras_node->ras_obj) { 4658 obj = ras_node->ras_obj; 4659 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4660 obj->ras_fini) 4661 obj->ras_fini(adev, &obj->ras_comm); 4662 else 4663 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4664 } 4665 4666 /* Clear ras blocks from ras_list and free ras block list node */ 4667 list_del(&ras_node->node); 4668 kfree(ras_node); 4669 } 4670 4671 amdgpu_ras_fs_fini(adev); 4672 amdgpu_ras_interrupt_remove_all(adev); 4673 4674 if (amdgpu_ras_aca_is_supported(adev)) { 4675 if (amdgpu_aca_is_enabled(adev)) 4676 amdgpu_aca_fini(adev); 4677 else 4678 amdgpu_mca_fini(adev); 4679 } 4680 4681 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4682 4683 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4684 amdgpu_ras_disable_all_features(adev, 0); 4685 4686 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4687 4688 amdgpu_ras_set_context(adev, NULL); 4689 kfree(con); 4690 4691 return 0; 4692 } 4693 4694 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4695 { 4696 struct amdgpu_ras *ras; 4697 4698 ras = amdgpu_ras_get_context(adev); 4699 if (!ras) 4700 return false; 4701 4702 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4703 } 4704 4705 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4706 { 4707 struct amdgpu_ras *ras; 4708 4709 ras = amdgpu_ras_get_context(adev); 4710 if (ras) { 4711 if (status) 4712 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4713 else 4714 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4715 } 4716 } 4717 4718 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4719 { 4720 struct amdgpu_ras *ras; 4721 4722 ras = amdgpu_ras_get_context(adev); 4723 if (ras) { 4724 ras->ras_err_state = 0; 4725 ras->gpu_reset_flags = 0; 4726 } 4727 } 4728 4729 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4730 enum amdgpu_ras_block block) 4731 { 4732 struct amdgpu_ras *ras; 4733 4734 ras = amdgpu_ras_get_context(adev); 4735 if (ras) 4736 set_bit(block, &ras->ras_err_state); 4737 } 4738 4739 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4740 { 4741 struct amdgpu_ras *ras; 4742 4743 ras = amdgpu_ras_get_context(adev); 4744 if (ras) { 4745 if (block == AMDGPU_RAS_BLOCK__ANY) 4746 return (ras->ras_err_state != 0); 4747 else 4748 return test_bit(block, &ras->ras_err_state) || 4749 test_bit(AMDGPU_RAS_BLOCK__LAST, 4750 &ras->ras_err_state); 4751 } 4752 4753 return false; 4754 } 4755 4756 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4757 { 4758 struct amdgpu_ras *ras; 4759 4760 ras = amdgpu_ras_get_context(adev); 4761 if (!ras) 4762 return NULL; 4763 4764 return ras->event_mgr; 4765 } 4766 4767 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4768 const void *caller) 4769 { 4770 struct ras_event_manager *event_mgr; 4771 struct ras_event_state *event_state; 4772 int ret = 0; 4773 4774 if (amdgpu_uniras_enabled(adev)) 4775 return 0; 4776 4777 if (type >= RAS_EVENT_TYPE_COUNT) { 4778 ret = -EINVAL; 4779 goto out; 4780 } 4781 4782 event_mgr = __get_ras_event_mgr(adev); 4783 if (!event_mgr) { 4784 ret = -EINVAL; 4785 goto out; 4786 } 4787 4788 event_state = &event_mgr->event_state[type]; 4789 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4790 atomic64_inc(&event_state->count); 4791 4792 out: 4793 if (ret && caller) 4794 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4795 (int)type, caller, ret); 4796 4797 return ret; 4798 } 4799 4800 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4801 { 4802 struct ras_event_manager *event_mgr; 4803 u64 id; 4804 4805 if (type >= RAS_EVENT_TYPE_COUNT) 4806 return RAS_EVENT_INVALID_ID; 4807 4808 switch (type) { 4809 case RAS_EVENT_TYPE_FATAL: 4810 case RAS_EVENT_TYPE_POISON_CREATION: 4811 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4812 event_mgr = __get_ras_event_mgr(adev); 4813 if (!event_mgr) 4814 return RAS_EVENT_INVALID_ID; 4815 4816 id = event_mgr->event_state[type].last_seqno; 4817 break; 4818 case RAS_EVENT_TYPE_INVALID: 4819 default: 4820 id = RAS_EVENT_INVALID_ID; 4821 break; 4822 } 4823 4824 return id; 4825 } 4826 4827 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4828 { 4829 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4830 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4831 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4832 u64 event_id = RAS_EVENT_INVALID_ID; 4833 4834 if (amdgpu_uniras_enabled(adev)) 4835 return 0; 4836 4837 if (!amdgpu_ras_mark_ras_event(adev, type)) 4838 event_id = amdgpu_ras_acquire_event_id(adev, type); 4839 4840 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4841 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4842 4843 amdgpu_ras_set_fed(adev, true); 4844 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4845 amdgpu_ras_reset_gpu(adev); 4846 } 4847 4848 return -EBUSY; 4849 } 4850 4851 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4852 { 4853 if (adev->asic_type == CHIP_VEGA20 && 4854 adev->pm.fw_version <= 0x283400) { 4855 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4856 amdgpu_ras_intr_triggered(); 4857 } 4858 4859 return false; 4860 } 4861 4862 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4863 { 4864 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4865 4866 if (!con) 4867 return; 4868 4869 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4870 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4871 amdgpu_ras_set_context(adev, NULL); 4872 kfree(con); 4873 } 4874 } 4875 4876 #ifdef CONFIG_X86_MCE_AMD 4877 static struct amdgpu_device *find_adev(uint32_t node_id) 4878 { 4879 int i; 4880 struct amdgpu_device *adev = NULL; 4881 4882 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4883 adev = mce_adev_list.devs[i]; 4884 4885 if (adev && adev->gmc.xgmi.connected_to_cpu && 4886 adev->gmc.xgmi.physical_node_id == node_id) 4887 break; 4888 adev = NULL; 4889 } 4890 4891 return adev; 4892 } 4893 4894 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4895 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4896 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4897 #define GPU_ID_OFFSET 8 4898 4899 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4900 unsigned long val, void *data) 4901 { 4902 struct mce *m = (struct mce *)data; 4903 struct amdgpu_device *adev = NULL; 4904 uint32_t gpu_id = 0; 4905 uint32_t umc_inst = 0, ch_inst = 0; 4906 4907 /* 4908 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4909 * and error occurred in DramECC (Extended error code = 0) then only 4910 * process the error, else bail out. 4911 */ 4912 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4913 (XEC(m->status, 0x3f) == 0x0))) 4914 return NOTIFY_DONE; 4915 4916 /* 4917 * If it is correctable error, return. 4918 */ 4919 if (mce_is_correctable(m)) 4920 return NOTIFY_OK; 4921 4922 /* 4923 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4924 */ 4925 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4926 4927 adev = find_adev(gpu_id); 4928 if (!adev) { 4929 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4930 gpu_id); 4931 return NOTIFY_DONE; 4932 } 4933 4934 /* 4935 * If it is uncorrectable error, then find out UMC instance and 4936 * channel index. 4937 */ 4938 umc_inst = GET_UMC_INST(m->ipid); 4939 ch_inst = GET_CHAN_INDEX(m->ipid); 4940 4941 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4942 umc_inst, ch_inst); 4943 4944 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4945 return NOTIFY_OK; 4946 else 4947 return NOTIFY_DONE; 4948 } 4949 4950 static struct notifier_block amdgpu_bad_page_nb = { 4951 .notifier_call = amdgpu_bad_page_notifier, 4952 .priority = MCE_PRIO_UC, 4953 }; 4954 4955 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4956 { 4957 /* 4958 * Add the adev to the mce_adev_list. 4959 * During mode2 reset, amdgpu device is temporarily 4960 * removed from the mgpu_info list which can cause 4961 * page retirement to fail. 4962 * Use this list instead of mgpu_info to find the amdgpu 4963 * device on which the UMC error was reported. 4964 */ 4965 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4966 4967 /* 4968 * Register the x86 notifier only once 4969 * with MCE subsystem. 4970 */ 4971 if (notifier_registered == false) { 4972 mce_register_decode_chain(&amdgpu_bad_page_nb); 4973 notifier_registered = true; 4974 } 4975 } 4976 #endif 4977 4978 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4979 { 4980 if (!adev) 4981 return NULL; 4982 4983 return adev->psp.ras_context.ras; 4984 } 4985 4986 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4987 { 4988 if (!adev) 4989 return -EINVAL; 4990 4991 adev->psp.ras_context.ras = ras_con; 4992 return 0; 4993 } 4994 4995 /* check if ras is supported on block, say, sdma, gfx */ 4996 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4997 unsigned int block) 4998 { 4999 int ret = 0; 5000 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5001 5002 if (block >= AMDGPU_RAS_BLOCK_COUNT) 5003 return 0; 5004 5005 ret = ras && (adev->ras_enabled & (1 << block)); 5006 5007 /* For the special asic with mem ecc enabled but sram ecc 5008 * not enabled, even if the ras block is not supported on 5009 * .ras_enabled, if the asic supports poison mode and the 5010 * ras block has ras configuration, it can be considered 5011 * that the ras block supports ras function. 5012 */ 5013 if (!ret && 5014 (block == AMDGPU_RAS_BLOCK__GFX || 5015 block == AMDGPU_RAS_BLOCK__SDMA || 5016 block == AMDGPU_RAS_BLOCK__VCN || 5017 block == AMDGPU_RAS_BLOCK__JPEG) && 5018 (amdgpu_ras_mask & (1 << block)) && 5019 amdgpu_ras_is_poison_mode_supported(adev) && 5020 amdgpu_ras_get_ras_block(adev, block, 0)) 5021 ret = 1; 5022 5023 return ret; 5024 } 5025 5026 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5027 { 5028 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5029 5030 /* mode1 is the only selection for RMA status */ 5031 if (amdgpu_ras_is_rma(adev)) { 5032 ras->gpu_reset_flags = 0; 5033 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5034 } 5035 5036 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5037 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5038 int hive_ras_recovery = 0; 5039 5040 if (hive) { 5041 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5042 amdgpu_put_xgmi_hive(hive); 5043 } 5044 /* In the case of multiple GPUs, after a GPU has started 5045 * resetting all GPUs on hive, other GPUs do not need to 5046 * trigger GPU reset again. 5047 */ 5048 if (!hive_ras_recovery) 5049 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5050 else 5051 atomic_set(&ras->in_recovery, 0); 5052 } else { 5053 flush_work(&ras->recovery_work); 5054 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5055 } 5056 5057 return 0; 5058 } 5059 5060 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5061 { 5062 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5063 int ret = 0; 5064 5065 if (con) { 5066 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5067 if (!ret) 5068 con->is_aca_debug_mode = enable; 5069 } 5070 5071 return ret; 5072 } 5073 5074 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5075 { 5076 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5077 int ret = 0; 5078 5079 if (con) { 5080 if (amdgpu_aca_is_enabled(adev)) 5081 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5082 else 5083 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5084 if (!ret) 5085 con->is_aca_debug_mode = enable; 5086 } 5087 5088 return ret; 5089 } 5090 5091 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5092 { 5093 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5094 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5095 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5096 5097 if (!con) 5098 return false; 5099 5100 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5101 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5102 return con->is_aca_debug_mode; 5103 else 5104 return true; 5105 } 5106 5107 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5108 unsigned int *error_query_mode) 5109 { 5110 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5111 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5112 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5113 5114 if (!con) { 5115 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5116 return false; 5117 } 5118 5119 if (amdgpu_sriov_vf(adev)) { 5120 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5121 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5122 *error_query_mode = 5123 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5124 } else { 5125 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5126 } 5127 5128 return true; 5129 } 5130 5131 /* Register each ip ras block into amdgpu ras */ 5132 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5133 struct amdgpu_ras_block_object *ras_block_obj) 5134 { 5135 struct amdgpu_ras_block_list *ras_node; 5136 if (!adev || !ras_block_obj) 5137 return -EINVAL; 5138 5139 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5140 if (!ras_node) 5141 return -ENOMEM; 5142 5143 INIT_LIST_HEAD(&ras_node->node); 5144 ras_node->ras_obj = ras_block_obj; 5145 list_add_tail(&ras_node->node, &adev->ras_list); 5146 5147 return 0; 5148 } 5149 5150 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5151 { 5152 if (!err_type_name) 5153 return; 5154 5155 switch (err_type) { 5156 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5157 sprintf(err_type_name, "correctable"); 5158 break; 5159 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5160 sprintf(err_type_name, "uncorrectable"); 5161 break; 5162 default: 5163 sprintf(err_type_name, "unknown"); 5164 break; 5165 } 5166 } 5167 5168 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5169 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5170 uint32_t instance, 5171 uint32_t *memory_id) 5172 { 5173 uint32_t err_status_lo_data, err_status_lo_offset; 5174 5175 if (!reg_entry) 5176 return false; 5177 5178 err_status_lo_offset = 5179 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5180 reg_entry->seg_lo, reg_entry->reg_lo); 5181 err_status_lo_data = RREG32(err_status_lo_offset); 5182 5183 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5184 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5185 return false; 5186 5187 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5188 5189 return true; 5190 } 5191 5192 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5193 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5194 uint32_t instance, 5195 unsigned long *err_cnt) 5196 { 5197 uint32_t err_status_hi_data, err_status_hi_offset; 5198 5199 if (!reg_entry) 5200 return false; 5201 5202 err_status_hi_offset = 5203 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5204 reg_entry->seg_hi, reg_entry->reg_hi); 5205 err_status_hi_data = RREG32(err_status_hi_offset); 5206 5207 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5208 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5209 /* keep the check here in case we need to refer to the result later */ 5210 dev_dbg(adev->dev, "Invalid err_info field\n"); 5211 5212 /* read err count */ 5213 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5214 5215 return true; 5216 } 5217 5218 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5219 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5220 uint32_t reg_list_size, 5221 const struct amdgpu_ras_memory_id_entry *mem_list, 5222 uint32_t mem_list_size, 5223 uint32_t instance, 5224 uint32_t err_type, 5225 unsigned long *err_count) 5226 { 5227 uint32_t memory_id; 5228 unsigned long err_cnt; 5229 char err_type_name[16]; 5230 uint32_t i, j; 5231 5232 for (i = 0; i < reg_list_size; i++) { 5233 /* query memory_id from err_status_lo */ 5234 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5235 instance, &memory_id)) 5236 continue; 5237 5238 /* query err_cnt from err_status_hi */ 5239 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5240 instance, &err_cnt) || 5241 !err_cnt) 5242 continue; 5243 5244 *err_count += err_cnt; 5245 5246 /* log the errors */ 5247 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5248 if (!mem_list) { 5249 /* memory_list is not supported */ 5250 dev_info(adev->dev, 5251 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5252 err_cnt, err_type_name, 5253 reg_list[i].block_name, 5254 instance, memory_id); 5255 } else { 5256 for (j = 0; j < mem_list_size; j++) { 5257 if (memory_id == mem_list[j].memory_id) { 5258 dev_info(adev->dev, 5259 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5260 err_cnt, err_type_name, 5261 reg_list[i].block_name, 5262 instance, mem_list[j].name); 5263 break; 5264 } 5265 } 5266 } 5267 } 5268 } 5269 5270 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5271 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5272 uint32_t reg_list_size, 5273 uint32_t instance) 5274 { 5275 uint32_t err_status_lo_offset, err_status_hi_offset; 5276 uint32_t i; 5277 5278 for (i = 0; i < reg_list_size; i++) { 5279 err_status_lo_offset = 5280 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5281 reg_list[i].seg_lo, reg_list[i].reg_lo); 5282 err_status_hi_offset = 5283 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5284 reg_list[i].seg_hi, reg_list[i].reg_hi); 5285 WREG32(err_status_lo_offset, 0); 5286 WREG32(err_status_hi_offset, 0); 5287 } 5288 } 5289 5290 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5291 { 5292 memset(err_data, 0, sizeof(*err_data)); 5293 5294 INIT_LIST_HEAD(&err_data->err_node_list); 5295 5296 return 0; 5297 } 5298 5299 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5300 { 5301 if (!err_node) 5302 return; 5303 5304 list_del(&err_node->node); 5305 kvfree(err_node); 5306 } 5307 5308 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5309 { 5310 struct ras_err_node *err_node, *tmp; 5311 5312 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5313 amdgpu_ras_error_node_release(err_node); 5314 } 5315 5316 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5317 struct amdgpu_smuio_mcm_config_info *mcm_info) 5318 { 5319 struct ras_err_node *err_node; 5320 struct amdgpu_smuio_mcm_config_info *ref_id; 5321 5322 if (!err_data || !mcm_info) 5323 return NULL; 5324 5325 for_each_ras_error(err_node, err_data) { 5326 ref_id = &err_node->err_info.mcm_info; 5327 5328 if (mcm_info->socket_id == ref_id->socket_id && 5329 mcm_info->die_id == ref_id->die_id) 5330 return err_node; 5331 } 5332 5333 return NULL; 5334 } 5335 5336 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5337 { 5338 struct ras_err_node *err_node; 5339 5340 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5341 if (!err_node) 5342 return NULL; 5343 5344 INIT_LIST_HEAD(&err_node->node); 5345 5346 return err_node; 5347 } 5348 5349 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5350 { 5351 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5352 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5353 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5354 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5355 5356 if (unlikely(infoa->socket_id != infob->socket_id)) 5357 return infoa->socket_id - infob->socket_id; 5358 else 5359 return infoa->die_id - infob->die_id; 5360 5361 return 0; 5362 } 5363 5364 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5365 struct amdgpu_smuio_mcm_config_info *mcm_info) 5366 { 5367 struct ras_err_node *err_node; 5368 5369 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5370 if (err_node) 5371 return &err_node->err_info; 5372 5373 err_node = amdgpu_ras_error_node_new(); 5374 if (!err_node) 5375 return NULL; 5376 5377 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5378 5379 err_data->err_list_count++; 5380 list_add_tail(&err_node->node, &err_data->err_node_list); 5381 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5382 5383 return &err_node->err_info; 5384 } 5385 5386 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5387 struct amdgpu_smuio_mcm_config_info *mcm_info, 5388 u64 count) 5389 { 5390 struct ras_err_info *err_info; 5391 5392 if (!err_data || !mcm_info) 5393 return -EINVAL; 5394 5395 if (!count) 5396 return 0; 5397 5398 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5399 if (!err_info) 5400 return -EINVAL; 5401 5402 err_info->ue_count += count; 5403 err_data->ue_count += count; 5404 5405 return 0; 5406 } 5407 5408 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5409 struct amdgpu_smuio_mcm_config_info *mcm_info, 5410 u64 count) 5411 { 5412 struct ras_err_info *err_info; 5413 5414 if (!err_data || !mcm_info) 5415 return -EINVAL; 5416 5417 if (!count) 5418 return 0; 5419 5420 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5421 if (!err_info) 5422 return -EINVAL; 5423 5424 err_info->ce_count += count; 5425 err_data->ce_count += count; 5426 5427 return 0; 5428 } 5429 5430 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5431 struct amdgpu_smuio_mcm_config_info *mcm_info, 5432 u64 count) 5433 { 5434 struct ras_err_info *err_info; 5435 5436 if (!err_data || !mcm_info) 5437 return -EINVAL; 5438 5439 if (!count) 5440 return 0; 5441 5442 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5443 if (!err_info) 5444 return -EINVAL; 5445 5446 err_info->de_count += count; 5447 err_data->de_count += count; 5448 5449 return 0; 5450 } 5451 5452 #define mmMP0_SMN_C2PMSG_92 0x1609C 5453 #define mmMP0_SMN_C2PMSG_126 0x160BE 5454 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5455 u32 instance) 5456 { 5457 u32 socket_id, aid_id, hbm_id; 5458 u32 fw_status; 5459 u32 boot_error; 5460 u64 reg_addr; 5461 5462 /* The pattern for smn addressing in other SOC could be different from 5463 * the one for aqua_vanjaram. We should revisit the code if the pattern 5464 * is changed. In such case, replace the aqua_vanjaram implementation 5465 * with more common helper */ 5466 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5467 aqua_vanjaram_encode_ext_smn_addressing(instance); 5468 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5469 5470 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5471 aqua_vanjaram_encode_ext_smn_addressing(instance); 5472 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5473 5474 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5475 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5476 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5477 5478 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5479 dev_info(adev->dev, 5480 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5481 socket_id, aid_id, hbm_id, fw_status); 5482 5483 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5484 dev_info(adev->dev, 5485 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5486 socket_id, aid_id, fw_status); 5487 5488 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5489 dev_info(adev->dev, 5490 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5491 socket_id, aid_id, fw_status); 5492 5493 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5494 dev_info(adev->dev, 5495 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5496 socket_id, aid_id, fw_status); 5497 5498 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5499 dev_info(adev->dev, 5500 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5501 socket_id, aid_id, fw_status); 5502 5503 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5504 dev_info(adev->dev, 5505 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5506 socket_id, aid_id, fw_status); 5507 5508 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5509 dev_info(adev->dev, 5510 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5511 socket_id, aid_id, hbm_id, fw_status); 5512 5513 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5514 dev_info(adev->dev, 5515 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5516 socket_id, aid_id, hbm_id, fw_status); 5517 5518 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5519 dev_info(adev->dev, 5520 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5521 socket_id, aid_id, fw_status); 5522 5523 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5524 dev_info(adev->dev, 5525 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5526 socket_id, aid_id, fw_status); 5527 } 5528 5529 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5530 u32 instance) 5531 { 5532 u64 reg_addr; 5533 u32 reg_data; 5534 int retry_loop; 5535 5536 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5537 aqua_vanjaram_encode_ext_smn_addressing(instance); 5538 5539 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5540 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5541 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5542 return false; 5543 else 5544 msleep(1); 5545 } 5546 5547 return true; 5548 } 5549 5550 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5551 { 5552 u32 i; 5553 5554 for (i = 0; i < num_instances; i++) { 5555 if (amdgpu_ras_boot_error_detected(adev, i)) 5556 amdgpu_ras_boot_time_error_reporting(adev, i); 5557 } 5558 } 5559 5560 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5561 { 5562 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5563 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5564 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5565 int ret = 0; 5566 5567 if (amdgpu_ras_check_critical_address(adev, start)) 5568 return 0; 5569 5570 mutex_lock(&con->page_rsv_lock); 5571 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5572 if (ret == -ENOENT) 5573 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5574 mutex_unlock(&con->page_rsv_lock); 5575 5576 return ret; 5577 } 5578 5579 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5580 const char *fmt, ...) 5581 { 5582 struct va_format vaf; 5583 va_list args; 5584 5585 va_start(args, fmt); 5586 vaf.fmt = fmt; 5587 vaf.va = &args; 5588 5589 if (RAS_EVENT_ID_IS_VALID(event_id)) 5590 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5591 else 5592 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5593 5594 va_end(args); 5595 } 5596 5597 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5598 { 5599 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5600 5601 if (amdgpu_uniras_enabled(adev)) 5602 return amdgpu_ras_mgr_is_rma(adev); 5603 5604 if (!con) 5605 return false; 5606 5607 return con->is_rma; 5608 } 5609 5610 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5611 struct amdgpu_bo *bo) 5612 { 5613 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5614 struct amdgpu_vram_mgr_resource *vres; 5615 struct ras_critical_region *region; 5616 struct drm_buddy_block *block; 5617 int ret = 0; 5618 5619 if (!bo || !bo->tbo.resource) 5620 return -EINVAL; 5621 5622 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5623 5624 mutex_lock(&con->critical_region_lock); 5625 5626 /* Check if the bo had been recorded */ 5627 list_for_each_entry(region, &con->critical_region_head, node) 5628 if (region->bo == bo) 5629 goto out; 5630 5631 /* Record new critical amdgpu bo */ 5632 list_for_each_entry(block, &vres->blocks, link) { 5633 region = kzalloc(sizeof(*region), GFP_KERNEL); 5634 if (!region) { 5635 ret = -ENOMEM; 5636 goto out; 5637 } 5638 region->bo = bo; 5639 region->start = amdgpu_vram_mgr_block_start(block); 5640 region->size = amdgpu_vram_mgr_block_size(block); 5641 list_add_tail(®ion->node, &con->critical_region_head); 5642 } 5643 5644 out: 5645 mutex_unlock(&con->critical_region_lock); 5646 5647 return ret; 5648 } 5649 5650 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5651 { 5652 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5653 } 5654 5655 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5656 { 5657 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5658 struct ras_critical_region *region, *tmp; 5659 5660 mutex_lock(&con->critical_region_lock); 5661 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5662 list_del(®ion->node); 5663 kfree(region); 5664 } 5665 mutex_unlock(&con->critical_region_lock); 5666 } 5667 5668 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5669 { 5670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5671 struct ras_critical_region *region; 5672 bool ret = false; 5673 5674 mutex_lock(&con->critical_region_lock); 5675 list_for_each_entry(region, &con->critical_region_head, node) { 5676 if ((region->start <= addr) && 5677 (addr < (region->start + region->size))) { 5678 ret = true; 5679 break; 5680 } 5681 } 5682 mutex_unlock(&con->critical_region_lock); 5683 5684 return ret; 5685 } 5686 5687 void amdgpu_ras_pre_reset(struct amdgpu_device *adev, 5688 struct list_head *device_list) 5689 { 5690 struct amdgpu_device *tmp_adev = NULL; 5691 5692 list_for_each_entry(tmp_adev, device_list, reset_list) { 5693 if (amdgpu_uniras_enabled(tmp_adev)) 5694 amdgpu_ras_mgr_pre_reset(tmp_adev); 5695 } 5696 } 5697 5698 void amdgpu_ras_post_reset(struct amdgpu_device *adev, 5699 struct list_head *device_list) 5700 { 5701 struct amdgpu_device *tmp_adev = NULL; 5702 5703 list_for_each_entry(tmp_adev, device_list, reset_list) { 5704 if (amdgpu_uniras_enabled(tmp_adev)) 5705 amdgpu_ras_mgr_post_reset(tmp_adev); 5706 } 5707 } 5708