1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 616 617 /** 618 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 619 * 620 * Some boards contain an EEPROM which is used to persistently store a list of 621 * bad pages which experiences ECC errors in vram. This interface provides 622 * a way to reset the EEPROM, e.g., after testing error injection. 623 * 624 * Usage: 625 * 626 * .. code-block:: bash 627 * 628 * echo 1 > ../ras/ras_eeprom_reset 629 * 630 * will reset EEPROM table to 0 entries. 631 * 632 */ 633 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 634 const char __user *buf, 635 size_t size, loff_t *pos) 636 { 637 struct amdgpu_device *adev = 638 (struct amdgpu_device *)file_inode(f)->i_private; 639 int ret; 640 641 if (amdgpu_uniras_enabled(adev)) { 642 ret = amdgpu_uniras_clear_badpages_info(adev); 643 return ret ? ret : size; 644 } 645 646 ret = amdgpu_ras_eeprom_reset_table( 647 &(amdgpu_ras_get_context(adev)->eeprom_control)); 648 649 if (!ret) { 650 /* Something was written to EEPROM. 651 */ 652 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 653 return size; 654 } else { 655 return ret; 656 } 657 } 658 659 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_ctrl_write, 663 .llseek = default_llseek 664 }; 665 666 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 667 .owner = THIS_MODULE, 668 .read = NULL, 669 .write = amdgpu_ras_debugfs_eeprom_write, 670 .llseek = default_llseek 671 }; 672 673 /** 674 * DOC: AMDGPU RAS sysfs Error Count Interface 675 * 676 * It allows the user to read the error count for each IP block on the gpu through 677 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 678 * 679 * It outputs the multiple lines which report the uncorrected (ue) and corrected 680 * (ce) error counts. 681 * 682 * The format of one line is below, 683 * 684 * [ce|ue]: count 685 * 686 * Example: 687 * 688 * .. code-block:: bash 689 * 690 * ue: 0 691 * ce: 1 692 * 693 */ 694 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 695 struct device_attribute *attr, char *buf) 696 { 697 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 698 struct ras_query_if info = { 699 .head = obj->head, 700 }; 701 702 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 703 return sysfs_emit(buf, "Query currently inaccessible\n"); 704 705 if (amdgpu_ras_query_error_status(obj->adev, &info)) 706 return -EINVAL; 707 708 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 709 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 710 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 711 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 712 } 713 714 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 715 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 716 "ce", info.ce_count, "de", info.de_count); 717 else 718 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 719 "ce", info.ce_count); 720 } 721 722 /* obj begin */ 723 724 #define get_obj(obj) do { (obj)->use++; } while (0) 725 #define alive_obj(obj) ((obj)->use) 726 727 static inline void put_obj(struct ras_manager *obj) 728 { 729 if (obj && (--obj->use == 0)) { 730 list_del(&obj->node); 731 amdgpu_ras_error_data_fini(&obj->err_data); 732 } 733 734 if (obj && (obj->use < 0)) 735 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 736 } 737 738 /* make one obj and return it. */ 739 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 740 struct ras_common_if *head) 741 { 742 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 743 struct ras_manager *obj; 744 745 if (!adev->ras_enabled || !con) 746 return NULL; 747 748 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 749 return NULL; 750 751 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 752 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 753 return NULL; 754 755 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 756 } else 757 obj = &con->objs[head->block]; 758 759 /* already exist. return obj? */ 760 if (alive_obj(obj)) 761 return NULL; 762 763 if (amdgpu_ras_error_data_init(&obj->err_data)) 764 return NULL; 765 766 obj->head = *head; 767 obj->adev = adev; 768 list_add(&obj->node, &con->head); 769 get_obj(obj); 770 771 return obj; 772 } 773 774 /* return an obj equal to head, or the first when head is NULL */ 775 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 776 struct ras_common_if *head) 777 { 778 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 779 struct ras_manager *obj; 780 int i; 781 782 if (!adev->ras_enabled || !con) 783 return NULL; 784 785 if (head) { 786 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 787 return NULL; 788 789 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 790 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 791 return NULL; 792 793 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 794 } else 795 obj = &con->objs[head->block]; 796 797 if (alive_obj(obj)) 798 return obj; 799 } else { 800 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 801 obj = &con->objs[i]; 802 if (alive_obj(obj)) 803 return obj; 804 } 805 } 806 807 return NULL; 808 } 809 /* obj end */ 810 811 /* feature ctl begin */ 812 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 813 struct ras_common_if *head) 814 { 815 return adev->ras_hw_enabled & BIT(head->block); 816 } 817 818 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 819 struct ras_common_if *head) 820 { 821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 822 823 return con->features & BIT(head->block); 824 } 825 826 /* 827 * if obj is not created, then create one. 828 * set feature enable flag. 829 */ 830 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 831 struct ras_common_if *head, int enable) 832 { 833 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 834 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 835 836 /* If hardware does not support ras, then do not create obj. 837 * But if hardware support ras, we can create the obj. 838 * Ras framework checks con->hw_supported to see if it need do 839 * corresponding initialization. 840 * IP checks con->support to see if it need disable ras. 841 */ 842 if (!amdgpu_ras_is_feature_allowed(adev, head)) 843 return 0; 844 845 if (enable) { 846 if (!obj) { 847 obj = amdgpu_ras_create_obj(adev, head); 848 if (!obj) 849 return -EINVAL; 850 } else { 851 /* In case we create obj somewhere else */ 852 get_obj(obj); 853 } 854 con->features |= BIT(head->block); 855 } else { 856 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 857 con->features &= ~BIT(head->block); 858 put_obj(obj); 859 } 860 } 861 862 return 0; 863 } 864 865 /* wrapper of psp_ras_enable_features */ 866 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 867 struct ras_common_if *head, bool enable) 868 { 869 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 870 union ta_ras_cmd_input *info; 871 int ret; 872 873 if (!con) 874 return -EINVAL; 875 876 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 877 /* For gfx ip, regardless of feature support status, */ 878 /* Force issue enable or disable ras feature commands */ 879 if (head->block != AMDGPU_RAS_BLOCK__GFX && 880 !amdgpu_ras_is_feature_allowed(adev, head)) 881 return 0; 882 883 /* Only enable gfx ras feature from host side */ 884 if (head->block == AMDGPU_RAS_BLOCK__GFX && 885 !amdgpu_sriov_vf(adev) && 886 !amdgpu_ras_intr_triggered()) { 887 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 888 if (!info) 889 return -ENOMEM; 890 891 if (!enable) { 892 info->disable_features = (struct ta_ras_disable_features_input) { 893 .block_id = amdgpu_ras_block_to_ta(head->block), 894 .error_type = amdgpu_ras_error_to_ta(head->type), 895 }; 896 } else { 897 info->enable_features = (struct ta_ras_enable_features_input) { 898 .block_id = amdgpu_ras_block_to_ta(head->block), 899 .error_type = amdgpu_ras_error_to_ta(head->type), 900 }; 901 } 902 903 ret = psp_ras_enable_features(&adev->psp, info, enable); 904 if (ret) { 905 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 906 enable ? "enable":"disable", 907 get_ras_block_str(head), 908 amdgpu_ras_is_poison_mode_supported(adev), ret); 909 kfree(info); 910 return ret; 911 } 912 913 kfree(info); 914 } 915 916 /* setup the obj */ 917 __amdgpu_ras_feature_enable(adev, head, enable); 918 919 return 0; 920 } 921 922 /* Only used in device probe stage and called only once. */ 923 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 924 struct ras_common_if *head, bool enable) 925 { 926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 927 int ret; 928 929 if (!con) 930 return -EINVAL; 931 932 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 933 if (enable) { 934 /* There is no harm to issue a ras TA cmd regardless of 935 * the currecnt ras state. 936 * If current state == target state, it will do nothing 937 * But sometimes it requests driver to reset and repost 938 * with error code -EAGAIN. 939 */ 940 ret = amdgpu_ras_feature_enable(adev, head, 1); 941 /* With old ras TA, we might fail to enable ras. 942 * Log it and just setup the object. 943 * TODO need remove this WA in the future. 944 */ 945 if (ret == -EINVAL) { 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (!ret) 948 dev_info(adev->dev, 949 "RAS INFO: %s setup object\n", 950 get_ras_block_str(head)); 951 } 952 } else { 953 /* setup the object then issue a ras TA disable cmd.*/ 954 ret = __amdgpu_ras_feature_enable(adev, head, 1); 955 if (ret) 956 return ret; 957 958 /* gfx block ras disable cmd must send to ras-ta */ 959 if (head->block == AMDGPU_RAS_BLOCK__GFX) 960 con->features |= BIT(head->block); 961 962 ret = amdgpu_ras_feature_enable(adev, head, 0); 963 964 /* clean gfx block ras features flag */ 965 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 966 con->features &= ~BIT(head->block); 967 } 968 } else 969 ret = amdgpu_ras_feature_enable(adev, head, enable); 970 971 return ret; 972 } 973 974 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 975 bool bypass) 976 { 977 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 978 struct ras_manager *obj, *tmp; 979 980 list_for_each_entry_safe(obj, tmp, &con->head, node) { 981 /* bypass psp. 982 * aka just release the obj and corresponding flags 983 */ 984 if (bypass) { 985 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 986 break; 987 } else { 988 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 989 break; 990 } 991 } 992 993 return con->features; 994 } 995 996 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 997 bool bypass) 998 { 999 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1000 int i; 1001 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1002 1003 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1004 struct ras_common_if head = { 1005 .block = i, 1006 .type = default_ras_type, 1007 .sub_block_index = 0, 1008 }; 1009 1010 if (i == AMDGPU_RAS_BLOCK__MCA) 1011 continue; 1012 1013 if (bypass) { 1014 /* 1015 * bypass psp. vbios enable ras for us. 1016 * so just create the obj 1017 */ 1018 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1019 break; 1020 } else { 1021 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1022 break; 1023 } 1024 } 1025 1026 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1027 struct ras_common_if head = { 1028 .block = AMDGPU_RAS_BLOCK__MCA, 1029 .type = default_ras_type, 1030 .sub_block_index = i, 1031 }; 1032 1033 if (bypass) { 1034 /* 1035 * bypass psp. vbios enable ras for us. 1036 * so just create the obj 1037 */ 1038 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1039 break; 1040 } else { 1041 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1042 break; 1043 } 1044 } 1045 1046 return con->features; 1047 } 1048 /* feature ctl end */ 1049 1050 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1051 enum amdgpu_ras_block block) 1052 { 1053 if (!block_obj) 1054 return -EINVAL; 1055 1056 if (block_obj->ras_comm.block == block) 1057 return 0; 1058 1059 return -EINVAL; 1060 } 1061 1062 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1063 enum amdgpu_ras_block block, uint32_t sub_block_index) 1064 { 1065 struct amdgpu_ras_block_list *node, *tmp; 1066 struct amdgpu_ras_block_object *obj; 1067 1068 if (block >= AMDGPU_RAS_BLOCK__LAST) 1069 return NULL; 1070 1071 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1072 if (!node->ras_obj) { 1073 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1074 continue; 1075 } 1076 1077 obj = node->ras_obj; 1078 if (obj->ras_block_match) { 1079 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1080 return obj; 1081 } else { 1082 if (amdgpu_ras_block_match_default(obj, block) == 0) 1083 return obj; 1084 } 1085 } 1086 1087 return NULL; 1088 } 1089 1090 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1091 { 1092 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1093 int ret = 0; 1094 1095 /* 1096 * choosing right query method according to 1097 * whether smu support query error information 1098 */ 1099 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1100 if (ret == -EOPNOTSUPP) { 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1104 1105 /* umc query_ras_error_address is also responsible for clearing 1106 * error status 1107 */ 1108 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1109 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1110 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1111 } else if (!ret) { 1112 if (adev->umc.ras && 1113 adev->umc.ras->ecc_info_query_ras_error_count) 1114 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1115 1116 if (adev->umc.ras && 1117 adev->umc.ras->ecc_info_query_ras_error_address) 1118 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1119 } 1120 } 1121 1122 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1123 struct ras_manager *ras_mgr, 1124 struct ras_err_data *err_data, 1125 struct ras_query_context *qctx, 1126 const char *blk_name, 1127 bool is_ue, 1128 bool is_de) 1129 { 1130 struct amdgpu_smuio_mcm_config_info *mcm_info; 1131 struct ras_err_node *err_node; 1132 struct ras_err_info *err_info; 1133 u64 event_id = qctx->evid.event_id; 1134 1135 if (is_ue) { 1136 for_each_ras_error(err_node, err_data) { 1137 err_info = &err_node->err_info; 1138 mcm_info = &err_info->mcm_info; 1139 if (err_info->ue_count) { 1140 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1141 "%lld new uncorrectable hardware errors detected in %s block\n", 1142 mcm_info->socket_id, 1143 mcm_info->die_id, 1144 err_info->ue_count, 1145 blk_name); 1146 } 1147 } 1148 1149 for_each_ras_error(err_node, &ras_mgr->err_data) { 1150 err_info = &err_node->err_info; 1151 mcm_info = &err_info->mcm_info; 1152 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1153 "%lld uncorrectable hardware errors detected in total in %s block\n", 1154 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1155 } 1156 1157 } else { 1158 if (is_de) { 1159 for_each_ras_error(err_node, err_data) { 1160 err_info = &err_node->err_info; 1161 mcm_info = &err_info->mcm_info; 1162 if (err_info->de_count) { 1163 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1164 "%lld new deferred hardware errors detected in %s block\n", 1165 mcm_info->socket_id, 1166 mcm_info->die_id, 1167 err_info->de_count, 1168 blk_name); 1169 } 1170 } 1171 1172 for_each_ras_error(err_node, &ras_mgr->err_data) { 1173 err_info = &err_node->err_info; 1174 mcm_info = &err_info->mcm_info; 1175 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1176 "%lld deferred hardware errors detected in total in %s block\n", 1177 mcm_info->socket_id, mcm_info->die_id, 1178 err_info->de_count, blk_name); 1179 } 1180 } else { 1181 if (adev->debug_disable_ce_logs) 1182 return; 1183 1184 for_each_ras_error(err_node, err_data) { 1185 err_info = &err_node->err_info; 1186 mcm_info = &err_info->mcm_info; 1187 if (err_info->ce_count) { 1188 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1189 "%lld new correctable hardware errors detected in %s block\n", 1190 mcm_info->socket_id, 1191 mcm_info->die_id, 1192 err_info->ce_count, 1193 blk_name); 1194 } 1195 } 1196 1197 for_each_ras_error(err_node, &ras_mgr->err_data) { 1198 err_info = &err_node->err_info; 1199 mcm_info = &err_info->mcm_info; 1200 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1201 "%lld correctable hardware errors detected in total in %s block\n", 1202 mcm_info->socket_id, mcm_info->die_id, 1203 err_info->ce_count, blk_name); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline bool err_data_has_source_info(struct ras_err_data *data) 1210 { 1211 return !list_empty(&data->err_node_list); 1212 } 1213 1214 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1215 struct ras_query_if *query_if, 1216 struct ras_err_data *err_data, 1217 struct ras_query_context *qctx) 1218 { 1219 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1220 const char *blk_name = get_ras_block_str(&query_if->head); 1221 u64 event_id = qctx->evid.event_id; 1222 1223 if (err_data->ce_count) { 1224 if (err_data_has_source_info(err_data)) { 1225 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1226 blk_name, false, false); 1227 } else if (!adev->aid_mask && 1228 adev->smuio.funcs && 1229 adev->smuio.funcs->get_socket_id && 1230 adev->smuio.funcs->get_die_id) { 1231 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1232 "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 adev->smuio.funcs->get_socket_id(adev), 1235 adev->smuio.funcs->get_die_id(adev), 1236 ras_mgr->err_data.ce_count, 1237 blk_name); 1238 } else { 1239 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1240 "detected in %s block\n", 1241 ras_mgr->err_data.ce_count, 1242 blk_name); 1243 } 1244 } 1245 1246 if (err_data->ue_count) { 1247 if (err_data_has_source_info(err_data)) { 1248 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1249 blk_name, true, false); 1250 } else if (!adev->aid_mask && 1251 adev->smuio.funcs && 1252 adev->smuio.funcs->get_socket_id && 1253 adev->smuio.funcs->get_die_id) { 1254 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1255 "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 adev->smuio.funcs->get_socket_id(adev), 1258 adev->smuio.funcs->get_die_id(adev), 1259 ras_mgr->err_data.ue_count, 1260 blk_name); 1261 } else { 1262 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1263 "detected in %s block\n", 1264 ras_mgr->err_data.ue_count, 1265 blk_name); 1266 } 1267 } 1268 1269 if (err_data->de_count) { 1270 if (err_data_has_source_info(err_data)) { 1271 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1272 blk_name, false, true); 1273 } else if (!adev->aid_mask && 1274 adev->smuio.funcs && 1275 adev->smuio.funcs->get_socket_id && 1276 adev->smuio.funcs->get_die_id) { 1277 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1278 "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 adev->smuio.funcs->get_socket_id(adev), 1281 adev->smuio.funcs->get_die_id(adev), 1282 ras_mgr->err_data.de_count, 1283 blk_name); 1284 } else { 1285 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1286 "detected in %s block\n", 1287 ras_mgr->err_data.de_count, 1288 blk_name); 1289 } 1290 } 1291 } 1292 1293 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1294 struct ras_query_if *query_if, 1295 struct ras_err_data *err_data, 1296 struct ras_query_context *qctx) 1297 { 1298 unsigned long new_ue, new_ce, new_de; 1299 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1300 const char *blk_name = get_ras_block_str(&query_if->head); 1301 u64 event_id = qctx->evid.event_id; 1302 1303 new_ce = err_data->ce_count - obj->err_data.ce_count; 1304 new_ue = err_data->ue_count - obj->err_data.ue_count; 1305 new_de = err_data->de_count - obj->err_data.de_count; 1306 1307 if (new_ce) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1309 "detected in %s block\n", 1310 new_ce, 1311 blk_name); 1312 } 1313 1314 if (new_ue) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1316 "detected in %s block\n", 1317 new_ue, 1318 blk_name); 1319 } 1320 1321 if (new_de) { 1322 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1323 "detected in %s block\n", 1324 new_de, 1325 blk_name); 1326 } 1327 } 1328 1329 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1330 { 1331 struct ras_err_node *err_node; 1332 struct ras_err_info *err_info; 1333 1334 if (err_data_has_source_info(err_data)) { 1335 for_each_ras_error(err_node, err_data) { 1336 err_info = &err_node->err_info; 1337 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1338 &err_info->mcm_info, err_info->de_count); 1339 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->ce_count); 1341 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ue_count); 1343 } 1344 } else { 1345 /* for legacy asic path which doesn't has error source info */ 1346 obj->err_data.ue_count += err_data->ue_count; 1347 obj->err_data.ce_count += err_data->ce_count; 1348 obj->err_data.de_count += err_data->de_count; 1349 } 1350 } 1351 1352 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1353 struct ras_err_data *err_data) 1354 { 1355 /* Host reports absolute counts */ 1356 obj->err_data.ue_count = err_data->ue_count; 1357 obj->err_data.ce_count = err_data->ce_count; 1358 obj->err_data.de_count = err_data->de_count; 1359 } 1360 1361 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1362 { 1363 struct ras_common_if head; 1364 1365 memset(&head, 0, sizeof(head)); 1366 head.block = blk; 1367 1368 return amdgpu_ras_find_obj(adev, &head); 1369 } 1370 1371 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1372 const struct aca_info *aca_info, void *data) 1373 { 1374 struct ras_manager *obj; 1375 1376 /* in resume phase, no need to create aca fs node */ 1377 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1378 return 0; 1379 1380 obj = get_ras_manager(adev, blk); 1381 if (!obj) 1382 return -EINVAL; 1383 1384 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1385 } 1386 1387 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1388 { 1389 struct ras_manager *obj; 1390 1391 obj = get_ras_manager(adev, blk); 1392 if (!obj) 1393 return -EINVAL; 1394 1395 amdgpu_aca_remove_handle(&obj->aca_handle); 1396 1397 return 0; 1398 } 1399 1400 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1401 enum aca_error_type type, struct ras_err_data *err_data, 1402 struct ras_query_context *qctx) 1403 { 1404 struct ras_manager *obj; 1405 1406 obj = get_ras_manager(adev, blk); 1407 if (!obj) 1408 return -EINVAL; 1409 1410 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1411 } 1412 1413 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1414 struct aca_handle *handle, char *buf, void *data) 1415 { 1416 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1417 struct ras_query_if info = { 1418 .head = obj->head, 1419 }; 1420 1421 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1422 return sysfs_emit(buf, "Query currently inaccessible\n"); 1423 1424 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1425 return -EINVAL; 1426 1427 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1428 "ce", info.ce_count, "de", info.de_count); 1429 } 1430 1431 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1432 struct ras_query_if *info, 1433 struct ras_err_data *err_data, 1434 struct ras_query_context *qctx, 1435 unsigned int error_query_mode) 1436 { 1437 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1438 struct amdgpu_ras_block_object *block_obj = NULL; 1439 int ret; 1440 1441 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1442 return -EINVAL; 1443 1444 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1445 return -EINVAL; 1446 1447 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1448 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1449 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1450 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1451 amdgpu_ras_get_ecc_info(adev, err_data); 1452 } else { 1453 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1454 if (!block_obj || !block_obj->hw_ops) { 1455 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1456 get_ras_block_str(&info->head)); 1457 return -EINVAL; 1458 } 1459 1460 if (block_obj->hw_ops->query_ras_error_count) 1461 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1462 1463 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1464 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1465 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1466 if (block_obj->hw_ops->query_ras_error_status) 1467 block_obj->hw_ops->query_ras_error_status(adev); 1468 } 1469 } 1470 } else { 1471 if (amdgpu_aca_is_enabled(adev)) { 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 1476 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1477 if (ret) 1478 return ret; 1479 1480 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1481 if (ret) 1482 return ret; 1483 } else { 1484 /* FIXME: add code to check return value later */ 1485 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1486 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1487 } 1488 } 1489 1490 return 0; 1491 } 1492 1493 /* query/inject/cure begin */ 1494 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1495 struct ras_query_if *info, 1496 enum ras_event_type type) 1497 { 1498 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1499 struct ras_err_data err_data; 1500 struct ras_query_context qctx; 1501 unsigned int error_query_mode; 1502 int ret; 1503 1504 if (!obj) 1505 return -EINVAL; 1506 1507 ret = amdgpu_ras_error_data_init(&err_data); 1508 if (ret) 1509 return ret; 1510 1511 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1512 return -EINVAL; 1513 1514 memset(&qctx, 0, sizeof(qctx)); 1515 qctx.evid.type = type; 1516 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1517 1518 if (!down_read_trylock(&adev->reset_domain->sem)) { 1519 ret = -EIO; 1520 goto out_fini_err_data; 1521 } 1522 1523 ret = amdgpu_ras_query_error_status_helper(adev, info, 1524 &err_data, 1525 &qctx, 1526 error_query_mode); 1527 up_read(&adev->reset_domain->sem); 1528 if (ret) 1529 goto out_fini_err_data; 1530 1531 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1532 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1533 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1534 } else { 1535 /* Host provides absolute error counts. First generate the report 1536 * using the previous VF internal count against new host count. 1537 * Then Update VF internal count. 1538 */ 1539 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1540 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1541 } 1542 1543 info->ue_count = obj->err_data.ue_count; 1544 info->ce_count = obj->err_data.ce_count; 1545 info->de_count = obj->err_data.de_count; 1546 1547 out_fini_err_data: 1548 amdgpu_ras_error_data_fini(&err_data); 1549 1550 return ret; 1551 } 1552 1553 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1554 { 1555 struct ras_cmd_dev_handle req = {0}; 1556 int ret; 1557 1558 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1559 &req, sizeof(req), NULL, 0); 1560 if (ret) { 1561 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1562 return ret; 1563 } 1564 1565 return 0; 1566 } 1567 1568 static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1569 struct ras_query_if *info) 1570 { 1571 struct ras_cmd_block_ecc_info_req req = {0}; 1572 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1573 int ret; 1574 1575 if (!info) 1576 return -EINVAL; 1577 1578 req.block_id = info->head.block; 1579 req.subblock_id = info->head.sub_block_index; 1580 1581 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1582 &req, sizeof(req), &rsp, sizeof(rsp)); 1583 if (!ret) { 1584 info->ce_count = rsp.ce_count; 1585 info->ue_count = rsp.ue_count; 1586 info->de_count = rsp.de_count; 1587 } 1588 1589 return ret; 1590 } 1591 1592 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1593 { 1594 if (amdgpu_uniras_enabled(adev)) 1595 return amdgpu_uniras_query_block_ecc(adev, info); 1596 else 1597 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1598 } 1599 1600 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1601 enum amdgpu_ras_block block) 1602 { 1603 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1604 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1605 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1606 1607 if (!block_obj || !block_obj->hw_ops) { 1608 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1609 ras_block_str(block)); 1610 return -EOPNOTSUPP; 1611 } 1612 1613 if (!amdgpu_ras_is_supported(adev, block) || 1614 !amdgpu_ras_get_aca_debug_mode(adev)) 1615 return -EOPNOTSUPP; 1616 1617 if (amdgpu_sriov_vf(adev)) 1618 return -EOPNOTSUPP; 1619 1620 /* skip ras error reset in gpu reset */ 1621 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1622 ((smu_funcs && smu_funcs->set_debug_mode) || 1623 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1624 return -EOPNOTSUPP; 1625 1626 if (block_obj->hw_ops->reset_ras_error_count) 1627 block_obj->hw_ops->reset_ras_error_count(adev); 1628 1629 return 0; 1630 } 1631 1632 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1633 enum amdgpu_ras_block block) 1634 { 1635 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1636 1637 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1638 return 0; 1639 1640 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1641 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1642 if (block_obj->hw_ops->reset_ras_error_status) 1643 block_obj->hw_ops->reset_ras_error_status(adev); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1650 struct ras_inject_if *info) 1651 { 1652 struct ras_cmd_inject_error_req inject_req; 1653 struct ras_cmd_inject_error_rsp rsp; 1654 1655 if (!info) 1656 return -EINVAL; 1657 1658 memset(&inject_req, 0, sizeof(inject_req)); 1659 inject_req.block_id = info->head.block; 1660 inject_req.subblock_id = info->head.sub_block_index; 1661 inject_req.address = info->address; 1662 inject_req.error_type = info->head.type; 1663 inject_req.instance_mask = info->instance_mask; 1664 inject_req.method = info->value; 1665 1666 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1667 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1668 } 1669 1670 /* wrapper of psp_ras_trigger_error */ 1671 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1672 struct ras_inject_if *info) 1673 { 1674 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1675 struct ta_ras_trigger_error_input block_info = { 1676 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1677 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1678 .sub_block_index = info->head.sub_block_index, 1679 .address = info->address, 1680 .value = info->value, 1681 }; 1682 int ret = -EINVAL; 1683 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1684 info->head.block, 1685 info->head.sub_block_index); 1686 1687 if (amdgpu_uniras_enabled(adev)) 1688 return amdgpu_uniras_error_inject(adev, info); 1689 1690 /* inject on guest isn't allowed, return success directly */ 1691 if (amdgpu_sriov_vf(adev)) 1692 return 0; 1693 1694 if (!obj) 1695 return -EINVAL; 1696 1697 if (!block_obj || !block_obj->hw_ops) { 1698 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1699 get_ras_block_str(&info->head)); 1700 return -EINVAL; 1701 } 1702 1703 /* Calculate XGMI relative offset */ 1704 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1705 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1706 block_info.address = 1707 amdgpu_xgmi_get_relative_phy_addr(adev, 1708 block_info.address); 1709 } 1710 1711 if (block_obj->hw_ops->ras_error_inject) { 1712 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1713 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1714 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1715 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1716 info->instance_mask); 1717 } else { 1718 /* default path */ 1719 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1720 } 1721 1722 if (ret) 1723 dev_err(adev->dev, "ras inject %s failed %d\n", 1724 get_ras_block_str(&info->head), ret); 1725 1726 return ret; 1727 } 1728 1729 /** 1730 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1731 * @adev: pointer to AMD GPU device 1732 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1733 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1734 * @query_info: pointer to ras_query_if 1735 * 1736 * Return 0 for query success or do nothing, otherwise return an error 1737 * on failures 1738 */ 1739 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1740 unsigned long *ce_count, 1741 unsigned long *ue_count, 1742 struct ras_query_if *query_info) 1743 { 1744 int ret; 1745 1746 if (!query_info) 1747 /* do nothing if query_info is not specified */ 1748 return 0; 1749 1750 ret = amdgpu_ras_query_error_status(adev, query_info); 1751 if (ret) 1752 return ret; 1753 1754 *ce_count += query_info->ce_count; 1755 *ue_count += query_info->ue_count; 1756 1757 /* some hardware/IP supports read to clear 1758 * no need to explictly reset the err status after the query call */ 1759 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1760 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1761 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1762 dev_warn(adev->dev, 1763 "Failed to reset error counter and error status\n"); 1764 } 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1771 * @adev: pointer to AMD GPU device 1772 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1773 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1774 * errors. 1775 * @query_info: pointer to ras_query_if if the query request is only for 1776 * specific ip block; if info is NULL, then the qurey request is for 1777 * all the ip blocks that support query ras error counters/status 1778 * 1779 * If set, @ce_count or @ue_count, count and return the corresponding 1780 * error counts in those integer pointers. Return 0 if the device 1781 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1782 */ 1783 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1784 unsigned long *ce_count, 1785 unsigned long *ue_count, 1786 struct ras_query_if *query_info) 1787 { 1788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1789 struct ras_manager *obj; 1790 unsigned long ce, ue; 1791 int ret; 1792 1793 if (!adev->ras_enabled || !con) 1794 return -EOPNOTSUPP; 1795 1796 /* Don't count since no reporting. 1797 */ 1798 if (!ce_count && !ue_count) 1799 return 0; 1800 1801 ce = 0; 1802 ue = 0; 1803 if (!query_info) { 1804 /* query all the ip blocks that support ras query interface */ 1805 list_for_each_entry(obj, &con->head, node) { 1806 struct ras_query_if info = { 1807 .head = obj->head, 1808 }; 1809 1810 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1811 } 1812 } else { 1813 /* query specific ip block */ 1814 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1815 } 1816 1817 if (ret) 1818 return ret; 1819 1820 if (ce_count) 1821 *ce_count = ce; 1822 1823 if (ue_count) 1824 *ue_count = ue; 1825 1826 return 0; 1827 } 1828 /* query/inject/cure end */ 1829 1830 1831 /* sysfs begin */ 1832 1833 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1834 struct ras_badpage *bps, uint32_t count, uint32_t start); 1835 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837 1838 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1839 { 1840 switch (flags) { 1841 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1842 return "R"; 1843 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1844 return "P"; 1845 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1846 default: 1847 return "F"; 1848 } 1849 } 1850 1851 /** 1852 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1853 * 1854 * It allows user to read the bad pages of vram on the gpu through 1855 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1856 * 1857 * It outputs multiple lines, and each line stands for one gpu page. 1858 * 1859 * The format of one line is below, 1860 * gpu pfn : gpu page size : flags 1861 * 1862 * gpu pfn and gpu page size are printed in hex format. 1863 * flags can be one of below character, 1864 * 1865 * R: reserved, this gpu page is reserved and not able to use. 1866 * 1867 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1868 * in next window of page_reserve. 1869 * 1870 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1871 * 1872 * Examples: 1873 * 1874 * .. code-block:: bash 1875 * 1876 * 0x00000001 : 0x00001000 : R 1877 * 0x00000002 : 0x00001000 : P 1878 * 1879 */ 1880 1881 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1882 struct kobject *kobj, const struct bin_attribute *attr, 1883 char *buf, loff_t ppos, size_t count) 1884 { 1885 struct amdgpu_ras *con = 1886 container_of(attr, struct amdgpu_ras, badpages_attr); 1887 struct amdgpu_device *adev = con->adev; 1888 const unsigned int element_size = 1889 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1890 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1891 unsigned int end = div64_ul(ppos + count - 1, element_size); 1892 ssize_t s = 0; 1893 struct ras_badpage *bps = NULL; 1894 int bps_count = 0, i, status; 1895 uint64_t address; 1896 1897 memset(buf, 0, count); 1898 1899 bps_count = end - start; 1900 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1901 if (!bps) 1902 return 0; 1903 1904 memset(bps, 0, sizeof(*bps) * bps_count); 1905 1906 if (amdgpu_uniras_enabled(adev)) 1907 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1908 else 1909 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1910 1911 if (bps_count <= 0) { 1912 kfree(bps); 1913 return 0; 1914 } 1915 1916 for (i = 0; i < bps_count; i++) { 1917 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1918 if (amdgpu_ras_check_critical_address(adev, address)) 1919 continue; 1920 1921 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1922 1923 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1924 address); 1925 if (status == -EBUSY) 1926 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1927 else if (status == -ENOENT) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1929 else 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1931 1932 s += scnprintf(&buf[s], element_size + 1, 1933 "0x%08x : 0x%08x : %1s\n", 1934 bps[i].bp, 1935 bps[i].size, 1936 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1937 } 1938 1939 kfree(bps); 1940 1941 return s; 1942 } 1943 1944 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1945 struct device_attribute *attr, char *buf) 1946 { 1947 struct amdgpu_ras *con = 1948 container_of(attr, struct amdgpu_ras, features_attr); 1949 1950 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1951 } 1952 1953 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1954 u32 *minor, u32 *rev) 1955 { 1956 int i; 1957 1958 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1959 return false; 1960 1961 for (i = 0; i < adev->num_ip_blocks; i++) { 1962 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1963 *major = adev->ip_blocks[i].version->major; 1964 *minor = adev->ip_blocks[i].version->minor; 1965 *rev = adev->ip_blocks[i].version->rev; 1966 return true; 1967 } 1968 } 1969 1970 return false; 1971 } 1972 1973 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1974 struct device_attribute *attr, char *buf) 1975 { 1976 struct amdgpu_ras *con = 1977 container_of(attr, struct amdgpu_ras, version_attr); 1978 u32 major, minor, rev; 1979 ssize_t size = 0; 1980 1981 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1982 con->eeprom_control.tbl_hdr.version); 1983 1984 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1985 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1986 major, minor, rev); 1987 1988 return size; 1989 } 1990 1991 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1992 struct device_attribute *attr, char *buf) 1993 { 1994 struct amdgpu_ras *con = 1995 container_of(attr, struct amdgpu_ras, schema_attr); 1996 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1997 } 1998 1999 static struct { 2000 enum ras_event_type type; 2001 const char *name; 2002 } dump_event[] = { 2003 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2004 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2005 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2006 }; 2007 2008 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2009 struct device_attribute *attr, char *buf) 2010 { 2011 struct amdgpu_ras *con = 2012 container_of(attr, struct amdgpu_ras, event_state_attr); 2013 struct ras_event_manager *event_mgr = con->event_mgr; 2014 struct ras_event_state *event_state; 2015 int i, size = 0; 2016 2017 if (!event_mgr) 2018 return -EINVAL; 2019 2020 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2021 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2022 event_state = &event_mgr->event_state[dump_event[i].type]; 2023 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2024 dump_event[i].name, 2025 atomic64_read(&event_state->count), 2026 event_state->last_seqno); 2027 } 2028 2029 return (ssize_t)size; 2030 } 2031 2032 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2033 { 2034 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2035 2036 if (adev->dev->kobj.sd) 2037 sysfs_remove_file_from_group(&adev->dev->kobj, 2038 &con->badpages_attr.attr, 2039 RAS_FS_NAME); 2040 } 2041 2042 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2043 { 2044 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2045 struct attribute *attrs[] = { 2046 &con->features_attr.attr, 2047 &con->version_attr.attr, 2048 &con->schema_attr.attr, 2049 &con->event_state_attr.attr, 2050 NULL 2051 }; 2052 struct attribute_group group = { 2053 .name = RAS_FS_NAME, 2054 .attrs = attrs, 2055 }; 2056 2057 if (adev->dev->kobj.sd) 2058 sysfs_remove_group(&adev->dev->kobj, &group); 2059 2060 return 0; 2061 } 2062 2063 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2064 struct ras_common_if *head) 2065 { 2066 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2067 2068 if (amdgpu_aca_is_enabled(adev)) 2069 return 0; 2070 2071 if (!obj || obj->attr_inuse) 2072 return -EINVAL; 2073 2074 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2075 return 0; 2076 2077 get_obj(obj); 2078 2079 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2080 "%s_err_count", head->name); 2081 2082 obj->sysfs_attr = (struct device_attribute){ 2083 .attr = { 2084 .name = obj->fs_data.sysfs_name, 2085 .mode = S_IRUGO, 2086 }, 2087 .show = amdgpu_ras_sysfs_read, 2088 }; 2089 sysfs_attr_init(&obj->sysfs_attr.attr); 2090 2091 if (sysfs_add_file_to_group(&adev->dev->kobj, 2092 &obj->sysfs_attr.attr, 2093 RAS_FS_NAME)) { 2094 put_obj(obj); 2095 return -EINVAL; 2096 } 2097 2098 obj->attr_inuse = 1; 2099 2100 return 0; 2101 } 2102 2103 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2104 struct ras_common_if *head) 2105 { 2106 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2107 2108 if (amdgpu_aca_is_enabled(adev)) 2109 return 0; 2110 2111 if (!obj || !obj->attr_inuse) 2112 return -EINVAL; 2113 2114 if (adev->dev->kobj.sd) 2115 sysfs_remove_file_from_group(&adev->dev->kobj, 2116 &obj->sysfs_attr.attr, 2117 RAS_FS_NAME); 2118 obj->attr_inuse = 0; 2119 put_obj(obj); 2120 2121 return 0; 2122 } 2123 2124 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2125 { 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2127 struct ras_manager *obj, *tmp; 2128 2129 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2130 amdgpu_ras_sysfs_remove(adev, &obj->head); 2131 } 2132 2133 if (amdgpu_bad_page_threshold != 0) 2134 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2135 2136 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2137 2138 return 0; 2139 } 2140 /* sysfs end */ 2141 2142 /** 2143 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2144 * 2145 * Normally when there is an uncorrectable error, the driver will reset 2146 * the GPU to recover. However, in the event of an unrecoverable error, 2147 * the driver provides an interface to reboot the system automatically 2148 * in that event. 2149 * 2150 * The following file in debugfs provides that interface: 2151 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2152 * 2153 * Usage: 2154 * 2155 * .. code-block:: bash 2156 * 2157 * echo true > .../ras/auto_reboot 2158 * 2159 */ 2160 /* debugfs begin */ 2161 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2162 { 2163 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2164 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2165 struct drm_minor *minor = adev_to_drm(adev)->primary; 2166 struct dentry *dir; 2167 2168 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2169 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2170 &amdgpu_ras_debugfs_ctrl_ops); 2171 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_eeprom_ops); 2173 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2174 &con->bad_page_cnt_threshold); 2175 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2176 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2177 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2178 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2179 &amdgpu_ras_debugfs_eeprom_size_ops); 2180 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2181 S_IRUGO, dir, adev, 2182 &amdgpu_ras_debugfs_eeprom_table_ops); 2183 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2184 2185 /* 2186 * After one uncorrectable error happens, usually GPU recovery will 2187 * be scheduled. But due to the known problem in GPU recovery failing 2188 * to bring GPU back, below interface provides one direct way to 2189 * user to reboot system automatically in such case within 2190 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2191 * will never be called. 2192 */ 2193 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2194 2195 /* 2196 * User could set this not to clean up hardware's error count register 2197 * of RAS IPs during ras recovery. 2198 */ 2199 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2200 &con->disable_ras_err_cnt_harvest); 2201 return dir; 2202 } 2203 2204 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2205 struct ras_fs_if *head, 2206 struct dentry *dir) 2207 { 2208 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2209 2210 if (!obj || !dir) 2211 return; 2212 2213 get_obj(obj); 2214 2215 memcpy(obj->fs_data.debugfs_name, 2216 head->debugfs_name, 2217 sizeof(obj->fs_data.debugfs_name)); 2218 2219 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2220 obj, &amdgpu_ras_debugfs_ops); 2221 } 2222 2223 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2224 { 2225 bool ret; 2226 2227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2228 case IP_VERSION(13, 0, 6): 2229 case IP_VERSION(13, 0, 12): 2230 case IP_VERSION(13, 0, 14): 2231 ret = true; 2232 break; 2233 default: 2234 ret = false; 2235 break; 2236 } 2237 2238 return ret; 2239 } 2240 2241 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2242 { 2243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2244 struct dentry *dir; 2245 struct ras_manager *obj; 2246 struct ras_fs_if fs_info; 2247 2248 /* 2249 * it won't be called in resume path, no need to check 2250 * suspend and gpu reset status 2251 */ 2252 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2253 return; 2254 2255 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2256 2257 list_for_each_entry(obj, &con->head, node) { 2258 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2259 (obj->attr_inuse == 1)) { 2260 sprintf(fs_info.debugfs_name, "%s_err_inject", 2261 get_ras_block_str(&obj->head)); 2262 fs_info.head = obj->head; 2263 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2264 } 2265 } 2266 2267 if (amdgpu_ras_aca_is_supported(adev)) { 2268 if (amdgpu_aca_is_enabled(adev)) 2269 amdgpu_aca_smu_debugfs_init(adev, dir); 2270 else 2271 amdgpu_mca_smu_debugfs_init(adev, dir); 2272 } 2273 } 2274 2275 /* debugfs end */ 2276 2277 /* ras fs */ 2278 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2279 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2280 static DEVICE_ATTR(features, S_IRUGO, 2281 amdgpu_ras_sysfs_features_read, NULL); 2282 static DEVICE_ATTR(version, 0444, 2283 amdgpu_ras_sysfs_version_show, NULL); 2284 static DEVICE_ATTR(schema, 0444, 2285 amdgpu_ras_sysfs_schema_show, NULL); 2286 static DEVICE_ATTR(event_state, 0444, 2287 amdgpu_ras_sysfs_event_state_show, NULL); 2288 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2289 { 2290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2291 struct attribute_group group = { 2292 .name = RAS_FS_NAME, 2293 }; 2294 struct attribute *attrs[] = { 2295 &con->features_attr.attr, 2296 &con->version_attr.attr, 2297 &con->schema_attr.attr, 2298 &con->event_state_attr.attr, 2299 NULL 2300 }; 2301 const struct bin_attribute *bin_attrs[] = { 2302 NULL, 2303 NULL, 2304 }; 2305 int r; 2306 2307 group.attrs = attrs; 2308 2309 /* add features entry */ 2310 con->features_attr = dev_attr_features; 2311 sysfs_attr_init(attrs[0]); 2312 2313 /* add version entry */ 2314 con->version_attr = dev_attr_version; 2315 sysfs_attr_init(attrs[1]); 2316 2317 /* add schema entry */ 2318 con->schema_attr = dev_attr_schema; 2319 sysfs_attr_init(attrs[2]); 2320 2321 /* add event_state entry */ 2322 con->event_state_attr = dev_attr_event_state; 2323 sysfs_attr_init(attrs[3]); 2324 2325 if (amdgpu_bad_page_threshold != 0) { 2326 /* add bad_page_features entry */ 2327 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2328 sysfs_bin_attr_init(&con->badpages_attr); 2329 bin_attrs[0] = &con->badpages_attr; 2330 group.bin_attrs = bin_attrs; 2331 } 2332 2333 r = sysfs_create_group(&adev->dev->kobj, &group); 2334 if (r) 2335 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2336 2337 return 0; 2338 } 2339 2340 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2341 { 2342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2343 struct ras_manager *con_obj, *ip_obj, *tmp; 2344 2345 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2346 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2347 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2348 if (ip_obj) 2349 put_obj(ip_obj); 2350 } 2351 } 2352 2353 amdgpu_ras_sysfs_remove_all(adev); 2354 return 0; 2355 } 2356 /* ras fs end */ 2357 2358 /* ih begin */ 2359 2360 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2361 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2362 * register to check whether the interrupt is triggered or not, and properly 2363 * ack the interrupt if it is there 2364 */ 2365 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2366 { 2367 /* Fatal error events are handled on host side */ 2368 if (amdgpu_sriov_vf(adev)) 2369 return; 2370 /* 2371 * If the current interrupt is caused by a non-fatal RAS error, skip 2372 * check for fatal error. For fatal errors, FED status of all devices 2373 * in XGMI hive gets set when the first device gets fatal error 2374 * interrupt. The error gets propagated to other devices as well, so 2375 * make sure to ack the interrupt regardless of FED status. 2376 */ 2377 if (!amdgpu_ras_get_fed_status(adev) && 2378 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2379 return; 2380 2381 if (amdgpu_uniras_enabled(adev)) { 2382 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2383 return; 2384 } 2385 2386 if (adev->nbio.ras && 2387 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2388 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2389 2390 if (adev->nbio.ras && 2391 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2392 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2393 } 2394 2395 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2396 struct amdgpu_iv_entry *entry) 2397 { 2398 bool poison_stat = false; 2399 struct amdgpu_device *adev = obj->adev; 2400 struct amdgpu_ras_block_object *block_obj = 2401 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2402 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2403 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2404 u64 event_id; 2405 int ret; 2406 2407 if (!block_obj || !con) 2408 return; 2409 2410 ret = amdgpu_ras_mark_ras_event(adev, type); 2411 if (ret) 2412 return; 2413 2414 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2415 /* both query_poison_status and handle_poison_consumption are optional, 2416 * but at least one of them should be implemented if we need poison 2417 * consumption handler 2418 */ 2419 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2420 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2421 if (!poison_stat) { 2422 /* Not poison consumption interrupt, no need to handle it */ 2423 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2424 block_obj->ras_comm.name); 2425 2426 return; 2427 } 2428 } 2429 2430 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2431 2432 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2433 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2434 2435 /* gpu reset is fallback for failed and default cases. 2436 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2437 */ 2438 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2439 event_id = amdgpu_ras_acquire_event_id(adev, type); 2440 RAS_EVENT_LOG(adev, event_id, 2441 "GPU reset for %s RAS poison consumption is issued!\n", 2442 block_obj->ras_comm.name); 2443 amdgpu_ras_reset_gpu(adev); 2444 } 2445 2446 if (!poison_stat) 2447 amdgpu_gfx_poison_consumption_handler(adev, entry); 2448 } 2449 2450 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2451 struct amdgpu_iv_entry *entry) 2452 { 2453 struct amdgpu_device *adev = obj->adev; 2454 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2455 u64 event_id; 2456 int ret; 2457 2458 ret = amdgpu_ras_mark_ras_event(adev, type); 2459 if (ret) 2460 return; 2461 2462 event_id = amdgpu_ras_acquire_event_id(adev, type); 2463 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2464 2465 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2466 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2467 2468 atomic_inc(&con->page_retirement_req_cnt); 2469 atomic_inc(&con->poison_creation_count); 2470 2471 wake_up(&con->page_retirement_wq); 2472 } 2473 } 2474 2475 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2476 struct amdgpu_iv_entry *entry) 2477 { 2478 struct ras_ih_data *data = &obj->ih_data; 2479 struct ras_err_data err_data; 2480 int ret; 2481 2482 if (!data->cb) 2483 return; 2484 2485 ret = amdgpu_ras_error_data_init(&err_data); 2486 if (ret) 2487 return; 2488 2489 /* Let IP handle its data, maybe we need get the output 2490 * from the callback to update the error type/count, etc 2491 */ 2492 amdgpu_ras_set_fed(obj->adev, true); 2493 ret = data->cb(obj->adev, &err_data, entry); 2494 /* ue will trigger an interrupt, and in that case 2495 * we need do a reset to recovery the whole system. 2496 * But leave IP do that recovery, here we just dispatch 2497 * the error. 2498 */ 2499 if (ret == AMDGPU_RAS_SUCCESS) { 2500 /* these counts could be left as 0 if 2501 * some blocks do not count error number 2502 */ 2503 obj->err_data.ue_count += err_data.ue_count; 2504 obj->err_data.ce_count += err_data.ce_count; 2505 obj->err_data.de_count += err_data.de_count; 2506 } 2507 2508 amdgpu_ras_error_data_fini(&err_data); 2509 } 2510 2511 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2512 { 2513 struct ras_ih_data *data = &obj->ih_data; 2514 struct amdgpu_iv_entry entry; 2515 2516 while (data->rptr != data->wptr) { 2517 rmb(); 2518 memcpy(&entry, &data->ring[data->rptr], 2519 data->element_size); 2520 2521 wmb(); 2522 data->rptr = (data->aligned_element_size + 2523 data->rptr) % data->ring_size; 2524 2525 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2526 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2527 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2528 else 2529 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2530 } else { 2531 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2532 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2533 else 2534 dev_warn(obj->adev->dev, 2535 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2536 } 2537 } 2538 } 2539 2540 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2541 { 2542 struct ras_ih_data *data = 2543 container_of(work, struct ras_ih_data, ih_work); 2544 struct ras_manager *obj = 2545 container_of(data, struct ras_manager, ih_data); 2546 2547 amdgpu_ras_interrupt_handler(obj); 2548 } 2549 2550 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2551 struct ras_dispatch_if *info) 2552 { 2553 struct ras_manager *obj; 2554 struct ras_ih_data *data; 2555 2556 if (amdgpu_uniras_enabled(adev)) { 2557 struct ras_ih_info ih_info; 2558 2559 memset(&ih_info, 0, sizeof(ih_info)); 2560 ih_info.block = info->head.block; 2561 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2562 2563 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2564 } 2565 2566 obj = amdgpu_ras_find_obj(adev, &info->head); 2567 if (!obj) 2568 return -EINVAL; 2569 2570 data = &obj->ih_data; 2571 2572 if (data->inuse == 0) 2573 return 0; 2574 2575 /* Might be overflow... */ 2576 memcpy(&data->ring[data->wptr], info->entry, 2577 data->element_size); 2578 2579 wmb(); 2580 data->wptr = (data->aligned_element_size + 2581 data->wptr) % data->ring_size; 2582 2583 schedule_work(&data->ih_work); 2584 2585 return 0; 2586 } 2587 2588 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2589 struct ras_common_if *head) 2590 { 2591 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2592 struct ras_ih_data *data; 2593 2594 if (!obj) 2595 return -EINVAL; 2596 2597 data = &obj->ih_data; 2598 if (data->inuse == 0) 2599 return 0; 2600 2601 cancel_work_sync(&data->ih_work); 2602 2603 kfree(data->ring); 2604 memset(data, 0, sizeof(*data)); 2605 put_obj(obj); 2606 2607 return 0; 2608 } 2609 2610 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2611 struct ras_common_if *head) 2612 { 2613 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2614 struct ras_ih_data *data; 2615 struct amdgpu_ras_block_object *ras_obj; 2616 2617 if (!obj) { 2618 /* in case we registe the IH before enable ras feature */ 2619 obj = amdgpu_ras_create_obj(adev, head); 2620 if (!obj) 2621 return -EINVAL; 2622 } else 2623 get_obj(obj); 2624 2625 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2626 2627 data = &obj->ih_data; 2628 /* add the callback.etc */ 2629 *data = (struct ras_ih_data) { 2630 .inuse = 0, 2631 .cb = ras_obj->ras_cb, 2632 .element_size = sizeof(struct amdgpu_iv_entry), 2633 .rptr = 0, 2634 .wptr = 0, 2635 }; 2636 2637 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2638 2639 data->aligned_element_size = ALIGN(data->element_size, 8); 2640 /* the ring can store 64 iv entries. */ 2641 data->ring_size = 64 * data->aligned_element_size; 2642 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2643 if (!data->ring) { 2644 put_obj(obj); 2645 return -ENOMEM; 2646 } 2647 2648 /* IH is ready */ 2649 data->inuse = 1; 2650 2651 return 0; 2652 } 2653 2654 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2655 { 2656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2657 struct ras_manager *obj, *tmp; 2658 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2660 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2661 } 2662 2663 return 0; 2664 } 2665 /* ih end */ 2666 2667 /* traversal all IPs except NBIO to query error counter */ 2668 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2669 { 2670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2671 struct ras_manager *obj; 2672 2673 if (!adev->ras_enabled || !con) 2674 return; 2675 2676 list_for_each_entry(obj, &con->head, node) { 2677 struct ras_query_if info = { 2678 .head = obj->head, 2679 }; 2680 2681 /* 2682 * PCIE_BIF IP has one different isr by ras controller 2683 * interrupt, the specific ras counter query will be 2684 * done in that isr. So skip such block from common 2685 * sync flood interrupt isr calling. 2686 */ 2687 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2688 continue; 2689 2690 /* 2691 * this is a workaround for aldebaran, skip send msg to 2692 * smu to get ecc_info table due to smu handle get ecc 2693 * info table failed temporarily. 2694 * should be removed until smu fix handle ecc_info table. 2695 */ 2696 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2697 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2698 IP_VERSION(13, 0, 2))) 2699 continue; 2700 2701 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2702 2703 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2704 IP_VERSION(11, 0, 2) && 2705 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 4) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(13, 0, 0)) { 2709 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2710 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2711 } 2712 } 2713 } 2714 2715 /* Parse RdRspStatus and WrRspStatus */ 2716 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2717 struct ras_query_if *info) 2718 { 2719 struct amdgpu_ras_block_object *block_obj; 2720 /* 2721 * Only two block need to query read/write 2722 * RspStatus at current state 2723 */ 2724 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2725 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2726 return; 2727 2728 block_obj = amdgpu_ras_get_ras_block(adev, 2729 info->head.block, 2730 info->head.sub_block_index); 2731 2732 if (!block_obj || !block_obj->hw_ops) { 2733 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2734 get_ras_block_str(&info->head)); 2735 return; 2736 } 2737 2738 if (block_obj->hw_ops->query_ras_error_status) 2739 block_obj->hw_ops->query_ras_error_status(adev); 2740 2741 } 2742 2743 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2744 { 2745 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2746 struct ras_manager *obj; 2747 2748 if (!adev->ras_enabled || !con) 2749 return; 2750 2751 list_for_each_entry(obj, &con->head, node) { 2752 struct ras_query_if info = { 2753 .head = obj->head, 2754 }; 2755 2756 amdgpu_ras_error_status_query(adev, &info); 2757 } 2758 } 2759 2760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2761 struct ras_badpage *bps, uint32_t count, uint32_t start) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_err_handler_data *data; 2765 int r = 0; 2766 uint32_t i; 2767 2768 if (!con || !con->eh_data || !bps || !count) 2769 return -EINVAL; 2770 2771 mutex_lock(&con->recovery_lock); 2772 data = con->eh_data; 2773 if (start < data->count) { 2774 for (i = start; i < data->count; i++) { 2775 if (!data->bps[i].ts) 2776 continue; 2777 2778 bps[r].bp = data->bps[i].retired_page; 2779 r++; 2780 if (r >= count) 2781 break; 2782 } 2783 } 2784 mutex_unlock(&con->recovery_lock); 2785 2786 return r; 2787 } 2788 2789 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2790 struct ras_badpage *bps, uint32_t count, uint32_t start) 2791 { 2792 struct ras_cmd_bad_pages_info_req cmd_input; 2793 struct ras_cmd_bad_pages_info_rsp *output; 2794 uint32_t group, start_group, end_group; 2795 uint32_t pos, pos_in_group; 2796 int r = 0, i; 2797 2798 if (!bps || !count) 2799 return -EINVAL; 2800 2801 output = kmalloc(sizeof(*output), GFP_KERNEL); 2802 if (!output) 2803 return -ENOMEM; 2804 2805 memset(&cmd_input, 0, sizeof(cmd_input)); 2806 2807 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2808 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2809 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2810 2811 pos = start; 2812 for (group = start_group; group < end_group; group++) { 2813 memset(output, 0, sizeof(*output)); 2814 cmd_input.group_index = group; 2815 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2816 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2817 goto out; 2818 2819 if (pos >= output->bp_total_cnt) 2820 goto out; 2821 2822 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2823 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2824 if (!output->records[i].ts) 2825 continue; 2826 2827 bps[r].bp = output->records[i].retired_page; 2828 r++; 2829 if (r >= count) 2830 goto out; 2831 } 2832 } 2833 2834 out: 2835 kfree(output); 2836 return r; 2837 } 2838 2839 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2840 struct amdgpu_hive_info *hive, bool status) 2841 { 2842 struct amdgpu_device *tmp_adev; 2843 2844 if (hive) { 2845 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2846 amdgpu_ras_set_fed(tmp_adev, status); 2847 } else { 2848 amdgpu_ras_set_fed(adev, status); 2849 } 2850 } 2851 2852 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2853 { 2854 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2856 int hive_ras_recovery = 0; 2857 2858 if (hive) { 2859 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2860 amdgpu_put_xgmi_hive(hive); 2861 } 2862 2863 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2864 return true; 2865 2866 return false; 2867 } 2868 2869 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2870 { 2871 if (amdgpu_ras_intr_triggered()) 2872 return RAS_EVENT_TYPE_FATAL; 2873 else 2874 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2875 } 2876 2877 static void amdgpu_ras_do_recovery(struct work_struct *work) 2878 { 2879 struct amdgpu_ras *ras = 2880 container_of(work, struct amdgpu_ras, recovery_work); 2881 struct amdgpu_device *remote_adev = NULL; 2882 struct amdgpu_device *adev = ras->adev; 2883 struct list_head device_list, *device_list_handle = NULL; 2884 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2885 unsigned int error_query_mode; 2886 enum ras_event_type type; 2887 2888 if (hive) { 2889 atomic_set(&hive->ras_recovery, 1); 2890 2891 /* If any device which is part of the hive received RAS fatal 2892 * error interrupt, set fatal error status on all. This 2893 * condition will need a recovery, and flag will be cleared 2894 * as part of recovery. 2895 */ 2896 list_for_each_entry(remote_adev, &hive->device_list, 2897 gmc.xgmi.head) 2898 if (amdgpu_ras_get_fed_status(remote_adev)) { 2899 amdgpu_ras_set_fed_all(adev, hive, true); 2900 break; 2901 } 2902 } 2903 if (!ras->disable_ras_err_cnt_harvest) { 2904 2905 /* Build list of devices to query RAS related errors */ 2906 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2907 device_list_handle = &hive->device_list; 2908 } else { 2909 INIT_LIST_HEAD(&device_list); 2910 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2911 device_list_handle = &device_list; 2912 } 2913 2914 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2915 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2916 /* wait 500ms to ensure pmfw polling mca bank info done */ 2917 msleep(500); 2918 } 2919 } 2920 2921 type = amdgpu_ras_get_fatal_error_event(adev); 2922 list_for_each_entry(remote_adev, 2923 device_list_handle, gmc.xgmi.head) { 2924 if (amdgpu_uniras_enabled(remote_adev)) { 2925 amdgpu_ras_mgr_update_ras_ecc(remote_adev); 2926 } else { 2927 amdgpu_ras_query_err_status(remote_adev); 2928 amdgpu_ras_log_on_err_counter(remote_adev, type); 2929 } 2930 } 2931 2932 } 2933 2934 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2935 struct amdgpu_reset_context reset_context; 2936 memset(&reset_context, 0, sizeof(reset_context)); 2937 2938 reset_context.method = AMD_RESET_METHOD_NONE; 2939 reset_context.reset_req_dev = adev; 2940 reset_context.src = AMDGPU_RESET_SRC_RAS; 2941 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2942 2943 /* Perform full reset in fatal error mode */ 2944 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2945 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2946 else { 2947 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2948 2949 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2950 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2951 reset_context.method = AMD_RESET_METHOD_MODE2; 2952 } 2953 2954 /* Fatal error occurs in poison mode, mode1 reset is used to 2955 * recover gpu. 2956 */ 2957 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2958 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2959 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2960 2961 psp_fatal_error_recovery_quirk(&adev->psp); 2962 } 2963 } 2964 2965 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2966 } 2967 atomic_set(&ras->in_recovery, 0); 2968 if (hive) { 2969 atomic_set(&hive->ras_recovery, 0); 2970 amdgpu_put_xgmi_hive(hive); 2971 } 2972 } 2973 2974 /* alloc/realloc bps array */ 2975 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2976 struct ras_err_handler_data *data, int pages) 2977 { 2978 unsigned int old_space = data->count + data->space_left; 2979 unsigned int new_space = old_space + pages; 2980 unsigned int align_space = ALIGN(new_space, 512); 2981 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2982 2983 if (!bps) { 2984 return -ENOMEM; 2985 } 2986 2987 if (data->bps) { 2988 memcpy(bps, data->bps, 2989 data->count * sizeof(*data->bps)); 2990 kfree(data->bps); 2991 } 2992 2993 data->bps = bps; 2994 data->space_left += align_space - old_space; 2995 return 0; 2996 } 2997 2998 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2999 struct eeprom_table_record *bps, 3000 struct ras_err_data *err_data) 3001 { 3002 struct ta_ras_query_address_input addr_in; 3003 uint32_t socket = 0; 3004 int ret = 0; 3005 3006 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3007 socket = adev->smuio.funcs->get_socket_id(adev); 3008 3009 /* reinit err_data */ 3010 err_data->err_addr_cnt = 0; 3011 err_data->err_addr_len = adev->umc.retire_unit; 3012 3013 memset(&addr_in, 0, sizeof(addr_in)); 3014 addr_in.ma.err_addr = bps->address; 3015 addr_in.ma.socket_id = socket; 3016 addr_in.ma.ch_inst = bps->mem_channel; 3017 /* tell RAS TA the node instance is not used */ 3018 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3019 3020 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3021 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3022 &addr_in, NULL, false); 3023 3024 return ret; 3025 } 3026 3027 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3028 struct eeprom_table_record *bps, 3029 struct ras_err_data *err_data) 3030 { 3031 struct ta_ras_query_address_input addr_in; 3032 uint32_t die_id, socket = 0; 3033 3034 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3035 socket = adev->smuio.funcs->get_socket_id(adev); 3036 3037 /* although die id is gotten from PA in nps1 mode, the id is 3038 * fitable for any nps mode 3039 */ 3040 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3041 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3042 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3043 else 3044 return -EINVAL; 3045 3046 /* reinit err_data */ 3047 err_data->err_addr_cnt = 0; 3048 err_data->err_addr_len = adev->umc.retire_unit; 3049 3050 memset(&addr_in, 0, sizeof(addr_in)); 3051 addr_in.ma.err_addr = bps->address; 3052 addr_in.ma.ch_inst = bps->mem_channel; 3053 addr_in.ma.umc_inst = bps->mcumc_id; 3054 addr_in.ma.node_inst = die_id; 3055 addr_in.ma.socket_id = socket; 3056 3057 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3058 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3059 &addr_in, NULL, false); 3060 else 3061 return -EINVAL; 3062 } 3063 3064 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3065 struct eeprom_table_record *bps, int count) 3066 { 3067 int j; 3068 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3069 struct ras_err_handler_data *data = con->eh_data; 3070 3071 for (j = 0; j < count; j++) { 3072 if (amdgpu_ras_check_bad_page_unlock(con, 3073 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3074 data->count++; 3075 data->space_left--; 3076 continue; 3077 } 3078 3079 if (!data->space_left && 3080 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3081 return -ENOMEM; 3082 } 3083 3084 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3085 3086 memcpy(&data->bps[data->count], &(bps[j]), 3087 sizeof(struct eeprom_table_record)); 3088 data->count++; 3089 data->space_left--; 3090 con->bad_page_num++; 3091 } 3092 3093 return 0; 3094 } 3095 3096 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3097 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3098 enum amdgpu_memory_partition nps) 3099 { 3100 int i = 0; 3101 enum amdgpu_memory_partition save_nps; 3102 3103 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3104 3105 /*old asics just have pa in eeprom*/ 3106 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3107 memcpy(err_data->err_addr, bps, 3108 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3109 goto out; 3110 } 3111 3112 for (i = 0; i < adev->umc.retire_unit; i++) 3113 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3114 3115 if (save_nps) { 3116 if (save_nps == nps) { 3117 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3118 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3119 return -EINVAL; 3120 for (i = 0; i < adev->umc.retire_unit; i++) { 3121 err_data->err_addr[i].address = bps[0].address; 3122 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3123 err_data->err_addr[i].bank = bps[0].bank; 3124 err_data->err_addr[i].err_type = bps[0].err_type; 3125 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3126 } 3127 } else { 3128 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3129 return -EINVAL; 3130 } 3131 } else { 3132 if (bps[0].address == 0) { 3133 /* for specific old eeprom data, mca address is not stored, 3134 * calc it from pa 3135 */ 3136 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3137 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3138 return -EINVAL; 3139 } 3140 3141 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3142 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3143 memcpy(err_data->err_addr, bps, 3144 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3145 else 3146 return -EOPNOTSUPP; 3147 } 3148 } 3149 3150 out: 3151 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3152 } 3153 3154 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3155 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3156 enum amdgpu_memory_partition nps) 3157 { 3158 int i = 0; 3159 enum amdgpu_memory_partition save_nps; 3160 3161 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3162 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3163 3164 if (save_nps == nps) { 3165 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3166 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3167 return -EINVAL; 3168 for (i = 0; i < adev->umc.retire_unit; i++) { 3169 err_data->err_addr[i].address = bps->address; 3170 err_data->err_addr[i].mem_channel = bps->mem_channel; 3171 err_data->err_addr[i].bank = bps->bank; 3172 err_data->err_addr[i].err_type = bps->err_type; 3173 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3174 } 3175 } else { 3176 if (bps->address) { 3177 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3178 return -EINVAL; 3179 } else { 3180 /* for specific old eeprom data, mca address is not stored, 3181 * calc it from pa 3182 */ 3183 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3184 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3185 return -EINVAL; 3186 3187 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3188 return -EOPNOTSUPP; 3189 } 3190 } 3191 3192 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3193 adev->umc.retire_unit); 3194 } 3195 3196 /* it deal with vram only. */ 3197 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3198 struct eeprom_table_record *bps, int pages, bool from_rom) 3199 { 3200 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3201 struct ras_err_data err_data; 3202 struct amdgpu_ras_eeprom_control *control = 3203 &adev->psp.ras_context.ras->eeprom_control; 3204 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3205 int ret = 0; 3206 uint32_t i = 0; 3207 3208 if (!con || !con->eh_data || !bps || pages <= 0) 3209 return 0; 3210 3211 if (from_rom) { 3212 err_data.err_addr = 3213 kcalloc(adev->umc.retire_unit, 3214 sizeof(struct eeprom_table_record), GFP_KERNEL); 3215 if (!err_data.err_addr) { 3216 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3217 return -ENOMEM; 3218 } 3219 3220 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3221 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3222 } 3223 3224 mutex_lock(&con->recovery_lock); 3225 3226 if (from_rom) { 3227 /* there is no pa recs in V3, so skip pa recs processing */ 3228 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3229 for (i = 0; i < pages; i++) { 3230 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3231 if ((bps[i].address == bps[i + 1].address) && 3232 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3233 /* deal with retire_unit records a time */ 3234 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3235 &bps[i], &err_data, nps); 3236 if (ret) 3237 con->bad_page_num -= adev->umc.retire_unit; 3238 i += (adev->umc.retire_unit - 1); 3239 } else { 3240 break; 3241 } 3242 } else { 3243 break; 3244 } 3245 } 3246 } 3247 for (; i < pages; i++) { 3248 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3249 &bps[i], &err_data, nps); 3250 if (ret) 3251 con->bad_page_num -= adev->umc.retire_unit; 3252 } 3253 3254 con->eh_data->count_saved = con->eh_data->count; 3255 } else { 3256 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3257 } 3258 3259 if (from_rom) 3260 kfree(err_data.err_addr); 3261 mutex_unlock(&con->recovery_lock); 3262 3263 return ret; 3264 } 3265 3266 /* 3267 * write error record array to eeprom, the function should be 3268 * protected by recovery_lock 3269 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3270 */ 3271 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3272 unsigned long *new_cnt) 3273 { 3274 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3275 struct ras_err_handler_data *data; 3276 struct amdgpu_ras_eeprom_control *control; 3277 int save_count, unit_num, i; 3278 3279 if (!con || !con->eh_data) { 3280 if (new_cnt) 3281 *new_cnt = 0; 3282 3283 return 0; 3284 } 3285 3286 if (!con->eeprom_control.is_eeprom_valid) { 3287 dev_warn(adev->dev, 3288 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3289 if (new_cnt) 3290 *new_cnt = 0; 3291 3292 return 0; 3293 } 3294 3295 mutex_lock(&con->recovery_lock); 3296 control = &con->eeprom_control; 3297 data = con->eh_data; 3298 unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; 3299 save_count = con->bad_page_num - control->ras_num_bad_pages; 3300 mutex_unlock(&con->recovery_lock); 3301 3302 if (new_cnt) 3303 *new_cnt = unit_num; 3304 3305 /* only new entries are saved */ 3306 if (unit_num && save_count) { 3307 /*old asics only save pa to eeprom like before*/ 3308 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3309 if (amdgpu_ras_eeprom_append(control, 3310 &data->bps[data->count_saved], unit_num)) { 3311 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3312 return -EIO; 3313 } 3314 } else { 3315 for (i = 0; i < unit_num; i++) { 3316 if (amdgpu_ras_eeprom_append(control, 3317 &data->bps[data->count_saved + 3318 i * adev->umc.retire_unit], 1)) { 3319 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3320 return -EIO; 3321 } 3322 } 3323 } 3324 3325 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3326 data->count_saved = data->count; 3327 } 3328 3329 return 0; 3330 } 3331 3332 /* 3333 * read error record array in eeprom and reserve enough space for 3334 * storing new bad pages 3335 */ 3336 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3337 { 3338 struct amdgpu_ras_eeprom_control *control = 3339 &adev->psp.ras_context.ras->eeprom_control; 3340 struct eeprom_table_record *bps; 3341 int ret, i = 0; 3342 3343 /* no bad page record, skip eeprom access */ 3344 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3345 return 0; 3346 3347 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3348 if (!bps) 3349 return -ENOMEM; 3350 3351 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3352 if (ret) { 3353 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3354 } else { 3355 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3356 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3357 as pa recs, so add verion check to avoid it. 3358 */ 3359 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3360 for (i = 0; i < control->ras_num_recs; i++) { 3361 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3362 if ((bps[i].address == bps[i + 1].address) && 3363 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3364 control->ras_num_pa_recs += adev->umc.retire_unit; 3365 i += (adev->umc.retire_unit - 1); 3366 } else { 3367 control->ras_num_mca_recs += 3368 (control->ras_num_recs - i); 3369 break; 3370 } 3371 } else { 3372 control->ras_num_mca_recs += (control->ras_num_recs - i); 3373 break; 3374 } 3375 } 3376 } else { 3377 control->ras_num_mca_recs = control->ras_num_recs; 3378 } 3379 } 3380 3381 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3382 if (ret) 3383 goto out; 3384 3385 ret = amdgpu_ras_eeprom_check(control); 3386 if (ret) 3387 goto out; 3388 3389 /* HW not usable */ 3390 if (amdgpu_ras_is_rma(adev)) 3391 ret = -EHWPOISON; 3392 } 3393 3394 out: 3395 kfree(bps); 3396 return ret; 3397 } 3398 3399 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3400 uint64_t addr) 3401 { 3402 struct ras_err_handler_data *data = con->eh_data; 3403 struct amdgpu_device *adev = con->adev; 3404 int i; 3405 3406 if ((addr >= adev->gmc.mc_vram_size && 3407 adev->gmc.mc_vram_size) || 3408 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3409 return -EINVAL; 3410 3411 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3412 for (i = 0; i < data->count; i++) 3413 if (addr == data->bps[i].retired_page) 3414 return 1; 3415 3416 return 0; 3417 } 3418 3419 /* 3420 * check if an address belongs to bad page 3421 * 3422 * Note: this check is only for umc block 3423 */ 3424 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3425 uint64_t addr) 3426 { 3427 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3428 int ret = 0; 3429 3430 if (!con || !con->eh_data) 3431 return ret; 3432 3433 mutex_lock(&con->recovery_lock); 3434 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3435 mutex_unlock(&con->recovery_lock); 3436 return ret; 3437 } 3438 3439 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3440 uint32_t max_count) 3441 { 3442 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3443 3444 /* 3445 * amdgpu_bad_page_threshold is used to config 3446 * the threshold for the number of bad pages. 3447 * -1: Threshold is set to default value 3448 * Driver will issue a warning message when threshold is reached 3449 * and continue runtime services. 3450 * 0: Disable bad page retirement 3451 * Driver will not retire bad pages 3452 * which is intended for debugging purpose. 3453 * -2: Threshold is determined by a formula 3454 * that assumes 1 bad page per 100M of local memory. 3455 * Driver will continue runtime services when threhold is reached. 3456 * 0 < threshold < max number of bad page records in EEPROM, 3457 * A user-defined threshold is set 3458 * Driver will halt runtime services when this custom threshold is reached. 3459 */ 3460 if (amdgpu_bad_page_threshold == -2) { 3461 u64 val = adev->gmc.mc_vram_size; 3462 3463 do_div(val, RAS_BAD_PAGE_COVER); 3464 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3465 max_count); 3466 } else if (amdgpu_bad_page_threshold == -1) { 3467 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3468 } else { 3469 con->bad_page_cnt_threshold = min_t(int, max_count, 3470 amdgpu_bad_page_threshold); 3471 } 3472 } 3473 3474 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3475 enum amdgpu_ras_block block, uint16_t pasid, 3476 pasid_notify pasid_fn, void *data, uint32_t reset) 3477 { 3478 int ret = 0; 3479 struct ras_poison_msg poison_msg; 3480 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3481 3482 memset(&poison_msg, 0, sizeof(poison_msg)); 3483 poison_msg.block = block; 3484 poison_msg.pasid = pasid; 3485 poison_msg.reset = reset; 3486 poison_msg.pasid_fn = pasid_fn; 3487 poison_msg.data = data; 3488 3489 ret = kfifo_put(&con->poison_fifo, poison_msg); 3490 if (!ret) { 3491 dev_err(adev->dev, "Poison message fifo is full!\n"); 3492 return -ENOSPC; 3493 } 3494 3495 return 0; 3496 } 3497 3498 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3499 struct ras_poison_msg *poison_msg) 3500 { 3501 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3502 3503 return kfifo_get(&con->poison_fifo, poison_msg); 3504 } 3505 3506 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3507 { 3508 mutex_init(&ecc_log->lock); 3509 3510 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3511 ecc_log->de_queried_count = 0; 3512 ecc_log->consumption_q_count = 0; 3513 } 3514 3515 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3516 { 3517 struct radix_tree_iter iter; 3518 void __rcu **slot; 3519 struct ras_ecc_err *ecc_err; 3520 3521 mutex_lock(&ecc_log->lock); 3522 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3523 ecc_err = radix_tree_deref_slot(slot); 3524 kfree(ecc_err->err_pages.pfn); 3525 kfree(ecc_err); 3526 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3527 } 3528 mutex_unlock(&ecc_log->lock); 3529 3530 mutex_destroy(&ecc_log->lock); 3531 ecc_log->de_queried_count = 0; 3532 ecc_log->consumption_q_count = 0; 3533 } 3534 3535 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3536 uint32_t delayed_ms) 3537 { 3538 int ret; 3539 3540 mutex_lock(&con->umc_ecc_log.lock); 3541 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3542 UMC_ECC_NEW_DETECTED_TAG); 3543 mutex_unlock(&con->umc_ecc_log.lock); 3544 3545 if (ret) 3546 schedule_delayed_work(&con->page_retirement_dwork, 3547 msecs_to_jiffies(delayed_ms)); 3548 3549 return ret ? true : false; 3550 } 3551 3552 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3553 { 3554 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3555 page_retirement_dwork.work); 3556 struct amdgpu_device *adev = con->adev; 3557 struct ras_err_data err_data; 3558 3559 /* If gpu reset is ongoing, delay retiring the bad pages */ 3560 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3561 amdgpu_ras_schedule_retirement_dwork(con, 3562 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3563 return; 3564 } 3565 3566 amdgpu_ras_error_data_init(&err_data); 3567 3568 amdgpu_umc_handle_bad_pages(adev, &err_data); 3569 3570 amdgpu_ras_error_data_fini(&err_data); 3571 3572 amdgpu_ras_schedule_retirement_dwork(con, 3573 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3574 } 3575 3576 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3577 uint32_t poison_creation_count) 3578 { 3579 int ret = 0; 3580 struct ras_ecc_log_info *ecc_log; 3581 struct ras_query_if info; 3582 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3583 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3584 u64 de_queried_count; 3585 u64 consumption_q_count; 3586 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3587 3588 memset(&info, 0, sizeof(info)); 3589 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3590 3591 ecc_log = &ras->umc_ecc_log; 3592 ecc_log->de_queried_count = 0; 3593 ecc_log->consumption_q_count = 0; 3594 3595 do { 3596 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3597 if (ret) 3598 return ret; 3599 3600 de_queried_count = ecc_log->de_queried_count; 3601 consumption_q_count = ecc_log->consumption_q_count; 3602 3603 if (de_queried_count && consumption_q_count) 3604 break; 3605 3606 msleep(100); 3607 } while (--timeout); 3608 3609 if (de_queried_count) 3610 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3611 3612 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3613 amdgpu_ras_reset_gpu(adev); 3614 3615 return 0; 3616 } 3617 3618 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3619 { 3620 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3621 struct ras_poison_msg msg; 3622 int ret; 3623 3624 do { 3625 ret = kfifo_get(&con->poison_fifo, &msg); 3626 } while (ret); 3627 } 3628 3629 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3630 uint32_t msg_count, uint32_t *gpu_reset) 3631 { 3632 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3633 uint32_t reset_flags = 0, reset = 0; 3634 struct ras_poison_msg msg; 3635 int ret, i; 3636 3637 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3638 3639 for (i = 0; i < msg_count; i++) { 3640 ret = amdgpu_ras_get_poison_req(adev, &msg); 3641 if (!ret) 3642 continue; 3643 3644 if (msg.pasid_fn) 3645 msg.pasid_fn(adev, msg.pasid, msg.data); 3646 3647 reset_flags |= msg.reset; 3648 } 3649 3650 /* 3651 * Try to ensure poison creation handler is completed first 3652 * to set rma if bad page exceed threshold. 3653 */ 3654 flush_delayed_work(&con->page_retirement_dwork); 3655 3656 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3657 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3658 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3659 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3660 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3661 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3662 else 3663 reset = reset_flags; 3664 3665 con->gpu_reset_flags |= reset; 3666 amdgpu_ras_reset_gpu(adev); 3667 3668 *gpu_reset = reset; 3669 3670 /* Wait for gpu recovery to complete */ 3671 flush_work(&con->recovery_work); 3672 } 3673 3674 return 0; 3675 } 3676 3677 static int amdgpu_ras_page_retirement_thread(void *param) 3678 { 3679 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3680 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3681 uint32_t poison_creation_count, msg_count; 3682 uint32_t gpu_reset; 3683 int ret; 3684 3685 while (!kthread_should_stop()) { 3686 3687 wait_event_interruptible(con->page_retirement_wq, 3688 kthread_should_stop() || 3689 atomic_read(&con->page_retirement_req_cnt)); 3690 3691 if (kthread_should_stop()) 3692 break; 3693 3694 mutex_lock(&con->poison_lock); 3695 gpu_reset = 0; 3696 3697 do { 3698 poison_creation_count = atomic_read(&con->poison_creation_count); 3699 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3700 if (ret == -EIO) 3701 break; 3702 3703 if (poison_creation_count) { 3704 atomic_sub(poison_creation_count, &con->poison_creation_count); 3705 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3706 } 3707 } while (atomic_read(&con->poison_creation_count) && 3708 !atomic_read(&con->poison_consumption_count)); 3709 3710 if (ret != -EIO) { 3711 msg_count = kfifo_len(&con->poison_fifo); 3712 if (msg_count) { 3713 ret = amdgpu_ras_poison_consumption_handler(adev, 3714 msg_count, &gpu_reset); 3715 if ((ret != -EIO) && 3716 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3717 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3718 } 3719 } 3720 3721 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3722 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3723 /* Clear poison creation request */ 3724 atomic_set(&con->poison_creation_count, 0); 3725 atomic_set(&con->poison_consumption_count, 0); 3726 3727 /* Clear poison fifo */ 3728 amdgpu_ras_clear_poison_fifo(adev); 3729 3730 /* Clear all poison requests */ 3731 atomic_set(&con->page_retirement_req_cnt, 0); 3732 3733 if (ret == -EIO) { 3734 /* Wait for mode-1 reset to complete */ 3735 down_read(&adev->reset_domain->sem); 3736 up_read(&adev->reset_domain->sem); 3737 } 3738 3739 /* Wake up work to save bad pages to eeprom */ 3740 schedule_delayed_work(&con->page_retirement_dwork, 0); 3741 } else if (gpu_reset) { 3742 /* gpu just completed mode-2 reset or other reset */ 3743 /* Clear poison consumption messages cached in fifo */ 3744 msg_count = kfifo_len(&con->poison_fifo); 3745 if (msg_count) { 3746 amdgpu_ras_clear_poison_fifo(adev); 3747 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3748 } 3749 3750 atomic_set(&con->poison_consumption_count, 0); 3751 3752 /* Wake up work to save bad pages to eeprom */ 3753 schedule_delayed_work(&con->page_retirement_dwork, 0); 3754 } 3755 mutex_unlock(&con->poison_lock); 3756 } 3757 3758 return 0; 3759 } 3760 3761 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3762 { 3763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3764 struct amdgpu_ras_eeprom_control *control; 3765 int ret; 3766 3767 if (!con || amdgpu_sriov_vf(adev)) 3768 return 0; 3769 3770 if (amdgpu_uniras_enabled(adev)) 3771 return 0; 3772 3773 control = &con->eeprom_control; 3774 con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev); 3775 3776 ret = amdgpu_ras_eeprom_init(control); 3777 control->is_eeprom_valid = !ret; 3778 3779 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3780 control->ras_num_pa_recs = control->ras_num_recs; 3781 3782 if (adev->umc.ras && 3783 adev->umc.ras->get_retire_flip_bits) 3784 adev->umc.ras->get_retire_flip_bits(adev); 3785 3786 if (control->ras_num_recs && control->is_eeprom_valid) { 3787 ret = amdgpu_ras_load_bad_pages(adev); 3788 if (ret) { 3789 control->is_eeprom_valid = false; 3790 return 0; 3791 } 3792 3793 amdgpu_dpm_send_hbm_bad_pages_num( 3794 adev, control->ras_num_bad_pages); 3795 3796 if (con->update_channel_flag == true) { 3797 amdgpu_dpm_send_hbm_bad_channel_flag( 3798 adev, control->bad_channel_bitmap); 3799 con->update_channel_flag = false; 3800 } 3801 3802 /* The format action is only applied to new ASICs */ 3803 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3804 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3805 if (!amdgpu_ras_eeprom_reset_table(control)) 3806 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3807 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3808 } 3809 3810 return 0; 3811 } 3812 3813 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3814 { 3815 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3816 struct ras_err_handler_data **data; 3817 u32 max_eeprom_records_count = 0; 3818 int ret; 3819 3820 if (!con || amdgpu_sriov_vf(adev)) 3821 return 0; 3822 3823 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3824 * supports RAS and debugfs is enabled, but when 3825 * adev->ras_enabled is unset, i.e. when "ras_enable" 3826 * module parameter is set to 0. 3827 */ 3828 con->adev = adev; 3829 3830 if (!adev->ras_enabled) 3831 return 0; 3832 3833 data = &con->eh_data; 3834 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3835 if (!*data) { 3836 ret = -ENOMEM; 3837 goto out; 3838 } 3839 3840 mutex_init(&con->recovery_lock); 3841 mutex_init(&con->poison_lock); 3842 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3843 atomic_set(&con->in_recovery, 0); 3844 atomic_set(&con->rma_in_recovery, 0); 3845 con->eeprom_control.bad_channel_bitmap = 0; 3846 3847 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3848 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3849 3850 if (init_bp_info) { 3851 ret = amdgpu_ras_init_badpage_info(adev); 3852 if (ret) 3853 goto free; 3854 } 3855 3856 mutex_init(&con->page_rsv_lock); 3857 INIT_KFIFO(con->poison_fifo); 3858 mutex_init(&con->page_retirement_lock); 3859 init_waitqueue_head(&con->page_retirement_wq); 3860 atomic_set(&con->page_retirement_req_cnt, 0); 3861 atomic_set(&con->poison_creation_count, 0); 3862 atomic_set(&con->poison_consumption_count, 0); 3863 con->page_retirement_thread = 3864 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3865 if (IS_ERR(con->page_retirement_thread)) { 3866 con->page_retirement_thread = NULL; 3867 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3868 } 3869 3870 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3871 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3872 #ifdef CONFIG_X86_MCE_AMD 3873 if ((adev->asic_type == CHIP_ALDEBARAN) && 3874 (adev->gmc.xgmi.connected_to_cpu)) 3875 amdgpu_register_bad_pages_mca_notifier(adev); 3876 #endif 3877 return 0; 3878 3879 free: 3880 kfree((*data)->bps); 3881 kfree(*data); 3882 con->eh_data = NULL; 3883 out: 3884 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3885 3886 /* 3887 * Except error threshold exceeding case, other failure cases in this 3888 * function would not fail amdgpu driver init. 3889 */ 3890 if (!amdgpu_ras_is_rma(adev)) 3891 ret = 0; 3892 else 3893 ret = -EINVAL; 3894 3895 return ret; 3896 } 3897 3898 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3899 { 3900 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3901 struct ras_err_handler_data *data = con->eh_data; 3902 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3903 bool ret; 3904 3905 /* recovery_init failed to init it, fini is useless */ 3906 if (!data) 3907 return 0; 3908 3909 /* Save all cached bad pages to eeprom */ 3910 do { 3911 flush_delayed_work(&con->page_retirement_dwork); 3912 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3913 } while (ret && max_flush_timeout--); 3914 3915 if (con->page_retirement_thread) 3916 kthread_stop(con->page_retirement_thread); 3917 3918 atomic_set(&con->page_retirement_req_cnt, 0); 3919 atomic_set(&con->poison_creation_count, 0); 3920 3921 mutex_destroy(&con->page_rsv_lock); 3922 3923 cancel_work_sync(&con->recovery_work); 3924 3925 cancel_delayed_work_sync(&con->page_retirement_dwork); 3926 3927 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3928 3929 mutex_lock(&con->recovery_lock); 3930 con->eh_data = NULL; 3931 kfree(data->bps); 3932 kfree(data); 3933 mutex_unlock(&con->recovery_lock); 3934 3935 amdgpu_ras_critical_region_init(adev); 3936 3937 return 0; 3938 } 3939 /* recovery end */ 3940 3941 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3942 { 3943 if (amdgpu_sriov_vf(adev)) { 3944 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3945 case IP_VERSION(13, 0, 2): 3946 case IP_VERSION(13, 0, 6): 3947 case IP_VERSION(13, 0, 12): 3948 case IP_VERSION(13, 0, 14): 3949 return true; 3950 default: 3951 return false; 3952 } 3953 } 3954 3955 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3956 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3957 case IP_VERSION(13, 0, 0): 3958 case IP_VERSION(13, 0, 6): 3959 case IP_VERSION(13, 0, 10): 3960 case IP_VERSION(13, 0, 12): 3961 case IP_VERSION(13, 0, 14): 3962 case IP_VERSION(14, 0, 3): 3963 return true; 3964 default: 3965 return false; 3966 } 3967 } 3968 3969 return adev->asic_type == CHIP_VEGA10 || 3970 adev->asic_type == CHIP_VEGA20 || 3971 adev->asic_type == CHIP_ARCTURUS || 3972 adev->asic_type == CHIP_ALDEBARAN || 3973 adev->asic_type == CHIP_SIENNA_CICHLID; 3974 } 3975 3976 /* 3977 * this is workaround for vega20 workstation sku, 3978 * force enable gfx ras, ignore vbios gfx ras flag 3979 * due to GC EDC can not write 3980 */ 3981 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3982 { 3983 struct atom_context *ctx = adev->mode_info.atom_context; 3984 3985 if (!ctx) 3986 return; 3987 3988 if (strnstr(ctx->vbios_pn, "D16406", 3989 sizeof(ctx->vbios_pn)) || 3990 strnstr(ctx->vbios_pn, "D36002", 3991 sizeof(ctx->vbios_pn))) 3992 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3993 } 3994 3995 /* Query ras capablity via atomfirmware interface */ 3996 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3997 { 3998 /* mem_ecc cap */ 3999 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 4000 dev_info(adev->dev, "MEM ECC is active.\n"); 4001 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 4002 1 << AMDGPU_RAS_BLOCK__DF); 4003 } else { 4004 dev_info(adev->dev, "MEM ECC is not presented.\n"); 4005 } 4006 4007 /* sram_ecc cap */ 4008 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4009 dev_info(adev->dev, "SRAM ECC is active.\n"); 4010 if (!amdgpu_sriov_vf(adev)) 4011 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4012 1 << AMDGPU_RAS_BLOCK__DF); 4013 else 4014 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4015 1 << AMDGPU_RAS_BLOCK__SDMA | 4016 1 << AMDGPU_RAS_BLOCK__GFX); 4017 4018 /* 4019 * VCN/JPEG RAS can be supported on both bare metal and 4020 * SRIOV environment 4021 */ 4022 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4023 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4024 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4025 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4026 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4027 1 << AMDGPU_RAS_BLOCK__JPEG); 4028 else 4029 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4030 1 << AMDGPU_RAS_BLOCK__JPEG); 4031 4032 /* 4033 * XGMI RAS is not supported if xgmi num physical nodes 4034 * is zero 4035 */ 4036 if (!adev->gmc.xgmi.num_physical_nodes) 4037 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4038 } else { 4039 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4040 } 4041 } 4042 4043 /* Query poison mode from umc/df IP callbacks */ 4044 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4045 { 4046 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4047 bool df_poison, umc_poison; 4048 4049 /* poison setting is useless on SRIOV guest */ 4050 if (amdgpu_sriov_vf(adev) || !con) 4051 return; 4052 4053 /* Init poison supported flag, the default value is false */ 4054 if (adev->gmc.xgmi.connected_to_cpu || 4055 adev->gmc.is_app_apu) { 4056 /* enabled by default when GPU is connected to CPU */ 4057 con->poison_supported = true; 4058 } else if (adev->df.funcs && 4059 adev->df.funcs->query_ras_poison_mode && 4060 adev->umc.ras && 4061 adev->umc.ras->query_ras_poison_mode) { 4062 df_poison = 4063 adev->df.funcs->query_ras_poison_mode(adev); 4064 umc_poison = 4065 adev->umc.ras->query_ras_poison_mode(adev); 4066 4067 /* Only poison is set in both DF and UMC, we can support it */ 4068 if (df_poison && umc_poison) 4069 con->poison_supported = true; 4070 else if (df_poison != umc_poison) 4071 dev_warn(adev->dev, 4072 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4073 df_poison, umc_poison); 4074 } 4075 } 4076 4077 /* 4078 * check hardware's ras ability which will be saved in hw_supported. 4079 * if hardware does not support ras, we can skip some ras initializtion and 4080 * forbid some ras operations from IP. 4081 * if software itself, say boot parameter, limit the ras ability. We still 4082 * need allow IP do some limited operations, like disable. In such case, 4083 * we have to initialize ras as normal. but need check if operation is 4084 * allowed or not in each function. 4085 */ 4086 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4087 { 4088 adev->ras_hw_enabled = adev->ras_enabled = 0; 4089 4090 if (!amdgpu_ras_asic_supported(adev)) 4091 return; 4092 4093 if (amdgpu_sriov_vf(adev)) { 4094 if (amdgpu_virt_get_ras_capability(adev)) 4095 goto init_ras_enabled_flag; 4096 } 4097 4098 /* query ras capability from psp */ 4099 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4100 goto init_ras_enabled_flag; 4101 4102 /* query ras capablity from bios */ 4103 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4104 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4105 } else { 4106 /* driver only manages a few IP blocks RAS feature 4107 * when GPU is connected cpu through XGMI */ 4108 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4109 1 << AMDGPU_RAS_BLOCK__SDMA | 4110 1 << AMDGPU_RAS_BLOCK__MMHUB); 4111 } 4112 4113 /* apply asic specific settings (vega20 only for now) */ 4114 amdgpu_ras_get_quirks(adev); 4115 4116 /* query poison mode from umc/df ip callback */ 4117 amdgpu_ras_query_poison_mode(adev); 4118 4119 init_ras_enabled_flag: 4120 /* hw_supported needs to be aligned with RAS block mask. */ 4121 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4122 4123 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4124 adev->ras_hw_enabled & amdgpu_ras_mask; 4125 4126 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4127 if (!amdgpu_sriov_vf(adev)) { 4128 adev->aca.is_enabled = 4129 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4130 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4131 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4132 } 4133 4134 /* bad page feature is not applicable to specific app platform */ 4135 if (adev->gmc.is_app_apu && 4136 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4137 amdgpu_bad_page_threshold = 0; 4138 } 4139 4140 static void amdgpu_ras_counte_dw(struct work_struct *work) 4141 { 4142 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4143 ras_counte_delay_work.work); 4144 struct amdgpu_device *adev = con->adev; 4145 struct drm_device *dev = adev_to_drm(adev); 4146 unsigned long ce_count, ue_count; 4147 int res; 4148 4149 res = pm_runtime_get_sync(dev->dev); 4150 if (res < 0) 4151 goto Out; 4152 4153 /* Cache new values. 4154 */ 4155 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4156 atomic_set(&con->ras_ce_count, ce_count); 4157 atomic_set(&con->ras_ue_count, ue_count); 4158 } 4159 4160 Out: 4161 pm_runtime_put_autosuspend(dev->dev); 4162 } 4163 4164 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4165 { 4166 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4167 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4168 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4169 AMDGPU_RAS_ERROR__PARITY; 4170 } 4171 4172 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4173 { 4174 struct ras_event_state *event_state; 4175 int i; 4176 4177 memset(mgr, 0, sizeof(*mgr)); 4178 atomic64_set(&mgr->seqno, 0); 4179 4180 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4181 event_state = &mgr->event_state[i]; 4182 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4183 atomic64_set(&event_state->count, 0); 4184 } 4185 } 4186 4187 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4188 { 4189 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4190 struct amdgpu_hive_info *hive; 4191 4192 if (!ras) 4193 return; 4194 4195 hive = amdgpu_get_xgmi_hive(adev); 4196 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4197 4198 /* init event manager with node 0 on xgmi system */ 4199 if (!amdgpu_reset_in_recovery(adev)) { 4200 if (!hive || adev->gmc.xgmi.node_id == 0) 4201 ras_event_mgr_init(ras->event_mgr); 4202 } 4203 4204 if (hive) 4205 amdgpu_put_xgmi_hive(hive); 4206 } 4207 4208 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4209 { 4210 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4211 4212 if (!con || (adev->flags & AMD_IS_APU)) 4213 return; 4214 4215 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4216 case IP_VERSION(13, 0, 2): 4217 case IP_VERSION(13, 0, 6): 4218 case IP_VERSION(13, 0, 12): 4219 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4220 break; 4221 case IP_VERSION(13, 0, 14): 4222 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4223 break; 4224 default: 4225 break; 4226 } 4227 } 4228 4229 int amdgpu_ras_init(struct amdgpu_device *adev) 4230 { 4231 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4232 int r; 4233 4234 if (con) 4235 return 0; 4236 4237 con = kzalloc(sizeof(*con) + 4238 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4239 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4240 GFP_KERNEL); 4241 if (!con) 4242 return -ENOMEM; 4243 4244 con->adev = adev; 4245 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4246 atomic_set(&con->ras_ce_count, 0); 4247 atomic_set(&con->ras_ue_count, 0); 4248 4249 con->objs = (struct ras_manager *)(con + 1); 4250 4251 amdgpu_ras_set_context(adev, con); 4252 4253 amdgpu_ras_check_supported(adev); 4254 4255 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4256 /* set gfx block ras context feature for VEGA20 Gaming 4257 * send ras disable cmd to ras ta during ras late init. 4258 */ 4259 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4260 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4261 4262 return 0; 4263 } 4264 4265 r = 0; 4266 goto release_con; 4267 } 4268 4269 con->update_channel_flag = false; 4270 con->features = 0; 4271 con->schema = 0; 4272 INIT_LIST_HEAD(&con->head); 4273 /* Might need get this flag from vbios. */ 4274 con->flags = RAS_DEFAULT_FLAGS; 4275 4276 /* initialize nbio ras function ahead of any other 4277 * ras functions so hardware fatal error interrupt 4278 * can be enabled as early as possible */ 4279 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4280 case IP_VERSION(7, 4, 0): 4281 case IP_VERSION(7, 4, 1): 4282 case IP_VERSION(7, 4, 4): 4283 if (!adev->gmc.xgmi.connected_to_cpu) 4284 adev->nbio.ras = &nbio_v7_4_ras; 4285 break; 4286 case IP_VERSION(4, 3, 0): 4287 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4288 /* unlike other generation of nbio ras, 4289 * nbio v4_3 only support fatal error interrupt 4290 * to inform software that DF is freezed due to 4291 * system fatal error event. driver should not 4292 * enable nbio ras in such case. Instead, 4293 * check DF RAS */ 4294 adev->nbio.ras = &nbio_v4_3_ras; 4295 break; 4296 case IP_VERSION(6, 3, 1): 4297 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4298 /* unlike other generation of nbio ras, 4299 * nbif v6_3_1 only support fatal error interrupt 4300 * to inform software that DF is freezed due to 4301 * system fatal error event. driver should not 4302 * enable nbio ras in such case. Instead, 4303 * check DF RAS 4304 */ 4305 adev->nbio.ras = &nbif_v6_3_1_ras; 4306 break; 4307 case IP_VERSION(7, 9, 0): 4308 case IP_VERSION(7, 9, 1): 4309 if (!adev->gmc.is_app_apu) 4310 adev->nbio.ras = &nbio_v7_9_ras; 4311 break; 4312 default: 4313 /* nbio ras is not available */ 4314 break; 4315 } 4316 4317 /* nbio ras block needs to be enabled ahead of other ras blocks 4318 * to handle fatal error */ 4319 r = amdgpu_nbio_ras_sw_init(adev); 4320 if (r) 4321 return r; 4322 4323 if (adev->nbio.ras && 4324 adev->nbio.ras->init_ras_controller_interrupt) { 4325 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4326 if (r) 4327 goto release_con; 4328 } 4329 4330 if (adev->nbio.ras && 4331 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4332 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4333 if (r) 4334 goto release_con; 4335 } 4336 4337 /* Packed socket_id to ras feature mask bits[31:29] */ 4338 if (adev->smuio.funcs && 4339 adev->smuio.funcs->get_socket_id) 4340 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4341 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4342 4343 /* Get RAS schema for particular SOC */ 4344 con->schema = amdgpu_get_ras_schema(adev); 4345 4346 amdgpu_ras_init_reserved_vram_size(adev); 4347 4348 if (amdgpu_ras_fs_init(adev)) { 4349 r = -EINVAL; 4350 goto release_con; 4351 } 4352 4353 if (amdgpu_ras_aca_is_supported(adev)) { 4354 if (amdgpu_aca_is_enabled(adev)) 4355 r = amdgpu_aca_init(adev); 4356 else 4357 r = amdgpu_mca_init(adev); 4358 if (r) 4359 goto release_con; 4360 } 4361 4362 con->init_task_pid = task_pid_nr(current); 4363 get_task_comm(con->init_task_comm, current); 4364 4365 mutex_init(&con->critical_region_lock); 4366 INIT_LIST_HEAD(&con->critical_region_head); 4367 4368 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4369 "hardware ability[%x] ras_mask[%x]\n", 4370 adev->ras_hw_enabled, adev->ras_enabled); 4371 4372 return 0; 4373 release_con: 4374 amdgpu_ras_set_context(adev, NULL); 4375 kfree(con); 4376 4377 return r; 4378 } 4379 4380 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4381 { 4382 if (adev->gmc.xgmi.connected_to_cpu || 4383 adev->gmc.is_app_apu) 4384 return 1; 4385 return 0; 4386 } 4387 4388 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4389 struct ras_common_if *ras_block) 4390 { 4391 struct ras_query_if info = { 4392 .head = *ras_block, 4393 }; 4394 4395 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4396 return 0; 4397 4398 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4399 DRM_WARN("RAS init harvest failure"); 4400 4401 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4402 DRM_WARN("RAS init harvest reset failure"); 4403 4404 return 0; 4405 } 4406 4407 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4408 { 4409 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4410 4411 if (!con) 4412 return false; 4413 4414 return con->poison_supported; 4415 } 4416 4417 /* helper function to handle common stuff in ip late init phase */ 4418 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4419 struct ras_common_if *ras_block) 4420 { 4421 struct amdgpu_ras_block_object *ras_obj = NULL; 4422 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4423 struct ras_query_if *query_info; 4424 unsigned long ue_count, ce_count; 4425 int r; 4426 4427 /* disable RAS feature per IP block if it is not supported */ 4428 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4429 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4430 return 0; 4431 } 4432 4433 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4434 if (r) { 4435 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4436 /* in resume phase, if fail to enable ras, 4437 * clean up all ras fs nodes, and disable ras */ 4438 goto cleanup; 4439 } else 4440 return r; 4441 } 4442 4443 /* check for errors on warm reset edc persisant supported ASIC */ 4444 amdgpu_persistent_edc_harvesting(adev, ras_block); 4445 4446 /* in resume phase, no need to create ras fs node */ 4447 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4448 return 0; 4449 4450 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4451 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4452 (ras_obj->hw_ops->query_poison_status || 4453 ras_obj->hw_ops->handle_poison_consumption))) { 4454 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4455 if (r) 4456 goto cleanup; 4457 } 4458 4459 if (ras_obj->hw_ops && 4460 (ras_obj->hw_ops->query_ras_error_count || 4461 ras_obj->hw_ops->query_ras_error_status)) { 4462 r = amdgpu_ras_sysfs_create(adev, ras_block); 4463 if (r) 4464 goto interrupt; 4465 4466 /* Those are the cached values at init. 4467 */ 4468 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4469 if (!query_info) 4470 return -ENOMEM; 4471 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4472 4473 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4474 atomic_set(&con->ras_ce_count, ce_count); 4475 atomic_set(&con->ras_ue_count, ue_count); 4476 } 4477 4478 kfree(query_info); 4479 } 4480 4481 return 0; 4482 4483 interrupt: 4484 if (ras_obj->ras_cb) 4485 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4486 cleanup: 4487 amdgpu_ras_feature_enable(adev, ras_block, 0); 4488 return r; 4489 } 4490 4491 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4492 struct ras_common_if *ras_block) 4493 { 4494 return amdgpu_ras_block_late_init(adev, ras_block); 4495 } 4496 4497 /* helper function to remove ras fs node and interrupt handler */ 4498 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4499 struct ras_common_if *ras_block) 4500 { 4501 struct amdgpu_ras_block_object *ras_obj; 4502 if (!ras_block) 4503 return; 4504 4505 amdgpu_ras_sysfs_remove(adev, ras_block); 4506 4507 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4508 if (ras_obj->ras_cb) 4509 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4510 } 4511 4512 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4513 struct ras_common_if *ras_block) 4514 { 4515 return amdgpu_ras_block_late_fini(adev, ras_block); 4516 } 4517 4518 /* do some init work after IP late init as dependence. 4519 * and it runs in resume/gpu reset/booting up cases. 4520 */ 4521 void amdgpu_ras_resume(struct amdgpu_device *adev) 4522 { 4523 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4524 struct ras_manager *obj, *tmp; 4525 4526 if (!adev->ras_enabled || !con) { 4527 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4528 amdgpu_release_ras_context(adev); 4529 4530 return; 4531 } 4532 4533 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4534 /* Set up all other IPs which are not implemented. There is a 4535 * tricky thing that IP's actual ras error type should be 4536 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4537 * ERROR_NONE make sense anyway. 4538 */ 4539 amdgpu_ras_enable_all_features(adev, 1); 4540 4541 /* We enable ras on all hw_supported block, but as boot 4542 * parameter might disable some of them and one or more IP has 4543 * not implemented yet. So we disable them on behalf. 4544 */ 4545 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4546 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4547 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4548 /* there should be no any reference. */ 4549 WARN_ON(alive_obj(obj)); 4550 } 4551 } 4552 } 4553 } 4554 4555 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4556 { 4557 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4558 4559 if (!adev->ras_enabled || !con) 4560 return; 4561 4562 amdgpu_ras_disable_all_features(adev, 0); 4563 /* Make sure all ras objects are disabled. */ 4564 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4565 amdgpu_ras_disable_all_features(adev, 1); 4566 } 4567 4568 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4569 { 4570 struct amdgpu_ras_block_list *node, *tmp; 4571 struct amdgpu_ras_block_object *obj; 4572 int r; 4573 4574 amdgpu_ras_event_mgr_init(adev); 4575 4576 if (amdgpu_ras_aca_is_supported(adev)) { 4577 if (amdgpu_reset_in_recovery(adev)) { 4578 if (amdgpu_aca_is_enabled(adev)) 4579 r = amdgpu_aca_reset(adev); 4580 else 4581 r = amdgpu_mca_reset(adev); 4582 if (r) 4583 return r; 4584 } 4585 4586 if (!amdgpu_sriov_vf(adev)) { 4587 if (amdgpu_aca_is_enabled(adev)) 4588 amdgpu_ras_set_aca_debug_mode(adev, false); 4589 else 4590 amdgpu_ras_set_mca_debug_mode(adev, false); 4591 } 4592 } 4593 4594 /* Guest side doesn't need init ras feature */ 4595 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4596 return 0; 4597 4598 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4599 obj = node->ras_obj; 4600 if (!obj) { 4601 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4602 continue; 4603 } 4604 4605 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4606 continue; 4607 4608 if (obj->ras_late_init) { 4609 r = obj->ras_late_init(adev, &obj->ras_comm); 4610 if (r) { 4611 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4612 obj->ras_comm.name, r); 4613 return r; 4614 } 4615 } else 4616 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4617 } 4618 4619 return 0; 4620 } 4621 4622 /* do some fini work before IP fini as dependence */ 4623 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4624 { 4625 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4626 4627 if (!adev->ras_enabled || !con) 4628 return 0; 4629 4630 4631 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4632 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4633 amdgpu_ras_disable_all_features(adev, 0); 4634 amdgpu_ras_recovery_fini(adev); 4635 return 0; 4636 } 4637 4638 int amdgpu_ras_fini(struct amdgpu_device *adev) 4639 { 4640 struct amdgpu_ras_block_list *ras_node, *tmp; 4641 struct amdgpu_ras_block_object *obj = NULL; 4642 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4643 4644 if (!adev->ras_enabled || !con) 4645 return 0; 4646 4647 amdgpu_ras_critical_region_fini(adev); 4648 mutex_destroy(&con->critical_region_lock); 4649 4650 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4651 if (ras_node->ras_obj) { 4652 obj = ras_node->ras_obj; 4653 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4654 obj->ras_fini) 4655 obj->ras_fini(adev, &obj->ras_comm); 4656 else 4657 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4658 } 4659 4660 /* Clear ras blocks from ras_list and free ras block list node */ 4661 list_del(&ras_node->node); 4662 kfree(ras_node); 4663 } 4664 4665 amdgpu_ras_fs_fini(adev); 4666 amdgpu_ras_interrupt_remove_all(adev); 4667 4668 if (amdgpu_ras_aca_is_supported(adev)) { 4669 if (amdgpu_aca_is_enabled(adev)) 4670 amdgpu_aca_fini(adev); 4671 else 4672 amdgpu_mca_fini(adev); 4673 } 4674 4675 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4676 4677 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4678 amdgpu_ras_disable_all_features(adev, 0); 4679 4680 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4681 4682 amdgpu_ras_set_context(adev, NULL); 4683 kfree(con); 4684 4685 return 0; 4686 } 4687 4688 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4689 { 4690 struct amdgpu_ras *ras; 4691 4692 ras = amdgpu_ras_get_context(adev); 4693 if (!ras) 4694 return false; 4695 4696 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4697 } 4698 4699 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4700 { 4701 struct amdgpu_ras *ras; 4702 4703 ras = amdgpu_ras_get_context(adev); 4704 if (ras) { 4705 if (status) 4706 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4707 else 4708 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4709 } 4710 } 4711 4712 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4713 { 4714 struct amdgpu_ras *ras; 4715 4716 ras = amdgpu_ras_get_context(adev); 4717 if (ras) { 4718 ras->ras_err_state = 0; 4719 ras->gpu_reset_flags = 0; 4720 } 4721 } 4722 4723 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4724 enum amdgpu_ras_block block) 4725 { 4726 struct amdgpu_ras *ras; 4727 4728 ras = amdgpu_ras_get_context(adev); 4729 if (ras) 4730 set_bit(block, &ras->ras_err_state); 4731 } 4732 4733 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4734 { 4735 struct amdgpu_ras *ras; 4736 4737 ras = amdgpu_ras_get_context(adev); 4738 if (ras) { 4739 if (block == AMDGPU_RAS_BLOCK__ANY) 4740 return (ras->ras_err_state != 0); 4741 else 4742 return test_bit(block, &ras->ras_err_state) || 4743 test_bit(AMDGPU_RAS_BLOCK__LAST, 4744 &ras->ras_err_state); 4745 } 4746 4747 return false; 4748 } 4749 4750 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4751 { 4752 struct amdgpu_ras *ras; 4753 4754 ras = amdgpu_ras_get_context(adev); 4755 if (!ras) 4756 return NULL; 4757 4758 return ras->event_mgr; 4759 } 4760 4761 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4762 const void *caller) 4763 { 4764 struct ras_event_manager *event_mgr; 4765 struct ras_event_state *event_state; 4766 int ret = 0; 4767 4768 if (amdgpu_uniras_enabled(adev)) 4769 return 0; 4770 4771 if (type >= RAS_EVENT_TYPE_COUNT) { 4772 ret = -EINVAL; 4773 goto out; 4774 } 4775 4776 event_mgr = __get_ras_event_mgr(adev); 4777 if (!event_mgr) { 4778 ret = -EINVAL; 4779 goto out; 4780 } 4781 4782 event_state = &event_mgr->event_state[type]; 4783 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4784 atomic64_inc(&event_state->count); 4785 4786 out: 4787 if (ret && caller) 4788 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4789 (int)type, caller, ret); 4790 4791 return ret; 4792 } 4793 4794 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4795 { 4796 struct ras_event_manager *event_mgr; 4797 u64 id; 4798 4799 if (type >= RAS_EVENT_TYPE_COUNT) 4800 return RAS_EVENT_INVALID_ID; 4801 4802 switch (type) { 4803 case RAS_EVENT_TYPE_FATAL: 4804 case RAS_EVENT_TYPE_POISON_CREATION: 4805 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4806 event_mgr = __get_ras_event_mgr(adev); 4807 if (!event_mgr) 4808 return RAS_EVENT_INVALID_ID; 4809 4810 id = event_mgr->event_state[type].last_seqno; 4811 break; 4812 case RAS_EVENT_TYPE_INVALID: 4813 default: 4814 id = RAS_EVENT_INVALID_ID; 4815 break; 4816 } 4817 4818 return id; 4819 } 4820 4821 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4822 { 4823 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4824 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4825 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4826 u64 event_id = RAS_EVENT_INVALID_ID; 4827 4828 if (amdgpu_uniras_enabled(adev)) 4829 return 0; 4830 4831 if (!amdgpu_ras_mark_ras_event(adev, type)) 4832 event_id = amdgpu_ras_acquire_event_id(adev, type); 4833 4834 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4835 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4836 4837 amdgpu_ras_set_fed(adev, true); 4838 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4839 amdgpu_ras_reset_gpu(adev); 4840 } 4841 4842 return -EBUSY; 4843 } 4844 4845 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4846 { 4847 if (adev->asic_type == CHIP_VEGA20 && 4848 adev->pm.fw_version <= 0x283400) { 4849 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4850 amdgpu_ras_intr_triggered(); 4851 } 4852 4853 return false; 4854 } 4855 4856 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4857 { 4858 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4859 4860 if (!con) 4861 return; 4862 4863 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4864 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4865 amdgpu_ras_set_context(adev, NULL); 4866 kfree(con); 4867 } 4868 } 4869 4870 #ifdef CONFIG_X86_MCE_AMD 4871 static struct amdgpu_device *find_adev(uint32_t node_id) 4872 { 4873 int i; 4874 struct amdgpu_device *adev = NULL; 4875 4876 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4877 adev = mce_adev_list.devs[i]; 4878 4879 if (adev && adev->gmc.xgmi.connected_to_cpu && 4880 adev->gmc.xgmi.physical_node_id == node_id) 4881 break; 4882 adev = NULL; 4883 } 4884 4885 return adev; 4886 } 4887 4888 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4889 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4890 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4891 #define GPU_ID_OFFSET 8 4892 4893 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4894 unsigned long val, void *data) 4895 { 4896 struct mce *m = (struct mce *)data; 4897 struct amdgpu_device *adev = NULL; 4898 uint32_t gpu_id = 0; 4899 uint32_t umc_inst = 0, ch_inst = 0; 4900 4901 /* 4902 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4903 * and error occurred in DramECC (Extended error code = 0) then only 4904 * process the error, else bail out. 4905 */ 4906 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4907 (XEC(m->status, 0x3f) == 0x0))) 4908 return NOTIFY_DONE; 4909 4910 /* 4911 * If it is correctable error, return. 4912 */ 4913 if (mce_is_correctable(m)) 4914 return NOTIFY_OK; 4915 4916 /* 4917 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4918 */ 4919 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4920 4921 adev = find_adev(gpu_id); 4922 if (!adev) { 4923 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4924 gpu_id); 4925 return NOTIFY_DONE; 4926 } 4927 4928 /* 4929 * If it is uncorrectable error, then find out UMC instance and 4930 * channel index. 4931 */ 4932 umc_inst = GET_UMC_INST(m->ipid); 4933 ch_inst = GET_CHAN_INDEX(m->ipid); 4934 4935 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4936 umc_inst, ch_inst); 4937 4938 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4939 return NOTIFY_OK; 4940 else 4941 return NOTIFY_DONE; 4942 } 4943 4944 static struct notifier_block amdgpu_bad_page_nb = { 4945 .notifier_call = amdgpu_bad_page_notifier, 4946 .priority = MCE_PRIO_UC, 4947 }; 4948 4949 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4950 { 4951 /* 4952 * Add the adev to the mce_adev_list. 4953 * During mode2 reset, amdgpu device is temporarily 4954 * removed from the mgpu_info list which can cause 4955 * page retirement to fail. 4956 * Use this list instead of mgpu_info to find the amdgpu 4957 * device on which the UMC error was reported. 4958 */ 4959 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4960 4961 /* 4962 * Register the x86 notifier only once 4963 * with MCE subsystem. 4964 */ 4965 if (notifier_registered == false) { 4966 mce_register_decode_chain(&amdgpu_bad_page_nb); 4967 notifier_registered = true; 4968 } 4969 } 4970 #endif 4971 4972 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4973 { 4974 if (!adev) 4975 return NULL; 4976 4977 return adev->psp.ras_context.ras; 4978 } 4979 4980 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4981 { 4982 if (!adev) 4983 return -EINVAL; 4984 4985 adev->psp.ras_context.ras = ras_con; 4986 return 0; 4987 } 4988 4989 /* check if ras is supported on block, say, sdma, gfx */ 4990 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4991 unsigned int block) 4992 { 4993 int ret = 0; 4994 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4995 4996 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4997 return 0; 4998 4999 ret = ras && (adev->ras_enabled & (1 << block)); 5000 5001 /* For the special asic with mem ecc enabled but sram ecc 5002 * not enabled, even if the ras block is not supported on 5003 * .ras_enabled, if the asic supports poison mode and the 5004 * ras block has ras configuration, it can be considered 5005 * that the ras block supports ras function. 5006 */ 5007 if (!ret && 5008 (block == AMDGPU_RAS_BLOCK__GFX || 5009 block == AMDGPU_RAS_BLOCK__SDMA || 5010 block == AMDGPU_RAS_BLOCK__VCN || 5011 block == AMDGPU_RAS_BLOCK__JPEG) && 5012 (amdgpu_ras_mask & (1 << block)) && 5013 amdgpu_ras_is_poison_mode_supported(adev) && 5014 amdgpu_ras_get_ras_block(adev, block, 0)) 5015 ret = 1; 5016 5017 return ret; 5018 } 5019 5020 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5021 { 5022 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5023 5024 /* mode1 is the only selection for RMA status */ 5025 if (amdgpu_ras_is_rma(adev)) { 5026 ras->gpu_reset_flags = 0; 5027 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5028 } 5029 5030 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5031 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5032 int hive_ras_recovery = 0; 5033 5034 if (hive) { 5035 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5036 amdgpu_put_xgmi_hive(hive); 5037 } 5038 /* In the case of multiple GPUs, after a GPU has started 5039 * resetting all GPUs on hive, other GPUs do not need to 5040 * trigger GPU reset again. 5041 */ 5042 if (!hive_ras_recovery) 5043 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5044 else 5045 atomic_set(&ras->in_recovery, 0); 5046 } else { 5047 flush_work(&ras->recovery_work); 5048 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5049 } 5050 5051 return 0; 5052 } 5053 5054 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5055 { 5056 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5057 int ret = 0; 5058 5059 if (con) { 5060 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5061 if (!ret) 5062 con->is_aca_debug_mode = enable; 5063 } 5064 5065 return ret; 5066 } 5067 5068 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5069 { 5070 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5071 int ret = 0; 5072 5073 if (con) { 5074 if (amdgpu_aca_is_enabled(adev)) 5075 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5076 else 5077 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5078 if (!ret) 5079 con->is_aca_debug_mode = enable; 5080 } 5081 5082 return ret; 5083 } 5084 5085 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5086 { 5087 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5088 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5089 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5090 5091 if (!con) 5092 return false; 5093 5094 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5095 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5096 return con->is_aca_debug_mode; 5097 else 5098 return true; 5099 } 5100 5101 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5102 unsigned int *error_query_mode) 5103 { 5104 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5105 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5106 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5107 5108 if (!con) { 5109 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5110 return false; 5111 } 5112 5113 if (amdgpu_sriov_vf(adev)) { 5114 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5115 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5116 *error_query_mode = 5117 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5118 } else { 5119 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5120 } 5121 5122 return true; 5123 } 5124 5125 /* Register each ip ras block into amdgpu ras */ 5126 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5127 struct amdgpu_ras_block_object *ras_block_obj) 5128 { 5129 struct amdgpu_ras_block_list *ras_node; 5130 if (!adev || !ras_block_obj) 5131 return -EINVAL; 5132 5133 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5134 if (!ras_node) 5135 return -ENOMEM; 5136 5137 INIT_LIST_HEAD(&ras_node->node); 5138 ras_node->ras_obj = ras_block_obj; 5139 list_add_tail(&ras_node->node, &adev->ras_list); 5140 5141 return 0; 5142 } 5143 5144 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5145 { 5146 if (!err_type_name) 5147 return; 5148 5149 switch (err_type) { 5150 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5151 sprintf(err_type_name, "correctable"); 5152 break; 5153 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5154 sprintf(err_type_name, "uncorrectable"); 5155 break; 5156 default: 5157 sprintf(err_type_name, "unknown"); 5158 break; 5159 } 5160 } 5161 5162 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5163 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5164 uint32_t instance, 5165 uint32_t *memory_id) 5166 { 5167 uint32_t err_status_lo_data, err_status_lo_offset; 5168 5169 if (!reg_entry) 5170 return false; 5171 5172 err_status_lo_offset = 5173 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5174 reg_entry->seg_lo, reg_entry->reg_lo); 5175 err_status_lo_data = RREG32(err_status_lo_offset); 5176 5177 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5178 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5179 return false; 5180 5181 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5182 5183 return true; 5184 } 5185 5186 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5187 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5188 uint32_t instance, 5189 unsigned long *err_cnt) 5190 { 5191 uint32_t err_status_hi_data, err_status_hi_offset; 5192 5193 if (!reg_entry) 5194 return false; 5195 5196 err_status_hi_offset = 5197 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5198 reg_entry->seg_hi, reg_entry->reg_hi); 5199 err_status_hi_data = RREG32(err_status_hi_offset); 5200 5201 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5202 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5203 /* keep the check here in case we need to refer to the result later */ 5204 dev_dbg(adev->dev, "Invalid err_info field\n"); 5205 5206 /* read err count */ 5207 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5208 5209 return true; 5210 } 5211 5212 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5213 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5214 uint32_t reg_list_size, 5215 const struct amdgpu_ras_memory_id_entry *mem_list, 5216 uint32_t mem_list_size, 5217 uint32_t instance, 5218 uint32_t err_type, 5219 unsigned long *err_count) 5220 { 5221 uint32_t memory_id; 5222 unsigned long err_cnt; 5223 char err_type_name[16]; 5224 uint32_t i, j; 5225 5226 for (i = 0; i < reg_list_size; i++) { 5227 /* query memory_id from err_status_lo */ 5228 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5229 instance, &memory_id)) 5230 continue; 5231 5232 /* query err_cnt from err_status_hi */ 5233 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5234 instance, &err_cnt) || 5235 !err_cnt) 5236 continue; 5237 5238 *err_count += err_cnt; 5239 5240 /* log the errors */ 5241 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5242 if (!mem_list) { 5243 /* memory_list is not supported */ 5244 dev_info(adev->dev, 5245 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5246 err_cnt, err_type_name, 5247 reg_list[i].block_name, 5248 instance, memory_id); 5249 } else { 5250 for (j = 0; j < mem_list_size; j++) { 5251 if (memory_id == mem_list[j].memory_id) { 5252 dev_info(adev->dev, 5253 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5254 err_cnt, err_type_name, 5255 reg_list[i].block_name, 5256 instance, mem_list[j].name); 5257 break; 5258 } 5259 } 5260 } 5261 } 5262 } 5263 5264 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5265 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5266 uint32_t reg_list_size, 5267 uint32_t instance) 5268 { 5269 uint32_t err_status_lo_offset, err_status_hi_offset; 5270 uint32_t i; 5271 5272 for (i = 0; i < reg_list_size; i++) { 5273 err_status_lo_offset = 5274 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5275 reg_list[i].seg_lo, reg_list[i].reg_lo); 5276 err_status_hi_offset = 5277 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5278 reg_list[i].seg_hi, reg_list[i].reg_hi); 5279 WREG32(err_status_lo_offset, 0); 5280 WREG32(err_status_hi_offset, 0); 5281 } 5282 } 5283 5284 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5285 { 5286 memset(err_data, 0, sizeof(*err_data)); 5287 5288 INIT_LIST_HEAD(&err_data->err_node_list); 5289 5290 return 0; 5291 } 5292 5293 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5294 { 5295 if (!err_node) 5296 return; 5297 5298 list_del(&err_node->node); 5299 kvfree(err_node); 5300 } 5301 5302 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5303 { 5304 struct ras_err_node *err_node, *tmp; 5305 5306 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5307 amdgpu_ras_error_node_release(err_node); 5308 } 5309 5310 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5311 struct amdgpu_smuio_mcm_config_info *mcm_info) 5312 { 5313 struct ras_err_node *err_node; 5314 struct amdgpu_smuio_mcm_config_info *ref_id; 5315 5316 if (!err_data || !mcm_info) 5317 return NULL; 5318 5319 for_each_ras_error(err_node, err_data) { 5320 ref_id = &err_node->err_info.mcm_info; 5321 5322 if (mcm_info->socket_id == ref_id->socket_id && 5323 mcm_info->die_id == ref_id->die_id) 5324 return err_node; 5325 } 5326 5327 return NULL; 5328 } 5329 5330 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5331 { 5332 struct ras_err_node *err_node; 5333 5334 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5335 if (!err_node) 5336 return NULL; 5337 5338 INIT_LIST_HEAD(&err_node->node); 5339 5340 return err_node; 5341 } 5342 5343 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5344 { 5345 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5346 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5347 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5348 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5349 5350 if (unlikely(infoa->socket_id != infob->socket_id)) 5351 return infoa->socket_id - infob->socket_id; 5352 else 5353 return infoa->die_id - infob->die_id; 5354 5355 return 0; 5356 } 5357 5358 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5359 struct amdgpu_smuio_mcm_config_info *mcm_info) 5360 { 5361 struct ras_err_node *err_node; 5362 5363 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5364 if (err_node) 5365 return &err_node->err_info; 5366 5367 err_node = amdgpu_ras_error_node_new(); 5368 if (!err_node) 5369 return NULL; 5370 5371 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5372 5373 err_data->err_list_count++; 5374 list_add_tail(&err_node->node, &err_data->err_node_list); 5375 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5376 5377 return &err_node->err_info; 5378 } 5379 5380 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5381 struct amdgpu_smuio_mcm_config_info *mcm_info, 5382 u64 count) 5383 { 5384 struct ras_err_info *err_info; 5385 5386 if (!err_data || !mcm_info) 5387 return -EINVAL; 5388 5389 if (!count) 5390 return 0; 5391 5392 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5393 if (!err_info) 5394 return -EINVAL; 5395 5396 err_info->ue_count += count; 5397 err_data->ue_count += count; 5398 5399 return 0; 5400 } 5401 5402 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5403 struct amdgpu_smuio_mcm_config_info *mcm_info, 5404 u64 count) 5405 { 5406 struct ras_err_info *err_info; 5407 5408 if (!err_data || !mcm_info) 5409 return -EINVAL; 5410 5411 if (!count) 5412 return 0; 5413 5414 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5415 if (!err_info) 5416 return -EINVAL; 5417 5418 err_info->ce_count += count; 5419 err_data->ce_count += count; 5420 5421 return 0; 5422 } 5423 5424 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5425 struct amdgpu_smuio_mcm_config_info *mcm_info, 5426 u64 count) 5427 { 5428 struct ras_err_info *err_info; 5429 5430 if (!err_data || !mcm_info) 5431 return -EINVAL; 5432 5433 if (!count) 5434 return 0; 5435 5436 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5437 if (!err_info) 5438 return -EINVAL; 5439 5440 err_info->de_count += count; 5441 err_data->de_count += count; 5442 5443 return 0; 5444 } 5445 5446 #define mmMP0_SMN_C2PMSG_92 0x1609C 5447 #define mmMP0_SMN_C2PMSG_126 0x160BE 5448 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5449 u32 instance) 5450 { 5451 u32 socket_id, aid_id, hbm_id; 5452 u32 fw_status; 5453 u32 boot_error; 5454 u64 reg_addr; 5455 5456 /* The pattern for smn addressing in other SOC could be different from 5457 * the one for aqua_vanjaram. We should revisit the code if the pattern 5458 * is changed. In such case, replace the aqua_vanjaram implementation 5459 * with more common helper */ 5460 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5461 aqua_vanjaram_encode_ext_smn_addressing(instance); 5462 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5463 5464 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5465 aqua_vanjaram_encode_ext_smn_addressing(instance); 5466 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5467 5468 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5469 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5470 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5471 5472 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5473 dev_info(adev->dev, 5474 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5475 socket_id, aid_id, hbm_id, fw_status); 5476 5477 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5478 dev_info(adev->dev, 5479 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5480 socket_id, aid_id, fw_status); 5481 5482 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5483 dev_info(adev->dev, 5484 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5485 socket_id, aid_id, fw_status); 5486 5487 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5488 dev_info(adev->dev, 5489 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5490 socket_id, aid_id, fw_status); 5491 5492 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5493 dev_info(adev->dev, 5494 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5495 socket_id, aid_id, fw_status); 5496 5497 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5498 dev_info(adev->dev, 5499 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5500 socket_id, aid_id, fw_status); 5501 5502 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5503 dev_info(adev->dev, 5504 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5505 socket_id, aid_id, hbm_id, fw_status); 5506 5507 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5508 dev_info(adev->dev, 5509 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5510 socket_id, aid_id, hbm_id, fw_status); 5511 5512 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5513 dev_info(adev->dev, 5514 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5515 socket_id, aid_id, fw_status); 5516 5517 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5518 dev_info(adev->dev, 5519 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5520 socket_id, aid_id, fw_status); 5521 } 5522 5523 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5524 u32 instance) 5525 { 5526 u64 reg_addr; 5527 u32 reg_data; 5528 int retry_loop; 5529 5530 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5531 aqua_vanjaram_encode_ext_smn_addressing(instance); 5532 5533 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5534 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5535 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5536 return false; 5537 else 5538 msleep(1); 5539 } 5540 5541 return true; 5542 } 5543 5544 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5545 { 5546 u32 i; 5547 5548 for (i = 0; i < num_instances; i++) { 5549 if (amdgpu_ras_boot_error_detected(adev, i)) 5550 amdgpu_ras_boot_time_error_reporting(adev, i); 5551 } 5552 } 5553 5554 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5555 { 5556 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5557 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5558 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5559 int ret = 0; 5560 5561 if (amdgpu_ras_check_critical_address(adev, start)) 5562 return 0; 5563 5564 mutex_lock(&con->page_rsv_lock); 5565 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5566 if (ret == -ENOENT) 5567 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5568 mutex_unlock(&con->page_rsv_lock); 5569 5570 return ret; 5571 } 5572 5573 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5574 const char *fmt, ...) 5575 { 5576 struct va_format vaf; 5577 va_list args; 5578 5579 va_start(args, fmt); 5580 vaf.fmt = fmt; 5581 vaf.va = &args; 5582 5583 if (RAS_EVENT_ID_IS_VALID(event_id)) 5584 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5585 else 5586 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5587 5588 va_end(args); 5589 } 5590 5591 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5592 { 5593 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5594 5595 if (amdgpu_uniras_enabled(adev)) 5596 return amdgpu_ras_mgr_is_rma(adev); 5597 5598 if (!con) 5599 return false; 5600 5601 return con->is_rma; 5602 } 5603 5604 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5605 struct amdgpu_bo *bo) 5606 { 5607 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5608 struct amdgpu_vram_mgr_resource *vres; 5609 struct ras_critical_region *region; 5610 struct drm_buddy_block *block; 5611 int ret = 0; 5612 5613 if (!bo || !bo->tbo.resource) 5614 return -EINVAL; 5615 5616 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5617 5618 mutex_lock(&con->critical_region_lock); 5619 5620 /* Check if the bo had been recorded */ 5621 list_for_each_entry(region, &con->critical_region_head, node) 5622 if (region->bo == bo) 5623 goto out; 5624 5625 /* Record new critical amdgpu bo */ 5626 list_for_each_entry(block, &vres->blocks, link) { 5627 region = kzalloc(sizeof(*region), GFP_KERNEL); 5628 if (!region) { 5629 ret = -ENOMEM; 5630 goto out; 5631 } 5632 region->bo = bo; 5633 region->start = amdgpu_vram_mgr_block_start(block); 5634 region->size = amdgpu_vram_mgr_block_size(block); 5635 list_add_tail(®ion->node, &con->critical_region_head); 5636 } 5637 5638 out: 5639 mutex_unlock(&con->critical_region_lock); 5640 5641 return ret; 5642 } 5643 5644 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5645 { 5646 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5647 } 5648 5649 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5650 { 5651 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5652 struct ras_critical_region *region, *tmp; 5653 5654 mutex_lock(&con->critical_region_lock); 5655 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5656 list_del(®ion->node); 5657 kfree(region); 5658 } 5659 mutex_unlock(&con->critical_region_lock); 5660 } 5661 5662 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5663 { 5664 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5665 struct ras_critical_region *region; 5666 bool ret = false; 5667 5668 mutex_lock(&con->critical_region_lock); 5669 list_for_each_entry(region, &con->critical_region_head, node) { 5670 if ((region->start <= addr) && 5671 (addr < (region->start + region->size))) { 5672 ret = true; 5673 break; 5674 } 5675 } 5676 mutex_unlock(&con->critical_region_lock); 5677 5678 return ret; 5679 } 5680 5681 void amdgpu_ras_pre_reset(struct amdgpu_device *adev, 5682 struct list_head *device_list) 5683 { 5684 struct amdgpu_device *tmp_adev = NULL; 5685 5686 list_for_each_entry(tmp_adev, device_list, reset_list) { 5687 if (amdgpu_uniras_enabled(tmp_adev)) 5688 amdgpu_ras_mgr_pre_reset(tmp_adev); 5689 } 5690 } 5691 5692 void amdgpu_ras_post_reset(struct amdgpu_device *adev, 5693 struct list_head *device_list) 5694 { 5695 struct amdgpu_device *tmp_adev = NULL; 5696 5697 list_for_each_entry(tmp_adev, device_list, reset_list) { 5698 if (amdgpu_uniras_enabled(tmp_adev)) 5699 amdgpu_ras_mgr_post_reset(tmp_adev); 5700 } 5701 } 5702