1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 616 617 /** 618 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 619 * 620 * Some boards contain an EEPROM which is used to persistently store a list of 621 * bad pages which experiences ECC errors in vram. This interface provides 622 * a way to reset the EEPROM, e.g., after testing error injection. 623 * 624 * Usage: 625 * 626 * .. code-block:: bash 627 * 628 * echo 1 > ../ras/ras_eeprom_reset 629 * 630 * will reset EEPROM table to 0 entries. 631 * 632 */ 633 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 634 const char __user *buf, 635 size_t size, loff_t *pos) 636 { 637 struct amdgpu_device *adev = 638 (struct amdgpu_device *)file_inode(f)->i_private; 639 int ret; 640 641 if (amdgpu_uniras_enabled(adev)) { 642 ret = amdgpu_uniras_clear_badpages_info(adev); 643 return ret ? ret : size; 644 } 645 646 ret = amdgpu_ras_eeprom_reset_table( 647 &(amdgpu_ras_get_context(adev)->eeprom_control)); 648 649 if (!ret) { 650 /* Something was written to EEPROM. 651 */ 652 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 653 return size; 654 } else { 655 return ret; 656 } 657 } 658 659 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_ctrl_write, 663 .llseek = default_llseek 664 }; 665 666 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 667 .owner = THIS_MODULE, 668 .read = NULL, 669 .write = amdgpu_ras_debugfs_eeprom_write, 670 .llseek = default_llseek 671 }; 672 673 /** 674 * DOC: AMDGPU RAS sysfs Error Count Interface 675 * 676 * It allows the user to read the error count for each IP block on the gpu through 677 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 678 * 679 * It outputs the multiple lines which report the uncorrected (ue) and corrected 680 * (ce) error counts. 681 * 682 * The format of one line is below, 683 * 684 * [ce|ue]: count 685 * 686 * Example: 687 * 688 * .. code-block:: bash 689 * 690 * ue: 0 691 * ce: 1 692 * 693 */ 694 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 695 struct device_attribute *attr, char *buf) 696 { 697 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 698 struct ras_query_if info = { 699 .head = obj->head, 700 }; 701 702 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 703 return sysfs_emit(buf, "Query currently inaccessible\n"); 704 705 if (amdgpu_ras_query_error_status(obj->adev, &info)) 706 return -EINVAL; 707 708 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 709 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 710 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 711 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 712 } 713 714 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 715 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 716 "ce", info.ce_count, "de", info.de_count); 717 else 718 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 719 "ce", info.ce_count); 720 } 721 722 /* obj begin */ 723 724 #define get_obj(obj) do { (obj)->use++; } while (0) 725 #define alive_obj(obj) ((obj)->use) 726 727 static inline void put_obj(struct ras_manager *obj) 728 { 729 if (obj && (--obj->use == 0)) { 730 list_del(&obj->node); 731 amdgpu_ras_error_data_fini(&obj->err_data); 732 } 733 734 if (obj && (obj->use < 0)) 735 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 736 } 737 738 /* make one obj and return it. */ 739 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 740 struct ras_common_if *head) 741 { 742 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 743 struct ras_manager *obj; 744 745 if (!adev->ras_enabled || !con) 746 return NULL; 747 748 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 749 return NULL; 750 751 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 752 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 753 return NULL; 754 755 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 756 } else 757 obj = &con->objs[head->block]; 758 759 /* already exist. return obj? */ 760 if (alive_obj(obj)) 761 return NULL; 762 763 if (amdgpu_ras_error_data_init(&obj->err_data)) 764 return NULL; 765 766 obj->head = *head; 767 obj->adev = adev; 768 list_add(&obj->node, &con->head); 769 get_obj(obj); 770 771 return obj; 772 } 773 774 /* return an obj equal to head, or the first when head is NULL */ 775 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 776 struct ras_common_if *head) 777 { 778 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 779 struct ras_manager *obj; 780 int i; 781 782 if (!adev->ras_enabled || !con) 783 return NULL; 784 785 if (head) { 786 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 787 return NULL; 788 789 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 790 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 791 return NULL; 792 793 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 794 } else 795 obj = &con->objs[head->block]; 796 797 if (alive_obj(obj)) 798 return obj; 799 } else { 800 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 801 obj = &con->objs[i]; 802 if (alive_obj(obj)) 803 return obj; 804 } 805 } 806 807 return NULL; 808 } 809 /* obj end */ 810 811 /* feature ctl begin */ 812 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 813 struct ras_common_if *head) 814 { 815 return adev->ras_hw_enabled & BIT(head->block); 816 } 817 818 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 819 struct ras_common_if *head) 820 { 821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 822 823 return con->features & BIT(head->block); 824 } 825 826 /* 827 * if obj is not created, then create one. 828 * set feature enable flag. 829 */ 830 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 831 struct ras_common_if *head, int enable) 832 { 833 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 834 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 835 836 /* If hardware does not support ras, then do not create obj. 837 * But if hardware support ras, we can create the obj. 838 * Ras framework checks con->hw_supported to see if it need do 839 * corresponding initialization. 840 * IP checks con->support to see if it need disable ras. 841 */ 842 if (!amdgpu_ras_is_feature_allowed(adev, head)) 843 return 0; 844 845 if (enable) { 846 if (!obj) { 847 obj = amdgpu_ras_create_obj(adev, head); 848 if (!obj) 849 return -EINVAL; 850 } else { 851 /* In case we create obj somewhere else */ 852 get_obj(obj); 853 } 854 con->features |= BIT(head->block); 855 } else { 856 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 857 con->features &= ~BIT(head->block); 858 put_obj(obj); 859 } 860 } 861 862 return 0; 863 } 864 865 /* wrapper of psp_ras_enable_features */ 866 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 867 struct ras_common_if *head, bool enable) 868 { 869 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 870 union ta_ras_cmd_input *info; 871 int ret; 872 873 if (!con) 874 return -EINVAL; 875 876 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 877 /* For gfx ip, regardless of feature support status, */ 878 /* Force issue enable or disable ras feature commands */ 879 if (head->block != AMDGPU_RAS_BLOCK__GFX && 880 !amdgpu_ras_is_feature_allowed(adev, head)) 881 return 0; 882 883 /* Only enable gfx ras feature from host side */ 884 if (head->block == AMDGPU_RAS_BLOCK__GFX && 885 !amdgpu_sriov_vf(adev) && 886 !amdgpu_ras_intr_triggered()) { 887 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 888 if (!info) 889 return -ENOMEM; 890 891 if (!enable) { 892 info->disable_features = (struct ta_ras_disable_features_input) { 893 .block_id = amdgpu_ras_block_to_ta(head->block), 894 .error_type = amdgpu_ras_error_to_ta(head->type), 895 }; 896 } else { 897 info->enable_features = (struct ta_ras_enable_features_input) { 898 .block_id = amdgpu_ras_block_to_ta(head->block), 899 .error_type = amdgpu_ras_error_to_ta(head->type), 900 }; 901 } 902 903 ret = psp_ras_enable_features(&adev->psp, info, enable); 904 if (ret) { 905 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 906 enable ? "enable":"disable", 907 get_ras_block_str(head), 908 amdgpu_ras_is_poison_mode_supported(adev), ret); 909 kfree(info); 910 return ret; 911 } 912 913 kfree(info); 914 } 915 916 /* setup the obj */ 917 __amdgpu_ras_feature_enable(adev, head, enable); 918 919 return 0; 920 } 921 922 /* Only used in device probe stage and called only once. */ 923 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 924 struct ras_common_if *head, bool enable) 925 { 926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 927 int ret; 928 929 if (!con) 930 return -EINVAL; 931 932 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 933 if (enable) { 934 /* There is no harm to issue a ras TA cmd regardless of 935 * the currecnt ras state. 936 * If current state == target state, it will do nothing 937 * But sometimes it requests driver to reset and repost 938 * with error code -EAGAIN. 939 */ 940 ret = amdgpu_ras_feature_enable(adev, head, 1); 941 /* With old ras TA, we might fail to enable ras. 942 * Log it and just setup the object. 943 * TODO need remove this WA in the future. 944 */ 945 if (ret == -EINVAL) { 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (!ret) 948 dev_info(adev->dev, 949 "RAS INFO: %s setup object\n", 950 get_ras_block_str(head)); 951 } 952 } else { 953 /* setup the object then issue a ras TA disable cmd.*/ 954 ret = __amdgpu_ras_feature_enable(adev, head, 1); 955 if (ret) 956 return ret; 957 958 /* gfx block ras disable cmd must send to ras-ta */ 959 if (head->block == AMDGPU_RAS_BLOCK__GFX) 960 con->features |= BIT(head->block); 961 962 ret = amdgpu_ras_feature_enable(adev, head, 0); 963 964 /* clean gfx block ras features flag */ 965 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 966 con->features &= ~BIT(head->block); 967 } 968 } else 969 ret = amdgpu_ras_feature_enable(adev, head, enable); 970 971 return ret; 972 } 973 974 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 975 bool bypass) 976 { 977 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 978 struct ras_manager *obj, *tmp; 979 980 list_for_each_entry_safe(obj, tmp, &con->head, node) { 981 /* bypass psp. 982 * aka just release the obj and corresponding flags 983 */ 984 if (bypass) { 985 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 986 break; 987 } else { 988 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 989 break; 990 } 991 } 992 993 return con->features; 994 } 995 996 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 997 bool bypass) 998 { 999 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1000 int i; 1001 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1002 1003 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1004 struct ras_common_if head = { 1005 .block = i, 1006 .type = default_ras_type, 1007 .sub_block_index = 0, 1008 }; 1009 1010 if (i == AMDGPU_RAS_BLOCK__MCA) 1011 continue; 1012 1013 if (bypass) { 1014 /* 1015 * bypass psp. vbios enable ras for us. 1016 * so just create the obj 1017 */ 1018 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1019 break; 1020 } else { 1021 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1022 break; 1023 } 1024 } 1025 1026 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1027 struct ras_common_if head = { 1028 .block = AMDGPU_RAS_BLOCK__MCA, 1029 .type = default_ras_type, 1030 .sub_block_index = i, 1031 }; 1032 1033 if (bypass) { 1034 /* 1035 * bypass psp. vbios enable ras for us. 1036 * so just create the obj 1037 */ 1038 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1039 break; 1040 } else { 1041 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1042 break; 1043 } 1044 } 1045 1046 return con->features; 1047 } 1048 /* feature ctl end */ 1049 1050 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1051 enum amdgpu_ras_block block) 1052 { 1053 if (!block_obj) 1054 return -EINVAL; 1055 1056 if (block_obj->ras_comm.block == block) 1057 return 0; 1058 1059 return -EINVAL; 1060 } 1061 1062 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1063 enum amdgpu_ras_block block, uint32_t sub_block_index) 1064 { 1065 struct amdgpu_ras_block_list *node, *tmp; 1066 struct amdgpu_ras_block_object *obj; 1067 1068 if (block >= AMDGPU_RAS_BLOCK__LAST) 1069 return NULL; 1070 1071 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1072 if (!node->ras_obj) { 1073 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1074 continue; 1075 } 1076 1077 obj = node->ras_obj; 1078 if (obj->ras_block_match) { 1079 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1080 return obj; 1081 } else { 1082 if (amdgpu_ras_block_match_default(obj, block) == 0) 1083 return obj; 1084 } 1085 } 1086 1087 return NULL; 1088 } 1089 1090 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1091 { 1092 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1093 int ret = 0; 1094 1095 /* 1096 * choosing right query method according to 1097 * whether smu support query error information 1098 */ 1099 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1100 if (ret == -EOPNOTSUPP) { 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1104 1105 /* umc query_ras_error_address is also responsible for clearing 1106 * error status 1107 */ 1108 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1109 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1110 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1111 } else if (!ret) { 1112 if (adev->umc.ras && 1113 adev->umc.ras->ecc_info_query_ras_error_count) 1114 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1115 1116 if (adev->umc.ras && 1117 adev->umc.ras->ecc_info_query_ras_error_address) 1118 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1119 } 1120 } 1121 1122 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1123 struct ras_manager *ras_mgr, 1124 struct ras_err_data *err_data, 1125 struct ras_query_context *qctx, 1126 const char *blk_name, 1127 bool is_ue, 1128 bool is_de) 1129 { 1130 struct amdgpu_smuio_mcm_config_info *mcm_info; 1131 struct ras_err_node *err_node; 1132 struct ras_err_info *err_info; 1133 u64 event_id = qctx->evid.event_id; 1134 1135 if (is_ue) { 1136 for_each_ras_error(err_node, err_data) { 1137 err_info = &err_node->err_info; 1138 mcm_info = &err_info->mcm_info; 1139 if (err_info->ue_count) { 1140 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1141 "%lld new uncorrectable hardware errors detected in %s block\n", 1142 mcm_info->socket_id, 1143 mcm_info->die_id, 1144 err_info->ue_count, 1145 blk_name); 1146 } 1147 } 1148 1149 for_each_ras_error(err_node, &ras_mgr->err_data) { 1150 err_info = &err_node->err_info; 1151 mcm_info = &err_info->mcm_info; 1152 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1153 "%lld uncorrectable hardware errors detected in total in %s block\n", 1154 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1155 } 1156 1157 } else { 1158 if (is_de) { 1159 for_each_ras_error(err_node, err_data) { 1160 err_info = &err_node->err_info; 1161 mcm_info = &err_info->mcm_info; 1162 if (err_info->de_count) { 1163 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1164 "%lld new deferred hardware errors detected in %s block\n", 1165 mcm_info->socket_id, 1166 mcm_info->die_id, 1167 err_info->de_count, 1168 blk_name); 1169 } 1170 } 1171 1172 for_each_ras_error(err_node, &ras_mgr->err_data) { 1173 err_info = &err_node->err_info; 1174 mcm_info = &err_info->mcm_info; 1175 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1176 "%lld deferred hardware errors detected in total in %s block\n", 1177 mcm_info->socket_id, mcm_info->die_id, 1178 err_info->de_count, blk_name); 1179 } 1180 } else { 1181 if (adev->debug_disable_ce_logs) 1182 return; 1183 1184 for_each_ras_error(err_node, err_data) { 1185 err_info = &err_node->err_info; 1186 mcm_info = &err_info->mcm_info; 1187 if (err_info->ce_count) { 1188 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1189 "%lld new correctable hardware errors detected in %s block\n", 1190 mcm_info->socket_id, 1191 mcm_info->die_id, 1192 err_info->ce_count, 1193 blk_name); 1194 } 1195 } 1196 1197 for_each_ras_error(err_node, &ras_mgr->err_data) { 1198 err_info = &err_node->err_info; 1199 mcm_info = &err_info->mcm_info; 1200 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1201 "%lld correctable hardware errors detected in total in %s block\n", 1202 mcm_info->socket_id, mcm_info->die_id, 1203 err_info->ce_count, blk_name); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline bool err_data_has_source_info(struct ras_err_data *data) 1210 { 1211 return !list_empty(&data->err_node_list); 1212 } 1213 1214 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1215 struct ras_query_if *query_if, 1216 struct ras_err_data *err_data, 1217 struct ras_query_context *qctx) 1218 { 1219 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1220 const char *blk_name = get_ras_block_str(&query_if->head); 1221 u64 event_id = qctx->evid.event_id; 1222 1223 if (err_data->ce_count) { 1224 if (err_data_has_source_info(err_data)) { 1225 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1226 blk_name, false, false); 1227 } else if (!adev->aid_mask && 1228 adev->smuio.funcs && 1229 adev->smuio.funcs->get_socket_id && 1230 adev->smuio.funcs->get_die_id) { 1231 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1232 "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 adev->smuio.funcs->get_socket_id(adev), 1235 adev->smuio.funcs->get_die_id(adev), 1236 ras_mgr->err_data.ce_count, 1237 blk_name); 1238 } else { 1239 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1240 "detected in %s block\n", 1241 ras_mgr->err_data.ce_count, 1242 blk_name); 1243 } 1244 } 1245 1246 if (err_data->ue_count) { 1247 if (err_data_has_source_info(err_data)) { 1248 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1249 blk_name, true, false); 1250 } else if (!adev->aid_mask && 1251 adev->smuio.funcs && 1252 adev->smuio.funcs->get_socket_id && 1253 adev->smuio.funcs->get_die_id) { 1254 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1255 "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 adev->smuio.funcs->get_socket_id(adev), 1258 adev->smuio.funcs->get_die_id(adev), 1259 ras_mgr->err_data.ue_count, 1260 blk_name); 1261 } else { 1262 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1263 "detected in %s block\n", 1264 ras_mgr->err_data.ue_count, 1265 blk_name); 1266 } 1267 } 1268 1269 if (err_data->de_count) { 1270 if (err_data_has_source_info(err_data)) { 1271 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1272 blk_name, false, true); 1273 } else if (!adev->aid_mask && 1274 adev->smuio.funcs && 1275 adev->smuio.funcs->get_socket_id && 1276 adev->smuio.funcs->get_die_id) { 1277 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1278 "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 adev->smuio.funcs->get_socket_id(adev), 1281 adev->smuio.funcs->get_die_id(adev), 1282 ras_mgr->err_data.de_count, 1283 blk_name); 1284 } else { 1285 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1286 "detected in %s block\n", 1287 ras_mgr->err_data.de_count, 1288 blk_name); 1289 } 1290 } 1291 } 1292 1293 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1294 struct ras_query_if *query_if, 1295 struct ras_err_data *err_data, 1296 struct ras_query_context *qctx) 1297 { 1298 unsigned long new_ue, new_ce, new_de; 1299 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1300 const char *blk_name = get_ras_block_str(&query_if->head); 1301 u64 event_id = qctx->evid.event_id; 1302 1303 new_ce = err_data->ce_count - obj->err_data.ce_count; 1304 new_ue = err_data->ue_count - obj->err_data.ue_count; 1305 new_de = err_data->de_count - obj->err_data.de_count; 1306 1307 if (new_ce) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1309 "detected in %s block\n", 1310 new_ce, 1311 blk_name); 1312 } 1313 1314 if (new_ue) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1316 "detected in %s block\n", 1317 new_ue, 1318 blk_name); 1319 } 1320 1321 if (new_de) { 1322 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1323 "detected in %s block\n", 1324 new_de, 1325 blk_name); 1326 } 1327 } 1328 1329 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1330 { 1331 struct ras_err_node *err_node; 1332 struct ras_err_info *err_info; 1333 1334 if (err_data_has_source_info(err_data)) { 1335 for_each_ras_error(err_node, err_data) { 1336 err_info = &err_node->err_info; 1337 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1338 &err_info->mcm_info, err_info->de_count); 1339 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->ce_count); 1341 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ue_count); 1343 } 1344 } else { 1345 /* for legacy asic path which doesn't has error source info */ 1346 obj->err_data.ue_count += err_data->ue_count; 1347 obj->err_data.ce_count += err_data->ce_count; 1348 obj->err_data.de_count += err_data->de_count; 1349 } 1350 } 1351 1352 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1353 struct ras_err_data *err_data) 1354 { 1355 /* Host reports absolute counts */ 1356 obj->err_data.ue_count = err_data->ue_count; 1357 obj->err_data.ce_count = err_data->ce_count; 1358 obj->err_data.de_count = err_data->de_count; 1359 } 1360 1361 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1362 { 1363 struct ras_common_if head; 1364 1365 memset(&head, 0, sizeof(head)); 1366 head.block = blk; 1367 1368 return amdgpu_ras_find_obj(adev, &head); 1369 } 1370 1371 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1372 const struct aca_info *aca_info, void *data) 1373 { 1374 struct ras_manager *obj; 1375 1376 /* in resume phase, no need to create aca fs node */ 1377 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1378 return 0; 1379 1380 obj = get_ras_manager(adev, blk); 1381 if (!obj) 1382 return -EINVAL; 1383 1384 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1385 } 1386 1387 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1388 { 1389 struct ras_manager *obj; 1390 1391 obj = get_ras_manager(adev, blk); 1392 if (!obj) 1393 return -EINVAL; 1394 1395 amdgpu_aca_remove_handle(&obj->aca_handle); 1396 1397 return 0; 1398 } 1399 1400 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1401 enum aca_error_type type, struct ras_err_data *err_data, 1402 struct ras_query_context *qctx) 1403 { 1404 struct ras_manager *obj; 1405 1406 obj = get_ras_manager(adev, blk); 1407 if (!obj) 1408 return -EINVAL; 1409 1410 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1411 } 1412 1413 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1414 struct aca_handle *handle, char *buf, void *data) 1415 { 1416 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1417 struct ras_query_if info = { 1418 .head = obj->head, 1419 }; 1420 1421 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1422 return sysfs_emit(buf, "Query currently inaccessible\n"); 1423 1424 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1425 return -EINVAL; 1426 1427 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1428 "ce", info.ce_count, "de", info.de_count); 1429 } 1430 1431 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1432 struct ras_query_if *info, 1433 struct ras_err_data *err_data, 1434 struct ras_query_context *qctx, 1435 unsigned int error_query_mode) 1436 { 1437 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1438 struct amdgpu_ras_block_object *block_obj = NULL; 1439 int ret; 1440 1441 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1442 return -EINVAL; 1443 1444 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1445 return -EINVAL; 1446 1447 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1448 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1449 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1450 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1451 amdgpu_ras_get_ecc_info(adev, err_data); 1452 } else { 1453 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1454 if (!block_obj || !block_obj->hw_ops) { 1455 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1456 get_ras_block_str(&info->head)); 1457 return -EINVAL; 1458 } 1459 1460 if (block_obj->hw_ops->query_ras_error_count) 1461 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1462 1463 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1464 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1465 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1466 if (block_obj->hw_ops->query_ras_error_status) 1467 block_obj->hw_ops->query_ras_error_status(adev); 1468 } 1469 } 1470 } else { 1471 if (amdgpu_aca_is_enabled(adev)) { 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 1476 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1477 if (ret) 1478 return ret; 1479 1480 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1481 if (ret) 1482 return ret; 1483 } else { 1484 /* FIXME: add code to check return value later */ 1485 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1486 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1487 } 1488 } 1489 1490 return 0; 1491 } 1492 1493 /* query/inject/cure begin */ 1494 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1495 struct ras_query_if *info, 1496 enum ras_event_type type) 1497 { 1498 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1499 struct ras_err_data err_data; 1500 struct ras_query_context qctx; 1501 unsigned int error_query_mode; 1502 int ret; 1503 1504 if (!obj) 1505 return -EINVAL; 1506 1507 ret = amdgpu_ras_error_data_init(&err_data); 1508 if (ret) 1509 return ret; 1510 1511 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1512 return -EINVAL; 1513 1514 memset(&qctx, 0, sizeof(qctx)); 1515 qctx.evid.type = type; 1516 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1517 1518 if (!down_read_trylock(&adev->reset_domain->sem)) { 1519 ret = -EIO; 1520 goto out_fini_err_data; 1521 } 1522 1523 ret = amdgpu_ras_query_error_status_helper(adev, info, 1524 &err_data, 1525 &qctx, 1526 error_query_mode); 1527 up_read(&adev->reset_domain->sem); 1528 if (ret) 1529 goto out_fini_err_data; 1530 1531 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1532 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1533 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1534 } else { 1535 /* Host provides absolute error counts. First generate the report 1536 * using the previous VF internal count against new host count. 1537 * Then Update VF internal count. 1538 */ 1539 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1540 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1541 } 1542 1543 info->ue_count = obj->err_data.ue_count; 1544 info->ce_count = obj->err_data.ce_count; 1545 info->de_count = obj->err_data.de_count; 1546 1547 out_fini_err_data: 1548 amdgpu_ras_error_data_fini(&err_data); 1549 1550 return ret; 1551 } 1552 1553 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1554 { 1555 struct ras_cmd_dev_handle req = {0}; 1556 int ret; 1557 1558 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1559 &req, sizeof(req), NULL, 0); 1560 if (ret) { 1561 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1562 return ret; 1563 } 1564 1565 return 0; 1566 } 1567 1568 static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1569 struct ras_query_if *info) 1570 { 1571 struct ras_cmd_block_ecc_info_req req = {0}; 1572 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1573 int ret; 1574 1575 if (!info) 1576 return -EINVAL; 1577 1578 req.block_id = info->head.block; 1579 req.subblock_id = info->head.sub_block_index; 1580 1581 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1582 &req, sizeof(req), &rsp, sizeof(rsp)); 1583 if (!ret) { 1584 info->ce_count = rsp.ce_count; 1585 info->ue_count = rsp.ue_count; 1586 info->de_count = rsp.de_count; 1587 } 1588 1589 return ret; 1590 } 1591 1592 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1593 { 1594 if (amdgpu_uniras_enabled(adev)) 1595 return amdgpu_uniras_query_block_ecc(adev, info); 1596 else 1597 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1598 } 1599 1600 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1601 enum amdgpu_ras_block block) 1602 { 1603 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1604 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1605 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1606 1607 if (!block_obj || !block_obj->hw_ops) { 1608 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1609 ras_block_str(block)); 1610 return -EOPNOTSUPP; 1611 } 1612 1613 if (!amdgpu_ras_is_supported(adev, block) || 1614 !amdgpu_ras_get_aca_debug_mode(adev)) 1615 return -EOPNOTSUPP; 1616 1617 if (amdgpu_sriov_vf(adev)) 1618 return -EOPNOTSUPP; 1619 1620 /* skip ras error reset in gpu reset */ 1621 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1622 ((smu_funcs && smu_funcs->set_debug_mode) || 1623 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1624 return -EOPNOTSUPP; 1625 1626 if (block_obj->hw_ops->reset_ras_error_count) 1627 block_obj->hw_ops->reset_ras_error_count(adev); 1628 1629 return 0; 1630 } 1631 1632 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1633 enum amdgpu_ras_block block) 1634 { 1635 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1636 1637 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1638 return 0; 1639 1640 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1641 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1642 if (block_obj->hw_ops->reset_ras_error_status) 1643 block_obj->hw_ops->reset_ras_error_status(adev); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1650 struct ras_inject_if *info) 1651 { 1652 struct ras_cmd_inject_error_req inject_req; 1653 struct ras_cmd_inject_error_rsp rsp; 1654 1655 if (!info) 1656 return -EINVAL; 1657 1658 memset(&inject_req, 0, sizeof(inject_req)); 1659 inject_req.block_id = info->head.block; 1660 inject_req.subblock_id = info->head.sub_block_index; 1661 inject_req.address = info->address; 1662 inject_req.error_type = info->head.type; 1663 inject_req.instance_mask = info->instance_mask; 1664 inject_req.method = info->value; 1665 1666 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1667 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1668 } 1669 1670 /* wrapper of psp_ras_trigger_error */ 1671 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1672 struct ras_inject_if *info) 1673 { 1674 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1675 struct ta_ras_trigger_error_input block_info = { 1676 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1677 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1678 .sub_block_index = info->head.sub_block_index, 1679 .address = info->address, 1680 .value = info->value, 1681 }; 1682 int ret = -EINVAL; 1683 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1684 info->head.block, 1685 info->head.sub_block_index); 1686 1687 if (amdgpu_uniras_enabled(adev)) 1688 return amdgpu_uniras_error_inject(adev, info); 1689 1690 /* inject on guest isn't allowed, return success directly */ 1691 if (amdgpu_sriov_vf(adev)) 1692 return 0; 1693 1694 if (!obj) 1695 return -EINVAL; 1696 1697 if (!block_obj || !block_obj->hw_ops) { 1698 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1699 get_ras_block_str(&info->head)); 1700 return -EINVAL; 1701 } 1702 1703 /* Calculate XGMI relative offset */ 1704 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1705 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1706 block_info.address = 1707 amdgpu_xgmi_get_relative_phy_addr(adev, 1708 block_info.address); 1709 } 1710 1711 if (block_obj->hw_ops->ras_error_inject) { 1712 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1713 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1714 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1715 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1716 info->instance_mask); 1717 } else { 1718 /* default path */ 1719 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1720 } 1721 1722 if (ret) 1723 dev_err(adev->dev, "ras inject %s failed %d\n", 1724 get_ras_block_str(&info->head), ret); 1725 1726 return ret; 1727 } 1728 1729 /** 1730 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1731 * @adev: pointer to AMD GPU device 1732 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1733 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1734 * @query_info: pointer to ras_query_if 1735 * 1736 * Return 0 for query success or do nothing, otherwise return an error 1737 * on failures 1738 */ 1739 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1740 unsigned long *ce_count, 1741 unsigned long *ue_count, 1742 struct ras_query_if *query_info) 1743 { 1744 int ret; 1745 1746 if (!query_info) 1747 /* do nothing if query_info is not specified */ 1748 return 0; 1749 1750 ret = amdgpu_ras_query_error_status(adev, query_info); 1751 if (ret) 1752 return ret; 1753 1754 *ce_count += query_info->ce_count; 1755 *ue_count += query_info->ue_count; 1756 1757 /* some hardware/IP supports read to clear 1758 * no need to explictly reset the err status after the query call */ 1759 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1760 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1761 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1762 dev_warn(adev->dev, 1763 "Failed to reset error counter and error status\n"); 1764 } 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1771 * @adev: pointer to AMD GPU device 1772 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1773 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1774 * errors. 1775 * @query_info: pointer to ras_query_if if the query request is only for 1776 * specific ip block; if info is NULL, then the qurey request is for 1777 * all the ip blocks that support query ras error counters/status 1778 * 1779 * If set, @ce_count or @ue_count, count and return the corresponding 1780 * error counts in those integer pointers. Return 0 if the device 1781 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1782 */ 1783 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1784 unsigned long *ce_count, 1785 unsigned long *ue_count, 1786 struct ras_query_if *query_info) 1787 { 1788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1789 struct ras_manager *obj; 1790 unsigned long ce, ue; 1791 int ret; 1792 1793 if (!adev->ras_enabled || !con) 1794 return -EOPNOTSUPP; 1795 1796 /* Don't count since no reporting. 1797 */ 1798 if (!ce_count && !ue_count) 1799 return 0; 1800 1801 ce = 0; 1802 ue = 0; 1803 if (!query_info) { 1804 /* query all the ip blocks that support ras query interface */ 1805 list_for_each_entry(obj, &con->head, node) { 1806 struct ras_query_if info = { 1807 .head = obj->head, 1808 }; 1809 1810 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1811 } 1812 } else { 1813 /* query specific ip block */ 1814 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1815 } 1816 1817 if (ret) 1818 return ret; 1819 1820 if (ce_count) 1821 *ce_count = ce; 1822 1823 if (ue_count) 1824 *ue_count = ue; 1825 1826 return 0; 1827 } 1828 /* query/inject/cure end */ 1829 1830 1831 /* sysfs begin */ 1832 1833 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1834 struct ras_badpage *bps, uint32_t count, uint32_t start); 1835 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837 1838 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1839 { 1840 switch (flags) { 1841 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1842 return "R"; 1843 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1844 return "P"; 1845 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1846 default: 1847 return "F"; 1848 } 1849 } 1850 1851 /** 1852 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1853 * 1854 * It allows user to read the bad pages of vram on the gpu through 1855 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1856 * 1857 * It outputs multiple lines, and each line stands for one gpu page. 1858 * 1859 * The format of one line is below, 1860 * gpu pfn : gpu page size : flags 1861 * 1862 * gpu pfn and gpu page size are printed in hex format. 1863 * flags can be one of below character, 1864 * 1865 * R: reserved, this gpu page is reserved and not able to use. 1866 * 1867 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1868 * in next window of page_reserve. 1869 * 1870 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1871 * 1872 * Examples: 1873 * 1874 * .. code-block:: bash 1875 * 1876 * 0x00000001 : 0x00001000 : R 1877 * 0x00000002 : 0x00001000 : P 1878 * 1879 */ 1880 1881 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1882 struct kobject *kobj, const struct bin_attribute *attr, 1883 char *buf, loff_t ppos, size_t count) 1884 { 1885 struct amdgpu_ras *con = 1886 container_of(attr, struct amdgpu_ras, badpages_attr); 1887 struct amdgpu_device *adev = con->adev; 1888 const unsigned int element_size = 1889 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1890 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1891 unsigned int end = div64_ul(ppos + count - 1, element_size); 1892 ssize_t s = 0; 1893 struct ras_badpage *bps = NULL; 1894 int bps_count = 0, i, status; 1895 uint64_t address; 1896 1897 memset(buf, 0, count); 1898 1899 bps_count = end - start; 1900 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1901 if (!bps) 1902 return 0; 1903 1904 memset(bps, 0, sizeof(*bps) * bps_count); 1905 1906 if (amdgpu_uniras_enabled(adev)) 1907 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1908 else 1909 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1910 1911 if (bps_count <= 0) { 1912 kfree(bps); 1913 return 0; 1914 } 1915 1916 for (i = 0; i < bps_count; i++) { 1917 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1918 if (amdgpu_ras_check_critical_address(adev, address)) 1919 continue; 1920 1921 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1922 1923 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1924 address); 1925 if (status == -EBUSY) 1926 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1927 else if (status == -ENOENT) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1929 else 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1931 1932 s += scnprintf(&buf[s], element_size + 1, 1933 "0x%08x : 0x%08x : %1s\n", 1934 bps[i].bp, 1935 bps[i].size, 1936 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1937 } 1938 1939 kfree(bps); 1940 1941 return s; 1942 } 1943 1944 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1945 struct device_attribute *attr, char *buf) 1946 { 1947 struct amdgpu_ras *con = 1948 container_of(attr, struct amdgpu_ras, features_attr); 1949 1950 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1951 } 1952 1953 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1954 u32 *minor, u32 *rev) 1955 { 1956 int i; 1957 1958 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1959 return false; 1960 1961 for (i = 0; i < adev->num_ip_blocks; i++) { 1962 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1963 *major = adev->ip_blocks[i].version->major; 1964 *minor = adev->ip_blocks[i].version->minor; 1965 *rev = adev->ip_blocks[i].version->rev; 1966 return true; 1967 } 1968 } 1969 1970 return false; 1971 } 1972 1973 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1974 struct device_attribute *attr, char *buf) 1975 { 1976 struct amdgpu_ras *con = 1977 container_of(attr, struct amdgpu_ras, version_attr); 1978 u32 major, minor, rev; 1979 ssize_t size = 0; 1980 1981 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1982 con->eeprom_control.tbl_hdr.version); 1983 1984 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1985 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1986 major, minor, rev); 1987 1988 return size; 1989 } 1990 1991 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1992 struct device_attribute *attr, char *buf) 1993 { 1994 struct amdgpu_ras *con = 1995 container_of(attr, struct amdgpu_ras, schema_attr); 1996 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1997 } 1998 1999 static struct { 2000 enum ras_event_type type; 2001 const char *name; 2002 } dump_event[] = { 2003 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2004 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2005 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2006 }; 2007 2008 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2009 struct device_attribute *attr, char *buf) 2010 { 2011 struct amdgpu_ras *con = 2012 container_of(attr, struct amdgpu_ras, event_state_attr); 2013 struct ras_event_manager *event_mgr = con->event_mgr; 2014 struct ras_event_state *event_state; 2015 int i, size = 0; 2016 2017 if (!event_mgr) 2018 return -EINVAL; 2019 2020 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2021 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2022 event_state = &event_mgr->event_state[dump_event[i].type]; 2023 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2024 dump_event[i].name, 2025 atomic64_read(&event_state->count), 2026 event_state->last_seqno); 2027 } 2028 2029 return (ssize_t)size; 2030 } 2031 2032 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2033 { 2034 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2035 2036 if (adev->dev->kobj.sd) 2037 sysfs_remove_file_from_group(&adev->dev->kobj, 2038 &con->badpages_attr.attr, 2039 RAS_FS_NAME); 2040 } 2041 2042 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2043 { 2044 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2045 struct attribute *attrs[] = { 2046 &con->features_attr.attr, 2047 &con->version_attr.attr, 2048 &con->schema_attr.attr, 2049 &con->event_state_attr.attr, 2050 NULL 2051 }; 2052 struct attribute_group group = { 2053 .name = RAS_FS_NAME, 2054 .attrs = attrs, 2055 }; 2056 2057 if (adev->dev->kobj.sd) 2058 sysfs_remove_group(&adev->dev->kobj, &group); 2059 2060 return 0; 2061 } 2062 2063 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2064 struct ras_common_if *head) 2065 { 2066 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2067 2068 if (amdgpu_aca_is_enabled(adev)) 2069 return 0; 2070 2071 if (!obj || obj->attr_inuse) 2072 return -EINVAL; 2073 2074 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2075 return 0; 2076 2077 get_obj(obj); 2078 2079 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2080 "%s_err_count", head->name); 2081 2082 obj->sysfs_attr = (struct device_attribute){ 2083 .attr = { 2084 .name = obj->fs_data.sysfs_name, 2085 .mode = S_IRUGO, 2086 }, 2087 .show = amdgpu_ras_sysfs_read, 2088 }; 2089 sysfs_attr_init(&obj->sysfs_attr.attr); 2090 2091 if (sysfs_add_file_to_group(&adev->dev->kobj, 2092 &obj->sysfs_attr.attr, 2093 RAS_FS_NAME)) { 2094 put_obj(obj); 2095 return -EINVAL; 2096 } 2097 2098 obj->attr_inuse = 1; 2099 2100 return 0; 2101 } 2102 2103 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2104 struct ras_common_if *head) 2105 { 2106 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2107 2108 if (amdgpu_aca_is_enabled(adev)) 2109 return 0; 2110 2111 if (!obj || !obj->attr_inuse) 2112 return -EINVAL; 2113 2114 if (adev->dev->kobj.sd) 2115 sysfs_remove_file_from_group(&adev->dev->kobj, 2116 &obj->sysfs_attr.attr, 2117 RAS_FS_NAME); 2118 obj->attr_inuse = 0; 2119 put_obj(obj); 2120 2121 return 0; 2122 } 2123 2124 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2125 { 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2127 struct ras_manager *obj, *tmp; 2128 2129 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2130 amdgpu_ras_sysfs_remove(adev, &obj->head); 2131 } 2132 2133 if (amdgpu_bad_page_threshold != 0) 2134 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2135 2136 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2137 2138 return 0; 2139 } 2140 /* sysfs end */ 2141 2142 /** 2143 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2144 * 2145 * Normally when there is an uncorrectable error, the driver will reset 2146 * the GPU to recover. However, in the event of an unrecoverable error, 2147 * the driver provides an interface to reboot the system automatically 2148 * in that event. 2149 * 2150 * The following file in debugfs provides that interface: 2151 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2152 * 2153 * Usage: 2154 * 2155 * .. code-block:: bash 2156 * 2157 * echo true > .../ras/auto_reboot 2158 * 2159 */ 2160 /* debugfs begin */ 2161 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2162 { 2163 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2164 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2165 struct drm_minor *minor = adev_to_drm(adev)->primary; 2166 struct dentry *dir; 2167 2168 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2169 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2170 &amdgpu_ras_debugfs_ctrl_ops); 2171 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_eeprom_ops); 2173 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2174 &con->bad_page_cnt_threshold); 2175 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2176 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2177 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2178 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2179 &amdgpu_ras_debugfs_eeprom_size_ops); 2180 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2181 S_IRUGO, dir, adev, 2182 &amdgpu_ras_debugfs_eeprom_table_ops); 2183 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2184 2185 /* 2186 * After one uncorrectable error happens, usually GPU recovery will 2187 * be scheduled. But due to the known problem in GPU recovery failing 2188 * to bring GPU back, below interface provides one direct way to 2189 * user to reboot system automatically in such case within 2190 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2191 * will never be called. 2192 */ 2193 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2194 2195 /* 2196 * User could set this not to clean up hardware's error count register 2197 * of RAS IPs during ras recovery. 2198 */ 2199 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2200 &con->disable_ras_err_cnt_harvest); 2201 return dir; 2202 } 2203 2204 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2205 struct ras_fs_if *head, 2206 struct dentry *dir) 2207 { 2208 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2209 2210 if (!obj || !dir) 2211 return; 2212 2213 get_obj(obj); 2214 2215 memcpy(obj->fs_data.debugfs_name, 2216 head->debugfs_name, 2217 sizeof(obj->fs_data.debugfs_name)); 2218 2219 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2220 obj, &amdgpu_ras_debugfs_ops); 2221 } 2222 2223 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2224 { 2225 bool ret; 2226 2227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2228 case IP_VERSION(13, 0, 6): 2229 case IP_VERSION(13, 0, 12): 2230 case IP_VERSION(13, 0, 14): 2231 ret = true; 2232 break; 2233 default: 2234 ret = false; 2235 break; 2236 } 2237 2238 return ret; 2239 } 2240 2241 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2242 { 2243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2244 struct dentry *dir; 2245 struct ras_manager *obj; 2246 struct ras_fs_if fs_info; 2247 2248 /* 2249 * it won't be called in resume path, no need to check 2250 * suspend and gpu reset status 2251 */ 2252 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2253 return; 2254 2255 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2256 2257 list_for_each_entry(obj, &con->head, node) { 2258 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2259 (obj->attr_inuse == 1)) { 2260 sprintf(fs_info.debugfs_name, "%s_err_inject", 2261 get_ras_block_str(&obj->head)); 2262 fs_info.head = obj->head; 2263 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2264 } 2265 } 2266 2267 if (amdgpu_ras_aca_is_supported(adev)) { 2268 if (amdgpu_aca_is_enabled(adev)) 2269 amdgpu_aca_smu_debugfs_init(adev, dir); 2270 else 2271 amdgpu_mca_smu_debugfs_init(adev, dir); 2272 } 2273 } 2274 2275 /* debugfs end */ 2276 2277 /* ras fs */ 2278 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2279 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2280 static DEVICE_ATTR(features, S_IRUGO, 2281 amdgpu_ras_sysfs_features_read, NULL); 2282 static DEVICE_ATTR(version, 0444, 2283 amdgpu_ras_sysfs_version_show, NULL); 2284 static DEVICE_ATTR(schema, 0444, 2285 amdgpu_ras_sysfs_schema_show, NULL); 2286 static DEVICE_ATTR(event_state, 0444, 2287 amdgpu_ras_sysfs_event_state_show, NULL); 2288 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2289 { 2290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2291 struct attribute_group group = { 2292 .name = RAS_FS_NAME, 2293 }; 2294 struct attribute *attrs[] = { 2295 &con->features_attr.attr, 2296 &con->version_attr.attr, 2297 &con->schema_attr.attr, 2298 &con->event_state_attr.attr, 2299 NULL 2300 }; 2301 const struct bin_attribute *bin_attrs[] = { 2302 NULL, 2303 NULL, 2304 }; 2305 int r; 2306 2307 group.attrs = attrs; 2308 2309 /* add features entry */ 2310 con->features_attr = dev_attr_features; 2311 sysfs_attr_init(attrs[0]); 2312 2313 /* add version entry */ 2314 con->version_attr = dev_attr_version; 2315 sysfs_attr_init(attrs[1]); 2316 2317 /* add schema entry */ 2318 con->schema_attr = dev_attr_schema; 2319 sysfs_attr_init(attrs[2]); 2320 2321 /* add event_state entry */ 2322 con->event_state_attr = dev_attr_event_state; 2323 sysfs_attr_init(attrs[3]); 2324 2325 if (amdgpu_bad_page_threshold != 0) { 2326 /* add bad_page_features entry */ 2327 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2328 sysfs_bin_attr_init(&con->badpages_attr); 2329 bin_attrs[0] = &con->badpages_attr; 2330 group.bin_attrs = bin_attrs; 2331 } 2332 2333 r = sysfs_create_group(&adev->dev->kobj, &group); 2334 if (r) 2335 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2336 2337 return 0; 2338 } 2339 2340 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2341 { 2342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2343 struct ras_manager *con_obj, *ip_obj, *tmp; 2344 2345 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2346 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2347 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2348 if (ip_obj) 2349 put_obj(ip_obj); 2350 } 2351 } 2352 2353 amdgpu_ras_sysfs_remove_all(adev); 2354 return 0; 2355 } 2356 /* ras fs end */ 2357 2358 /* ih begin */ 2359 2360 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2361 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2362 * register to check whether the interrupt is triggered or not, and properly 2363 * ack the interrupt if it is there 2364 */ 2365 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2366 { 2367 /* Fatal error events are handled on host side */ 2368 if (amdgpu_sriov_vf(adev)) 2369 return; 2370 /* 2371 * If the current interrupt is caused by a non-fatal RAS error, skip 2372 * check for fatal error. For fatal errors, FED status of all devices 2373 * in XGMI hive gets set when the first device gets fatal error 2374 * interrupt. The error gets propagated to other devices as well, so 2375 * make sure to ack the interrupt regardless of FED status. 2376 */ 2377 if (!amdgpu_ras_get_fed_status(adev) && 2378 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2379 return; 2380 2381 if (amdgpu_uniras_enabled(adev)) { 2382 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2383 return; 2384 } 2385 2386 if (adev->nbio.ras && 2387 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2388 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2389 2390 if (adev->nbio.ras && 2391 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2392 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2393 } 2394 2395 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2396 struct amdgpu_iv_entry *entry) 2397 { 2398 bool poison_stat = false; 2399 struct amdgpu_device *adev = obj->adev; 2400 struct amdgpu_ras_block_object *block_obj = 2401 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2402 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2403 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2404 u64 event_id; 2405 int ret; 2406 2407 if (!block_obj || !con) 2408 return; 2409 2410 ret = amdgpu_ras_mark_ras_event(adev, type); 2411 if (ret) 2412 return; 2413 2414 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2415 /* both query_poison_status and handle_poison_consumption are optional, 2416 * but at least one of them should be implemented if we need poison 2417 * consumption handler 2418 */ 2419 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2420 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2421 if (!poison_stat) { 2422 /* Not poison consumption interrupt, no need to handle it */ 2423 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2424 block_obj->ras_comm.name); 2425 2426 return; 2427 } 2428 } 2429 2430 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2431 2432 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2433 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2434 2435 /* gpu reset is fallback for failed and default cases. 2436 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2437 */ 2438 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2439 event_id = amdgpu_ras_acquire_event_id(adev, type); 2440 RAS_EVENT_LOG(adev, event_id, 2441 "GPU reset for %s RAS poison consumption is issued!\n", 2442 block_obj->ras_comm.name); 2443 amdgpu_ras_reset_gpu(adev); 2444 } 2445 2446 if (!poison_stat) 2447 amdgpu_gfx_poison_consumption_handler(adev, entry); 2448 } 2449 2450 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2451 struct amdgpu_iv_entry *entry) 2452 { 2453 struct amdgpu_device *adev = obj->adev; 2454 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2455 u64 event_id; 2456 int ret; 2457 2458 ret = amdgpu_ras_mark_ras_event(adev, type); 2459 if (ret) 2460 return; 2461 2462 event_id = amdgpu_ras_acquire_event_id(adev, type); 2463 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2464 2465 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2466 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2467 2468 atomic_inc(&con->page_retirement_req_cnt); 2469 atomic_inc(&con->poison_creation_count); 2470 2471 wake_up(&con->page_retirement_wq); 2472 } 2473 } 2474 2475 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2476 struct amdgpu_iv_entry *entry) 2477 { 2478 struct ras_ih_data *data = &obj->ih_data; 2479 struct ras_err_data err_data; 2480 int ret; 2481 2482 if (!data->cb) 2483 return; 2484 2485 ret = amdgpu_ras_error_data_init(&err_data); 2486 if (ret) 2487 return; 2488 2489 /* Let IP handle its data, maybe we need get the output 2490 * from the callback to update the error type/count, etc 2491 */ 2492 amdgpu_ras_set_fed(obj->adev, true); 2493 ret = data->cb(obj->adev, &err_data, entry); 2494 /* ue will trigger an interrupt, and in that case 2495 * we need do a reset to recovery the whole system. 2496 * But leave IP do that recovery, here we just dispatch 2497 * the error. 2498 */ 2499 if (ret == AMDGPU_RAS_SUCCESS) { 2500 /* these counts could be left as 0 if 2501 * some blocks do not count error number 2502 */ 2503 obj->err_data.ue_count += err_data.ue_count; 2504 obj->err_data.ce_count += err_data.ce_count; 2505 obj->err_data.de_count += err_data.de_count; 2506 } 2507 2508 amdgpu_ras_error_data_fini(&err_data); 2509 } 2510 2511 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2512 { 2513 struct ras_ih_data *data = &obj->ih_data; 2514 struct amdgpu_iv_entry entry; 2515 2516 while (data->rptr != data->wptr) { 2517 rmb(); 2518 memcpy(&entry, &data->ring[data->rptr], 2519 data->element_size); 2520 2521 wmb(); 2522 data->rptr = (data->aligned_element_size + 2523 data->rptr) % data->ring_size; 2524 2525 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2526 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2527 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2528 else 2529 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2530 } else { 2531 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2532 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2533 else 2534 dev_warn(obj->adev->dev, 2535 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2536 } 2537 } 2538 } 2539 2540 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2541 { 2542 struct ras_ih_data *data = 2543 container_of(work, struct ras_ih_data, ih_work); 2544 struct ras_manager *obj = 2545 container_of(data, struct ras_manager, ih_data); 2546 2547 amdgpu_ras_interrupt_handler(obj); 2548 } 2549 2550 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2551 struct ras_dispatch_if *info) 2552 { 2553 struct ras_manager *obj; 2554 struct ras_ih_data *data; 2555 2556 if (amdgpu_uniras_enabled(adev)) { 2557 struct ras_ih_info ih_info; 2558 2559 memset(&ih_info, 0, sizeof(ih_info)); 2560 ih_info.block = info->head.block; 2561 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2562 2563 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2564 } 2565 2566 obj = amdgpu_ras_find_obj(adev, &info->head); 2567 if (!obj) 2568 return -EINVAL; 2569 2570 data = &obj->ih_data; 2571 2572 if (data->inuse == 0) 2573 return 0; 2574 2575 /* Might be overflow... */ 2576 memcpy(&data->ring[data->wptr], info->entry, 2577 data->element_size); 2578 2579 wmb(); 2580 data->wptr = (data->aligned_element_size + 2581 data->wptr) % data->ring_size; 2582 2583 schedule_work(&data->ih_work); 2584 2585 return 0; 2586 } 2587 2588 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2589 struct ras_common_if *head) 2590 { 2591 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2592 struct ras_ih_data *data; 2593 2594 if (!obj) 2595 return -EINVAL; 2596 2597 data = &obj->ih_data; 2598 if (data->inuse == 0) 2599 return 0; 2600 2601 cancel_work_sync(&data->ih_work); 2602 2603 kfree(data->ring); 2604 memset(data, 0, sizeof(*data)); 2605 put_obj(obj); 2606 2607 return 0; 2608 } 2609 2610 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2611 struct ras_common_if *head) 2612 { 2613 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2614 struct ras_ih_data *data; 2615 struct amdgpu_ras_block_object *ras_obj; 2616 2617 if (!obj) { 2618 /* in case we registe the IH before enable ras feature */ 2619 obj = amdgpu_ras_create_obj(adev, head); 2620 if (!obj) 2621 return -EINVAL; 2622 } else 2623 get_obj(obj); 2624 2625 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2626 2627 data = &obj->ih_data; 2628 /* add the callback.etc */ 2629 *data = (struct ras_ih_data) { 2630 .inuse = 0, 2631 .cb = ras_obj->ras_cb, 2632 .element_size = sizeof(struct amdgpu_iv_entry), 2633 .rptr = 0, 2634 .wptr = 0, 2635 }; 2636 2637 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2638 2639 data->aligned_element_size = ALIGN(data->element_size, 8); 2640 /* the ring can store 64 iv entries. */ 2641 data->ring_size = 64 * data->aligned_element_size; 2642 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2643 if (!data->ring) { 2644 put_obj(obj); 2645 return -ENOMEM; 2646 } 2647 2648 /* IH is ready */ 2649 data->inuse = 1; 2650 2651 return 0; 2652 } 2653 2654 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2655 { 2656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2657 struct ras_manager *obj, *tmp; 2658 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2660 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2661 } 2662 2663 return 0; 2664 } 2665 /* ih end */ 2666 2667 /* traversal all IPs except NBIO to query error counter */ 2668 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2669 { 2670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2671 struct ras_manager *obj; 2672 2673 if (!adev->ras_enabled || !con) 2674 return; 2675 2676 list_for_each_entry(obj, &con->head, node) { 2677 struct ras_query_if info = { 2678 .head = obj->head, 2679 }; 2680 2681 /* 2682 * PCIE_BIF IP has one different isr by ras controller 2683 * interrupt, the specific ras counter query will be 2684 * done in that isr. So skip such block from common 2685 * sync flood interrupt isr calling. 2686 */ 2687 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2688 continue; 2689 2690 /* 2691 * this is a workaround for aldebaran, skip send msg to 2692 * smu to get ecc_info table due to smu handle get ecc 2693 * info table failed temporarily. 2694 * should be removed until smu fix handle ecc_info table. 2695 */ 2696 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2697 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2698 IP_VERSION(13, 0, 2))) 2699 continue; 2700 2701 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2702 2703 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2704 IP_VERSION(11, 0, 2) && 2705 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 4) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(13, 0, 0)) { 2709 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2710 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2711 } 2712 } 2713 } 2714 2715 /* Parse RdRspStatus and WrRspStatus */ 2716 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2717 struct ras_query_if *info) 2718 { 2719 struct amdgpu_ras_block_object *block_obj; 2720 /* 2721 * Only two block need to query read/write 2722 * RspStatus at current state 2723 */ 2724 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2725 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2726 return; 2727 2728 block_obj = amdgpu_ras_get_ras_block(adev, 2729 info->head.block, 2730 info->head.sub_block_index); 2731 2732 if (!block_obj || !block_obj->hw_ops) { 2733 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2734 get_ras_block_str(&info->head)); 2735 return; 2736 } 2737 2738 if (block_obj->hw_ops->query_ras_error_status) 2739 block_obj->hw_ops->query_ras_error_status(adev); 2740 2741 } 2742 2743 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2744 { 2745 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2746 struct ras_manager *obj; 2747 2748 if (!adev->ras_enabled || !con) 2749 return; 2750 2751 list_for_each_entry(obj, &con->head, node) { 2752 struct ras_query_if info = { 2753 .head = obj->head, 2754 }; 2755 2756 amdgpu_ras_error_status_query(adev, &info); 2757 } 2758 } 2759 2760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2761 struct ras_badpage *bps, uint32_t count, uint32_t start) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_err_handler_data *data; 2765 int r = 0; 2766 uint32_t i; 2767 2768 if (!con || !con->eh_data || !bps || !count) 2769 return -EINVAL; 2770 2771 mutex_lock(&con->recovery_lock); 2772 data = con->eh_data; 2773 if (start < data->count) { 2774 for (i = start; i < data->count; i++) { 2775 if (!data->bps[i].ts) 2776 continue; 2777 2778 bps[r].bp = data->bps[i].retired_page; 2779 r++; 2780 if (r >= count) 2781 break; 2782 } 2783 } 2784 mutex_unlock(&con->recovery_lock); 2785 2786 return r; 2787 } 2788 2789 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2790 struct ras_badpage *bps, uint32_t count, uint32_t start) 2791 { 2792 struct ras_cmd_bad_pages_info_req cmd_input; 2793 struct ras_cmd_bad_pages_info_rsp *output; 2794 uint32_t group, start_group, end_group; 2795 uint32_t pos, pos_in_group; 2796 int r = 0, i; 2797 2798 if (!bps || !count) 2799 return -EINVAL; 2800 2801 output = kmalloc(sizeof(*output), GFP_KERNEL); 2802 if (!output) 2803 return -ENOMEM; 2804 2805 memset(&cmd_input, 0, sizeof(cmd_input)); 2806 2807 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2808 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2809 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2810 2811 pos = start; 2812 for (group = start_group; group < end_group; group++) { 2813 memset(output, 0, sizeof(*output)); 2814 cmd_input.group_index = group; 2815 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2816 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2817 goto out; 2818 2819 if (pos >= output->bp_total_cnt) 2820 goto out; 2821 2822 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2823 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2824 if (!output->records[i].ts) 2825 continue; 2826 2827 bps[r].bp = output->records[i].retired_page; 2828 r++; 2829 if (r >= count) 2830 goto out; 2831 } 2832 } 2833 2834 out: 2835 kfree(output); 2836 return r; 2837 } 2838 2839 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2840 struct amdgpu_hive_info *hive, bool status) 2841 { 2842 struct amdgpu_device *tmp_adev; 2843 2844 if (hive) { 2845 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2846 amdgpu_ras_set_fed(tmp_adev, status); 2847 } else { 2848 amdgpu_ras_set_fed(adev, status); 2849 } 2850 } 2851 2852 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2853 { 2854 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2856 int hive_ras_recovery = 0; 2857 2858 if (hive) { 2859 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2860 amdgpu_put_xgmi_hive(hive); 2861 } 2862 2863 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2864 return true; 2865 2866 return false; 2867 } 2868 2869 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2870 { 2871 if (amdgpu_ras_intr_triggered()) 2872 return RAS_EVENT_TYPE_FATAL; 2873 else 2874 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2875 } 2876 2877 static void amdgpu_ras_do_recovery(struct work_struct *work) 2878 { 2879 struct amdgpu_ras *ras = 2880 container_of(work, struct amdgpu_ras, recovery_work); 2881 struct amdgpu_device *remote_adev = NULL; 2882 struct amdgpu_device *adev = ras->adev; 2883 struct list_head device_list, *device_list_handle = NULL; 2884 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2885 unsigned int error_query_mode; 2886 enum ras_event_type type; 2887 2888 if (hive) { 2889 atomic_set(&hive->ras_recovery, 1); 2890 2891 /* If any device which is part of the hive received RAS fatal 2892 * error interrupt, set fatal error status on all. This 2893 * condition will need a recovery, and flag will be cleared 2894 * as part of recovery. 2895 */ 2896 list_for_each_entry(remote_adev, &hive->device_list, 2897 gmc.xgmi.head) 2898 if (amdgpu_ras_get_fed_status(remote_adev)) { 2899 amdgpu_ras_set_fed_all(adev, hive, true); 2900 break; 2901 } 2902 } 2903 if (!ras->disable_ras_err_cnt_harvest) { 2904 2905 /* Build list of devices to query RAS related errors */ 2906 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2907 device_list_handle = &hive->device_list; 2908 } else { 2909 INIT_LIST_HEAD(&device_list); 2910 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2911 device_list_handle = &device_list; 2912 } 2913 2914 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2915 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2916 /* wait 500ms to ensure pmfw polling mca bank info done */ 2917 msleep(500); 2918 } 2919 } 2920 2921 type = amdgpu_ras_get_fatal_error_event(adev); 2922 list_for_each_entry(remote_adev, 2923 device_list_handle, gmc.xgmi.head) { 2924 amdgpu_ras_query_err_status(remote_adev); 2925 amdgpu_ras_log_on_err_counter(remote_adev, type); 2926 } 2927 2928 } 2929 2930 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2931 struct amdgpu_reset_context reset_context; 2932 memset(&reset_context, 0, sizeof(reset_context)); 2933 2934 reset_context.method = AMD_RESET_METHOD_NONE; 2935 reset_context.reset_req_dev = adev; 2936 reset_context.src = AMDGPU_RESET_SRC_RAS; 2937 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2938 2939 /* Perform full reset in fatal error mode */ 2940 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2941 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2942 else { 2943 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2944 2945 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2946 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2947 reset_context.method = AMD_RESET_METHOD_MODE2; 2948 } 2949 2950 /* Fatal error occurs in poison mode, mode1 reset is used to 2951 * recover gpu. 2952 */ 2953 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2954 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2955 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2956 2957 psp_fatal_error_recovery_quirk(&adev->psp); 2958 } 2959 } 2960 2961 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2962 } 2963 atomic_set(&ras->in_recovery, 0); 2964 if (hive) { 2965 atomic_set(&hive->ras_recovery, 0); 2966 amdgpu_put_xgmi_hive(hive); 2967 } 2968 } 2969 2970 /* alloc/realloc bps array */ 2971 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2972 struct ras_err_handler_data *data, int pages) 2973 { 2974 unsigned int old_space = data->count + data->space_left; 2975 unsigned int new_space = old_space + pages; 2976 unsigned int align_space = ALIGN(new_space, 512); 2977 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2978 2979 if (!bps) { 2980 return -ENOMEM; 2981 } 2982 2983 if (data->bps) { 2984 memcpy(bps, data->bps, 2985 data->count * sizeof(*data->bps)); 2986 kfree(data->bps); 2987 } 2988 2989 data->bps = bps; 2990 data->space_left += align_space - old_space; 2991 return 0; 2992 } 2993 2994 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2995 struct eeprom_table_record *bps, 2996 struct ras_err_data *err_data) 2997 { 2998 struct ta_ras_query_address_input addr_in; 2999 uint32_t socket = 0; 3000 int ret = 0; 3001 3002 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3003 socket = adev->smuio.funcs->get_socket_id(adev); 3004 3005 /* reinit err_data */ 3006 err_data->err_addr_cnt = 0; 3007 err_data->err_addr_len = adev->umc.retire_unit; 3008 3009 memset(&addr_in, 0, sizeof(addr_in)); 3010 addr_in.ma.err_addr = bps->address; 3011 addr_in.ma.socket_id = socket; 3012 addr_in.ma.ch_inst = bps->mem_channel; 3013 /* tell RAS TA the node instance is not used */ 3014 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3015 3016 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3017 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3018 &addr_in, NULL, false); 3019 3020 return ret; 3021 } 3022 3023 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3024 struct eeprom_table_record *bps, 3025 struct ras_err_data *err_data) 3026 { 3027 struct ta_ras_query_address_input addr_in; 3028 uint32_t die_id, socket = 0; 3029 3030 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3031 socket = adev->smuio.funcs->get_socket_id(adev); 3032 3033 /* although die id is gotten from PA in nps1 mode, the id is 3034 * fitable for any nps mode 3035 */ 3036 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3037 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3038 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3039 else 3040 return -EINVAL; 3041 3042 /* reinit err_data */ 3043 err_data->err_addr_cnt = 0; 3044 err_data->err_addr_len = adev->umc.retire_unit; 3045 3046 memset(&addr_in, 0, sizeof(addr_in)); 3047 addr_in.ma.err_addr = bps->address; 3048 addr_in.ma.ch_inst = bps->mem_channel; 3049 addr_in.ma.umc_inst = bps->mcumc_id; 3050 addr_in.ma.node_inst = die_id; 3051 addr_in.ma.socket_id = socket; 3052 3053 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3054 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3055 &addr_in, NULL, false); 3056 else 3057 return -EINVAL; 3058 } 3059 3060 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3061 struct eeprom_table_record *bps, int count) 3062 { 3063 int j; 3064 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3065 struct ras_err_handler_data *data = con->eh_data; 3066 3067 for (j = 0; j < count; j++) { 3068 if (amdgpu_ras_check_bad_page_unlock(con, 3069 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3070 data->count++; 3071 data->space_left--; 3072 continue; 3073 } 3074 3075 if (!data->space_left && 3076 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3077 return -ENOMEM; 3078 } 3079 3080 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3081 3082 memcpy(&data->bps[data->count], &(bps[j]), 3083 sizeof(struct eeprom_table_record)); 3084 data->count++; 3085 data->space_left--; 3086 con->bad_page_num++; 3087 } 3088 3089 return 0; 3090 } 3091 3092 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3093 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3094 enum amdgpu_memory_partition nps) 3095 { 3096 int i = 0; 3097 enum amdgpu_memory_partition save_nps; 3098 3099 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3100 3101 /*old asics just have pa in eeprom*/ 3102 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3103 memcpy(err_data->err_addr, bps, 3104 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3105 goto out; 3106 } 3107 3108 for (i = 0; i < adev->umc.retire_unit; i++) 3109 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3110 3111 if (save_nps) { 3112 if (save_nps == nps) { 3113 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3114 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3115 return -EINVAL; 3116 for (i = 0; i < adev->umc.retire_unit; i++) { 3117 err_data->err_addr[i].address = bps[0].address; 3118 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3119 err_data->err_addr[i].bank = bps[0].bank; 3120 err_data->err_addr[i].err_type = bps[0].err_type; 3121 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3122 } 3123 } else { 3124 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3125 return -EINVAL; 3126 } 3127 } else { 3128 if (bps[0].address == 0) { 3129 /* for specific old eeprom data, mca address is not stored, 3130 * calc it from pa 3131 */ 3132 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3133 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3134 return -EINVAL; 3135 } 3136 3137 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3138 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3139 memcpy(err_data->err_addr, bps, 3140 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3141 else 3142 return -EOPNOTSUPP; 3143 } 3144 } 3145 3146 out: 3147 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3148 } 3149 3150 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3151 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3152 enum amdgpu_memory_partition nps) 3153 { 3154 int i = 0; 3155 enum amdgpu_memory_partition save_nps; 3156 3157 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3158 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3159 3160 if (save_nps == nps) { 3161 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3162 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3163 return -EINVAL; 3164 for (i = 0; i < adev->umc.retire_unit; i++) { 3165 err_data->err_addr[i].address = bps->address; 3166 err_data->err_addr[i].mem_channel = bps->mem_channel; 3167 err_data->err_addr[i].bank = bps->bank; 3168 err_data->err_addr[i].err_type = bps->err_type; 3169 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3170 } 3171 } else { 3172 if (bps->address) { 3173 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3174 return -EINVAL; 3175 } else { 3176 /* for specific old eeprom data, mca address is not stored, 3177 * calc it from pa 3178 */ 3179 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3180 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3181 return -EINVAL; 3182 3183 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3184 return -EOPNOTSUPP; 3185 } 3186 } 3187 3188 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3189 adev->umc.retire_unit); 3190 } 3191 3192 /* it deal with vram only. */ 3193 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3194 struct eeprom_table_record *bps, int pages, bool from_rom) 3195 { 3196 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3197 struct ras_err_data err_data; 3198 struct amdgpu_ras_eeprom_control *control = 3199 &adev->psp.ras_context.ras->eeprom_control; 3200 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3201 int ret = 0; 3202 uint32_t i = 0; 3203 3204 if (!con || !con->eh_data || !bps || pages <= 0) 3205 return 0; 3206 3207 if (from_rom) { 3208 err_data.err_addr = 3209 kcalloc(adev->umc.retire_unit, 3210 sizeof(struct eeprom_table_record), GFP_KERNEL); 3211 if (!err_data.err_addr) { 3212 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3213 return -ENOMEM; 3214 } 3215 3216 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3217 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3218 } 3219 3220 mutex_lock(&con->recovery_lock); 3221 3222 if (from_rom) { 3223 /* there is no pa recs in V3, so skip pa recs processing */ 3224 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3225 for (i = 0; i < pages; i++) { 3226 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3227 if ((bps[i].address == bps[i + 1].address) && 3228 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3229 /* deal with retire_unit records a time */ 3230 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3231 &bps[i], &err_data, nps); 3232 if (ret) 3233 con->bad_page_num -= adev->umc.retire_unit; 3234 i += (adev->umc.retire_unit - 1); 3235 } else { 3236 break; 3237 } 3238 } else { 3239 break; 3240 } 3241 } 3242 } 3243 for (; i < pages; i++) { 3244 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3245 &bps[i], &err_data, nps); 3246 if (ret) 3247 con->bad_page_num -= adev->umc.retire_unit; 3248 } 3249 3250 con->eh_data->count_saved = con->eh_data->count; 3251 } else { 3252 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3253 } 3254 3255 if (from_rom) 3256 kfree(err_data.err_addr); 3257 mutex_unlock(&con->recovery_lock); 3258 3259 return ret; 3260 } 3261 3262 /* 3263 * write error record array to eeprom, the function should be 3264 * protected by recovery_lock 3265 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3266 */ 3267 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3268 unsigned long *new_cnt) 3269 { 3270 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3271 struct ras_err_handler_data *data; 3272 struct amdgpu_ras_eeprom_control *control; 3273 int save_count, unit_num, i; 3274 3275 if (!con || !con->eh_data) { 3276 if (new_cnt) 3277 *new_cnt = 0; 3278 3279 return 0; 3280 } 3281 3282 if (!con->eeprom_control.is_eeprom_valid) { 3283 dev_warn(adev->dev, 3284 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3285 if (new_cnt) 3286 *new_cnt = 0; 3287 3288 return 0; 3289 } 3290 3291 mutex_lock(&con->recovery_lock); 3292 control = &con->eeprom_control; 3293 data = con->eh_data; 3294 unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; 3295 save_count = con->bad_page_num - control->ras_num_bad_pages; 3296 mutex_unlock(&con->recovery_lock); 3297 3298 if (new_cnt) 3299 *new_cnt = unit_num; 3300 3301 /* only new entries are saved */ 3302 if (unit_num && save_count) { 3303 /*old asics only save pa to eeprom like before*/ 3304 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3305 if (amdgpu_ras_eeprom_append(control, 3306 &data->bps[data->count_saved], unit_num)) { 3307 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3308 return -EIO; 3309 } 3310 } else { 3311 for (i = 0; i < unit_num; i++) { 3312 if (amdgpu_ras_eeprom_append(control, 3313 &data->bps[data->count_saved + 3314 i * adev->umc.retire_unit], 1)) { 3315 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3316 return -EIO; 3317 } 3318 } 3319 } 3320 3321 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3322 data->count_saved = data->count; 3323 } 3324 3325 return 0; 3326 } 3327 3328 /* 3329 * read error record array in eeprom and reserve enough space for 3330 * storing new bad pages 3331 */ 3332 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3333 { 3334 struct amdgpu_ras_eeprom_control *control = 3335 &adev->psp.ras_context.ras->eeprom_control; 3336 struct eeprom_table_record *bps; 3337 int ret, i = 0; 3338 3339 /* no bad page record, skip eeprom access */ 3340 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3341 return 0; 3342 3343 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3344 if (!bps) 3345 return -ENOMEM; 3346 3347 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3348 if (ret) { 3349 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3350 } else { 3351 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3352 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3353 as pa recs, so add verion check to avoid it. 3354 */ 3355 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { 3356 for (i = 0; i < control->ras_num_recs; i++) { 3357 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3358 if ((bps[i].address == bps[i + 1].address) && 3359 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3360 control->ras_num_pa_recs += adev->umc.retire_unit; 3361 i += (adev->umc.retire_unit - 1); 3362 } else { 3363 control->ras_num_mca_recs += 3364 (control->ras_num_recs - i); 3365 break; 3366 } 3367 } else { 3368 control->ras_num_mca_recs += (control->ras_num_recs - i); 3369 break; 3370 } 3371 } 3372 } else { 3373 control->ras_num_mca_recs = control->ras_num_recs; 3374 } 3375 } 3376 3377 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3378 if (ret) 3379 goto out; 3380 3381 ret = amdgpu_ras_eeprom_check(control); 3382 if (ret) 3383 goto out; 3384 3385 /* HW not usable */ 3386 if (amdgpu_ras_is_rma(adev)) 3387 ret = -EHWPOISON; 3388 } 3389 3390 out: 3391 kfree(bps); 3392 return ret; 3393 } 3394 3395 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3396 uint64_t addr) 3397 { 3398 struct ras_err_handler_data *data = con->eh_data; 3399 struct amdgpu_device *adev = con->adev; 3400 int i; 3401 3402 if ((addr >= adev->gmc.mc_vram_size && 3403 adev->gmc.mc_vram_size) || 3404 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3405 return -EINVAL; 3406 3407 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3408 for (i = 0; i < data->count; i++) 3409 if (addr == data->bps[i].retired_page) 3410 return 1; 3411 3412 return 0; 3413 } 3414 3415 /* 3416 * check if an address belongs to bad page 3417 * 3418 * Note: this check is only for umc block 3419 */ 3420 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3421 uint64_t addr) 3422 { 3423 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3424 int ret = 0; 3425 3426 if (!con || !con->eh_data) 3427 return ret; 3428 3429 mutex_lock(&con->recovery_lock); 3430 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3431 mutex_unlock(&con->recovery_lock); 3432 return ret; 3433 } 3434 3435 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3436 uint32_t max_count) 3437 { 3438 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3439 3440 /* 3441 * amdgpu_bad_page_threshold is used to config 3442 * the threshold for the number of bad pages. 3443 * -1: Threshold is set to default value 3444 * Driver will issue a warning message when threshold is reached 3445 * and continue runtime services. 3446 * 0: Disable bad page retirement 3447 * Driver will not retire bad pages 3448 * which is intended for debugging purpose. 3449 * -2: Threshold is determined by a formula 3450 * that assumes 1 bad page per 100M of local memory. 3451 * Driver will continue runtime services when threhold is reached. 3452 * 0 < threshold < max number of bad page records in EEPROM, 3453 * A user-defined threshold is set 3454 * Driver will halt runtime services when this custom threshold is reached. 3455 */ 3456 if (amdgpu_bad_page_threshold == -2) { 3457 u64 val = adev->gmc.mc_vram_size; 3458 3459 do_div(val, RAS_BAD_PAGE_COVER); 3460 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3461 max_count); 3462 } else if (amdgpu_bad_page_threshold == -1) { 3463 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3464 } else { 3465 con->bad_page_cnt_threshold = min_t(int, max_count, 3466 amdgpu_bad_page_threshold); 3467 } 3468 } 3469 3470 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3471 enum amdgpu_ras_block block, uint16_t pasid, 3472 pasid_notify pasid_fn, void *data, uint32_t reset) 3473 { 3474 int ret = 0; 3475 struct ras_poison_msg poison_msg; 3476 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3477 3478 memset(&poison_msg, 0, sizeof(poison_msg)); 3479 poison_msg.block = block; 3480 poison_msg.pasid = pasid; 3481 poison_msg.reset = reset; 3482 poison_msg.pasid_fn = pasid_fn; 3483 poison_msg.data = data; 3484 3485 ret = kfifo_put(&con->poison_fifo, poison_msg); 3486 if (!ret) { 3487 dev_err(adev->dev, "Poison message fifo is full!\n"); 3488 return -ENOSPC; 3489 } 3490 3491 return 0; 3492 } 3493 3494 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3495 struct ras_poison_msg *poison_msg) 3496 { 3497 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3498 3499 return kfifo_get(&con->poison_fifo, poison_msg); 3500 } 3501 3502 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3503 { 3504 mutex_init(&ecc_log->lock); 3505 3506 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3507 ecc_log->de_queried_count = 0; 3508 ecc_log->consumption_q_count = 0; 3509 } 3510 3511 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3512 { 3513 struct radix_tree_iter iter; 3514 void __rcu **slot; 3515 struct ras_ecc_err *ecc_err; 3516 3517 mutex_lock(&ecc_log->lock); 3518 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3519 ecc_err = radix_tree_deref_slot(slot); 3520 kfree(ecc_err->err_pages.pfn); 3521 kfree(ecc_err); 3522 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3523 } 3524 mutex_unlock(&ecc_log->lock); 3525 3526 mutex_destroy(&ecc_log->lock); 3527 ecc_log->de_queried_count = 0; 3528 ecc_log->consumption_q_count = 0; 3529 } 3530 3531 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3532 uint32_t delayed_ms) 3533 { 3534 int ret; 3535 3536 mutex_lock(&con->umc_ecc_log.lock); 3537 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3538 UMC_ECC_NEW_DETECTED_TAG); 3539 mutex_unlock(&con->umc_ecc_log.lock); 3540 3541 if (ret) 3542 schedule_delayed_work(&con->page_retirement_dwork, 3543 msecs_to_jiffies(delayed_ms)); 3544 3545 return ret ? true : false; 3546 } 3547 3548 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3549 { 3550 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3551 page_retirement_dwork.work); 3552 struct amdgpu_device *adev = con->adev; 3553 struct ras_err_data err_data; 3554 3555 /* If gpu reset is ongoing, delay retiring the bad pages */ 3556 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3557 amdgpu_ras_schedule_retirement_dwork(con, 3558 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3559 return; 3560 } 3561 3562 amdgpu_ras_error_data_init(&err_data); 3563 3564 amdgpu_umc_handle_bad_pages(adev, &err_data); 3565 3566 amdgpu_ras_error_data_fini(&err_data); 3567 3568 amdgpu_ras_schedule_retirement_dwork(con, 3569 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3570 } 3571 3572 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3573 uint32_t poison_creation_count) 3574 { 3575 int ret = 0; 3576 struct ras_ecc_log_info *ecc_log; 3577 struct ras_query_if info; 3578 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3579 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3580 u64 de_queried_count; 3581 u64 consumption_q_count; 3582 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3583 3584 memset(&info, 0, sizeof(info)); 3585 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3586 3587 ecc_log = &ras->umc_ecc_log; 3588 ecc_log->de_queried_count = 0; 3589 ecc_log->consumption_q_count = 0; 3590 3591 do { 3592 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3593 if (ret) 3594 return ret; 3595 3596 de_queried_count = ecc_log->de_queried_count; 3597 consumption_q_count = ecc_log->consumption_q_count; 3598 3599 if (de_queried_count && consumption_q_count) 3600 break; 3601 3602 msleep(100); 3603 } while (--timeout); 3604 3605 if (de_queried_count) 3606 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3607 3608 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3609 amdgpu_ras_reset_gpu(adev); 3610 3611 return 0; 3612 } 3613 3614 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3615 { 3616 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3617 struct ras_poison_msg msg; 3618 int ret; 3619 3620 do { 3621 ret = kfifo_get(&con->poison_fifo, &msg); 3622 } while (ret); 3623 } 3624 3625 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3626 uint32_t msg_count, uint32_t *gpu_reset) 3627 { 3628 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3629 uint32_t reset_flags = 0, reset = 0; 3630 struct ras_poison_msg msg; 3631 int ret, i; 3632 3633 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3634 3635 for (i = 0; i < msg_count; i++) { 3636 ret = amdgpu_ras_get_poison_req(adev, &msg); 3637 if (!ret) 3638 continue; 3639 3640 if (msg.pasid_fn) 3641 msg.pasid_fn(adev, msg.pasid, msg.data); 3642 3643 reset_flags |= msg.reset; 3644 } 3645 3646 /* 3647 * Try to ensure poison creation handler is completed first 3648 * to set rma if bad page exceed threshold. 3649 */ 3650 flush_delayed_work(&con->page_retirement_dwork); 3651 3652 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3653 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3654 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3655 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3656 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3657 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3658 else 3659 reset = reset_flags; 3660 3661 con->gpu_reset_flags |= reset; 3662 amdgpu_ras_reset_gpu(adev); 3663 3664 *gpu_reset = reset; 3665 3666 /* Wait for gpu recovery to complete */ 3667 flush_work(&con->recovery_work); 3668 } 3669 3670 return 0; 3671 } 3672 3673 static int amdgpu_ras_page_retirement_thread(void *param) 3674 { 3675 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3676 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3677 uint32_t poison_creation_count, msg_count; 3678 uint32_t gpu_reset; 3679 int ret; 3680 3681 while (!kthread_should_stop()) { 3682 3683 wait_event_interruptible(con->page_retirement_wq, 3684 kthread_should_stop() || 3685 atomic_read(&con->page_retirement_req_cnt)); 3686 3687 if (kthread_should_stop()) 3688 break; 3689 3690 mutex_lock(&con->poison_lock); 3691 gpu_reset = 0; 3692 3693 do { 3694 poison_creation_count = atomic_read(&con->poison_creation_count); 3695 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3696 if (ret == -EIO) 3697 break; 3698 3699 if (poison_creation_count) { 3700 atomic_sub(poison_creation_count, &con->poison_creation_count); 3701 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3702 } 3703 } while (atomic_read(&con->poison_creation_count) && 3704 !atomic_read(&con->poison_consumption_count)); 3705 3706 if (ret != -EIO) { 3707 msg_count = kfifo_len(&con->poison_fifo); 3708 if (msg_count) { 3709 ret = amdgpu_ras_poison_consumption_handler(adev, 3710 msg_count, &gpu_reset); 3711 if ((ret != -EIO) && 3712 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3713 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3714 } 3715 } 3716 3717 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3718 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3719 /* Clear poison creation request */ 3720 atomic_set(&con->poison_creation_count, 0); 3721 atomic_set(&con->poison_consumption_count, 0); 3722 3723 /* Clear poison fifo */ 3724 amdgpu_ras_clear_poison_fifo(adev); 3725 3726 /* Clear all poison requests */ 3727 atomic_set(&con->page_retirement_req_cnt, 0); 3728 3729 if (ret == -EIO) { 3730 /* Wait for mode-1 reset to complete */ 3731 down_read(&adev->reset_domain->sem); 3732 up_read(&adev->reset_domain->sem); 3733 } 3734 3735 /* Wake up work to save bad pages to eeprom */ 3736 schedule_delayed_work(&con->page_retirement_dwork, 0); 3737 } else if (gpu_reset) { 3738 /* gpu just completed mode-2 reset or other reset */ 3739 /* Clear poison consumption messages cached in fifo */ 3740 msg_count = kfifo_len(&con->poison_fifo); 3741 if (msg_count) { 3742 amdgpu_ras_clear_poison_fifo(adev); 3743 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3744 } 3745 3746 atomic_set(&con->poison_consumption_count, 0); 3747 3748 /* Wake up work to save bad pages to eeprom */ 3749 schedule_delayed_work(&con->page_retirement_dwork, 0); 3750 } 3751 mutex_unlock(&con->poison_lock); 3752 } 3753 3754 return 0; 3755 } 3756 3757 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3758 { 3759 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3760 struct amdgpu_ras_eeprom_control *control; 3761 int ret; 3762 3763 if (!con || amdgpu_sriov_vf(adev)) 3764 return 0; 3765 3766 if (amdgpu_uniras_enabled(adev)) 3767 return 0; 3768 3769 control = &con->eeprom_control; 3770 ret = amdgpu_ras_eeprom_init(control); 3771 control->is_eeprom_valid = !ret; 3772 3773 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3774 control->ras_num_pa_recs = control->ras_num_recs; 3775 3776 if (adev->umc.ras && 3777 adev->umc.ras->get_retire_flip_bits) 3778 adev->umc.ras->get_retire_flip_bits(adev); 3779 3780 if (control->ras_num_recs && control->is_eeprom_valid) { 3781 ret = amdgpu_ras_load_bad_pages(adev); 3782 if (ret) { 3783 control->is_eeprom_valid = false; 3784 return 0; 3785 } 3786 3787 amdgpu_dpm_send_hbm_bad_pages_num( 3788 adev, control->ras_num_bad_pages); 3789 3790 if (con->update_channel_flag == true) { 3791 amdgpu_dpm_send_hbm_bad_channel_flag( 3792 adev, control->bad_channel_bitmap); 3793 con->update_channel_flag = false; 3794 } 3795 3796 /* The format action is only applied to new ASICs */ 3797 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3798 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3799 if (!amdgpu_ras_eeprom_reset_table(control)) 3800 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3801 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3802 } 3803 3804 return 0; 3805 } 3806 3807 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3808 { 3809 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3810 struct ras_err_handler_data **data; 3811 u32 max_eeprom_records_count = 0; 3812 int ret; 3813 3814 if (!con || amdgpu_sriov_vf(adev)) 3815 return 0; 3816 3817 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3818 * supports RAS and debugfs is enabled, but when 3819 * adev->ras_enabled is unset, i.e. when "ras_enable" 3820 * module parameter is set to 0. 3821 */ 3822 con->adev = adev; 3823 3824 if (!adev->ras_enabled) 3825 return 0; 3826 3827 data = &con->eh_data; 3828 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3829 if (!*data) { 3830 ret = -ENOMEM; 3831 goto out; 3832 } 3833 3834 mutex_init(&con->recovery_lock); 3835 mutex_init(&con->poison_lock); 3836 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3837 atomic_set(&con->in_recovery, 0); 3838 atomic_set(&con->rma_in_recovery, 0); 3839 con->eeprom_control.bad_channel_bitmap = 0; 3840 3841 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3842 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3843 3844 if (init_bp_info) { 3845 ret = amdgpu_ras_init_badpage_info(adev); 3846 if (ret) 3847 goto free; 3848 } 3849 3850 mutex_init(&con->page_rsv_lock); 3851 INIT_KFIFO(con->poison_fifo); 3852 mutex_init(&con->page_retirement_lock); 3853 init_waitqueue_head(&con->page_retirement_wq); 3854 atomic_set(&con->page_retirement_req_cnt, 0); 3855 atomic_set(&con->poison_creation_count, 0); 3856 atomic_set(&con->poison_consumption_count, 0); 3857 con->page_retirement_thread = 3858 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3859 if (IS_ERR(con->page_retirement_thread)) { 3860 con->page_retirement_thread = NULL; 3861 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3862 } 3863 3864 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3865 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3866 #ifdef CONFIG_X86_MCE_AMD 3867 if ((adev->asic_type == CHIP_ALDEBARAN) && 3868 (adev->gmc.xgmi.connected_to_cpu)) 3869 amdgpu_register_bad_pages_mca_notifier(adev); 3870 #endif 3871 return 0; 3872 3873 free: 3874 kfree((*data)->bps); 3875 kfree(*data); 3876 con->eh_data = NULL; 3877 out: 3878 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3879 3880 /* 3881 * Except error threshold exceeding case, other failure cases in this 3882 * function would not fail amdgpu driver init. 3883 */ 3884 if (!amdgpu_ras_is_rma(adev)) 3885 ret = 0; 3886 else 3887 ret = -EINVAL; 3888 3889 return ret; 3890 } 3891 3892 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3893 { 3894 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3895 struct ras_err_handler_data *data = con->eh_data; 3896 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3897 bool ret; 3898 3899 /* recovery_init failed to init it, fini is useless */ 3900 if (!data) 3901 return 0; 3902 3903 /* Save all cached bad pages to eeprom */ 3904 do { 3905 flush_delayed_work(&con->page_retirement_dwork); 3906 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3907 } while (ret && max_flush_timeout--); 3908 3909 if (con->page_retirement_thread) 3910 kthread_stop(con->page_retirement_thread); 3911 3912 atomic_set(&con->page_retirement_req_cnt, 0); 3913 atomic_set(&con->poison_creation_count, 0); 3914 3915 mutex_destroy(&con->page_rsv_lock); 3916 3917 cancel_work_sync(&con->recovery_work); 3918 3919 cancel_delayed_work_sync(&con->page_retirement_dwork); 3920 3921 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3922 3923 mutex_lock(&con->recovery_lock); 3924 con->eh_data = NULL; 3925 kfree(data->bps); 3926 kfree(data); 3927 mutex_unlock(&con->recovery_lock); 3928 3929 amdgpu_ras_critical_region_init(adev); 3930 3931 return 0; 3932 } 3933 /* recovery end */ 3934 3935 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3936 { 3937 if (amdgpu_sriov_vf(adev)) { 3938 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3939 case IP_VERSION(13, 0, 2): 3940 case IP_VERSION(13, 0, 6): 3941 case IP_VERSION(13, 0, 12): 3942 case IP_VERSION(13, 0, 14): 3943 return true; 3944 default: 3945 return false; 3946 } 3947 } 3948 3949 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3950 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3951 case IP_VERSION(13, 0, 0): 3952 case IP_VERSION(13, 0, 6): 3953 case IP_VERSION(13, 0, 10): 3954 case IP_VERSION(13, 0, 12): 3955 case IP_VERSION(13, 0, 14): 3956 case IP_VERSION(14, 0, 3): 3957 return true; 3958 default: 3959 return false; 3960 } 3961 } 3962 3963 return adev->asic_type == CHIP_VEGA10 || 3964 adev->asic_type == CHIP_VEGA20 || 3965 adev->asic_type == CHIP_ARCTURUS || 3966 adev->asic_type == CHIP_ALDEBARAN || 3967 adev->asic_type == CHIP_SIENNA_CICHLID; 3968 } 3969 3970 /* 3971 * this is workaround for vega20 workstation sku, 3972 * force enable gfx ras, ignore vbios gfx ras flag 3973 * due to GC EDC can not write 3974 */ 3975 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3976 { 3977 struct atom_context *ctx = adev->mode_info.atom_context; 3978 3979 if (!ctx) 3980 return; 3981 3982 if (strnstr(ctx->vbios_pn, "D16406", 3983 sizeof(ctx->vbios_pn)) || 3984 strnstr(ctx->vbios_pn, "D36002", 3985 sizeof(ctx->vbios_pn))) 3986 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3987 } 3988 3989 /* Query ras capablity via atomfirmware interface */ 3990 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3991 { 3992 /* mem_ecc cap */ 3993 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3994 dev_info(adev->dev, "MEM ECC is active.\n"); 3995 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3996 1 << AMDGPU_RAS_BLOCK__DF); 3997 } else { 3998 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3999 } 4000 4001 /* sram_ecc cap */ 4002 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4003 dev_info(adev->dev, "SRAM ECC is active.\n"); 4004 if (!amdgpu_sriov_vf(adev)) 4005 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4006 1 << AMDGPU_RAS_BLOCK__DF); 4007 else 4008 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4009 1 << AMDGPU_RAS_BLOCK__SDMA | 4010 1 << AMDGPU_RAS_BLOCK__GFX); 4011 4012 /* 4013 * VCN/JPEG RAS can be supported on both bare metal and 4014 * SRIOV environment 4015 */ 4016 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4017 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4018 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4019 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4020 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4021 1 << AMDGPU_RAS_BLOCK__JPEG); 4022 else 4023 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4024 1 << AMDGPU_RAS_BLOCK__JPEG); 4025 4026 /* 4027 * XGMI RAS is not supported if xgmi num physical nodes 4028 * is zero 4029 */ 4030 if (!adev->gmc.xgmi.num_physical_nodes) 4031 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4032 } else { 4033 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4034 } 4035 } 4036 4037 /* Query poison mode from umc/df IP callbacks */ 4038 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4039 { 4040 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4041 bool df_poison, umc_poison; 4042 4043 /* poison setting is useless on SRIOV guest */ 4044 if (amdgpu_sriov_vf(adev) || !con) 4045 return; 4046 4047 /* Init poison supported flag, the default value is false */ 4048 if (adev->gmc.xgmi.connected_to_cpu || 4049 adev->gmc.is_app_apu) { 4050 /* enabled by default when GPU is connected to CPU */ 4051 con->poison_supported = true; 4052 } else if (adev->df.funcs && 4053 adev->df.funcs->query_ras_poison_mode && 4054 adev->umc.ras && 4055 adev->umc.ras->query_ras_poison_mode) { 4056 df_poison = 4057 adev->df.funcs->query_ras_poison_mode(adev); 4058 umc_poison = 4059 adev->umc.ras->query_ras_poison_mode(adev); 4060 4061 /* Only poison is set in both DF and UMC, we can support it */ 4062 if (df_poison && umc_poison) 4063 con->poison_supported = true; 4064 else if (df_poison != umc_poison) 4065 dev_warn(adev->dev, 4066 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4067 df_poison, umc_poison); 4068 } 4069 } 4070 4071 /* 4072 * check hardware's ras ability which will be saved in hw_supported. 4073 * if hardware does not support ras, we can skip some ras initializtion and 4074 * forbid some ras operations from IP. 4075 * if software itself, say boot parameter, limit the ras ability. We still 4076 * need allow IP do some limited operations, like disable. In such case, 4077 * we have to initialize ras as normal. but need check if operation is 4078 * allowed or not in each function. 4079 */ 4080 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4081 { 4082 adev->ras_hw_enabled = adev->ras_enabled = 0; 4083 4084 if (!amdgpu_ras_asic_supported(adev)) 4085 return; 4086 4087 if (amdgpu_sriov_vf(adev)) { 4088 if (amdgpu_virt_get_ras_capability(adev)) 4089 goto init_ras_enabled_flag; 4090 } 4091 4092 /* query ras capability from psp */ 4093 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4094 goto init_ras_enabled_flag; 4095 4096 /* query ras capablity from bios */ 4097 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4098 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4099 } else { 4100 /* driver only manages a few IP blocks RAS feature 4101 * when GPU is connected cpu through XGMI */ 4102 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4103 1 << AMDGPU_RAS_BLOCK__SDMA | 4104 1 << AMDGPU_RAS_BLOCK__MMHUB); 4105 } 4106 4107 /* apply asic specific settings (vega20 only for now) */ 4108 amdgpu_ras_get_quirks(adev); 4109 4110 /* query poison mode from umc/df ip callback */ 4111 amdgpu_ras_query_poison_mode(adev); 4112 4113 init_ras_enabled_flag: 4114 /* hw_supported needs to be aligned with RAS block mask. */ 4115 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4116 4117 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4118 adev->ras_hw_enabled & amdgpu_ras_mask; 4119 4120 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4121 if (!amdgpu_sriov_vf(adev)) { 4122 adev->aca.is_enabled = 4123 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4124 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4125 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4126 } 4127 4128 /* bad page feature is not applicable to specific app platform */ 4129 if (adev->gmc.is_app_apu && 4130 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4131 amdgpu_bad_page_threshold = 0; 4132 } 4133 4134 static void amdgpu_ras_counte_dw(struct work_struct *work) 4135 { 4136 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4137 ras_counte_delay_work.work); 4138 struct amdgpu_device *adev = con->adev; 4139 struct drm_device *dev = adev_to_drm(adev); 4140 unsigned long ce_count, ue_count; 4141 int res; 4142 4143 res = pm_runtime_get_sync(dev->dev); 4144 if (res < 0) 4145 goto Out; 4146 4147 /* Cache new values. 4148 */ 4149 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4150 atomic_set(&con->ras_ce_count, ce_count); 4151 atomic_set(&con->ras_ue_count, ue_count); 4152 } 4153 4154 Out: 4155 pm_runtime_put_autosuspend(dev->dev); 4156 } 4157 4158 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4159 { 4160 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4161 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4162 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4163 AMDGPU_RAS_ERROR__PARITY; 4164 } 4165 4166 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4167 { 4168 struct ras_event_state *event_state; 4169 int i; 4170 4171 memset(mgr, 0, sizeof(*mgr)); 4172 atomic64_set(&mgr->seqno, 0); 4173 4174 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4175 event_state = &mgr->event_state[i]; 4176 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4177 atomic64_set(&event_state->count, 0); 4178 } 4179 } 4180 4181 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4182 { 4183 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4184 struct amdgpu_hive_info *hive; 4185 4186 if (!ras) 4187 return; 4188 4189 hive = amdgpu_get_xgmi_hive(adev); 4190 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4191 4192 /* init event manager with node 0 on xgmi system */ 4193 if (!amdgpu_reset_in_recovery(adev)) { 4194 if (!hive || adev->gmc.xgmi.node_id == 0) 4195 ras_event_mgr_init(ras->event_mgr); 4196 } 4197 4198 if (hive) 4199 amdgpu_put_xgmi_hive(hive); 4200 } 4201 4202 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4203 { 4204 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4205 4206 if (!con || (adev->flags & AMD_IS_APU)) 4207 return; 4208 4209 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4210 case IP_VERSION(13, 0, 2): 4211 case IP_VERSION(13, 0, 6): 4212 case IP_VERSION(13, 0, 12): 4213 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4214 break; 4215 case IP_VERSION(13, 0, 14): 4216 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4217 break; 4218 default: 4219 break; 4220 } 4221 } 4222 4223 int amdgpu_ras_init(struct amdgpu_device *adev) 4224 { 4225 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4226 int r; 4227 4228 if (con) 4229 return 0; 4230 4231 con = kzalloc(sizeof(*con) + 4232 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4233 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4234 GFP_KERNEL); 4235 if (!con) 4236 return -ENOMEM; 4237 4238 con->adev = adev; 4239 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4240 atomic_set(&con->ras_ce_count, 0); 4241 atomic_set(&con->ras_ue_count, 0); 4242 4243 con->objs = (struct ras_manager *)(con + 1); 4244 4245 amdgpu_ras_set_context(adev, con); 4246 4247 amdgpu_ras_check_supported(adev); 4248 4249 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4250 /* set gfx block ras context feature for VEGA20 Gaming 4251 * send ras disable cmd to ras ta during ras late init. 4252 */ 4253 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4254 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4255 4256 return 0; 4257 } 4258 4259 r = 0; 4260 goto release_con; 4261 } 4262 4263 con->update_channel_flag = false; 4264 con->features = 0; 4265 con->schema = 0; 4266 INIT_LIST_HEAD(&con->head); 4267 /* Might need get this flag from vbios. */ 4268 con->flags = RAS_DEFAULT_FLAGS; 4269 4270 /* initialize nbio ras function ahead of any other 4271 * ras functions so hardware fatal error interrupt 4272 * can be enabled as early as possible */ 4273 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4274 case IP_VERSION(7, 4, 0): 4275 case IP_VERSION(7, 4, 1): 4276 case IP_VERSION(7, 4, 4): 4277 if (!adev->gmc.xgmi.connected_to_cpu) 4278 adev->nbio.ras = &nbio_v7_4_ras; 4279 break; 4280 case IP_VERSION(4, 3, 0): 4281 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4282 /* unlike other generation of nbio ras, 4283 * nbio v4_3 only support fatal error interrupt 4284 * to inform software that DF is freezed due to 4285 * system fatal error event. driver should not 4286 * enable nbio ras in such case. Instead, 4287 * check DF RAS */ 4288 adev->nbio.ras = &nbio_v4_3_ras; 4289 break; 4290 case IP_VERSION(6, 3, 1): 4291 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4292 /* unlike other generation of nbio ras, 4293 * nbif v6_3_1 only support fatal error interrupt 4294 * to inform software that DF is freezed due to 4295 * system fatal error event. driver should not 4296 * enable nbio ras in such case. Instead, 4297 * check DF RAS 4298 */ 4299 adev->nbio.ras = &nbif_v6_3_1_ras; 4300 break; 4301 case IP_VERSION(7, 9, 0): 4302 case IP_VERSION(7, 9, 1): 4303 if (!adev->gmc.is_app_apu) 4304 adev->nbio.ras = &nbio_v7_9_ras; 4305 break; 4306 default: 4307 /* nbio ras is not available */ 4308 break; 4309 } 4310 4311 /* nbio ras block needs to be enabled ahead of other ras blocks 4312 * to handle fatal error */ 4313 r = amdgpu_nbio_ras_sw_init(adev); 4314 if (r) 4315 return r; 4316 4317 if (adev->nbio.ras && 4318 adev->nbio.ras->init_ras_controller_interrupt) { 4319 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4320 if (r) 4321 goto release_con; 4322 } 4323 4324 if (adev->nbio.ras && 4325 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4326 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4327 if (r) 4328 goto release_con; 4329 } 4330 4331 /* Packed socket_id to ras feature mask bits[31:29] */ 4332 if (adev->smuio.funcs && 4333 adev->smuio.funcs->get_socket_id) 4334 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4335 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4336 4337 /* Get RAS schema for particular SOC */ 4338 con->schema = amdgpu_get_ras_schema(adev); 4339 4340 amdgpu_ras_init_reserved_vram_size(adev); 4341 4342 if (amdgpu_ras_fs_init(adev)) { 4343 r = -EINVAL; 4344 goto release_con; 4345 } 4346 4347 if (amdgpu_ras_aca_is_supported(adev)) { 4348 if (amdgpu_aca_is_enabled(adev)) 4349 r = amdgpu_aca_init(adev); 4350 else 4351 r = amdgpu_mca_init(adev); 4352 if (r) 4353 goto release_con; 4354 } 4355 4356 con->init_task_pid = task_pid_nr(current); 4357 get_task_comm(con->init_task_comm, current); 4358 4359 mutex_init(&con->critical_region_lock); 4360 INIT_LIST_HEAD(&con->critical_region_head); 4361 4362 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4363 "hardware ability[%x] ras_mask[%x]\n", 4364 adev->ras_hw_enabled, adev->ras_enabled); 4365 4366 return 0; 4367 release_con: 4368 amdgpu_ras_set_context(adev, NULL); 4369 kfree(con); 4370 4371 return r; 4372 } 4373 4374 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4375 { 4376 if (adev->gmc.xgmi.connected_to_cpu || 4377 adev->gmc.is_app_apu) 4378 return 1; 4379 return 0; 4380 } 4381 4382 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4383 struct ras_common_if *ras_block) 4384 { 4385 struct ras_query_if info = { 4386 .head = *ras_block, 4387 }; 4388 4389 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4390 return 0; 4391 4392 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4393 DRM_WARN("RAS init harvest failure"); 4394 4395 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4396 DRM_WARN("RAS init harvest reset failure"); 4397 4398 return 0; 4399 } 4400 4401 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4402 { 4403 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4404 4405 if (!con) 4406 return false; 4407 4408 return con->poison_supported; 4409 } 4410 4411 /* helper function to handle common stuff in ip late init phase */ 4412 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4413 struct ras_common_if *ras_block) 4414 { 4415 struct amdgpu_ras_block_object *ras_obj = NULL; 4416 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4417 struct ras_query_if *query_info; 4418 unsigned long ue_count, ce_count; 4419 int r; 4420 4421 /* disable RAS feature per IP block if it is not supported */ 4422 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4423 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4424 return 0; 4425 } 4426 4427 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4428 if (r) { 4429 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4430 /* in resume phase, if fail to enable ras, 4431 * clean up all ras fs nodes, and disable ras */ 4432 goto cleanup; 4433 } else 4434 return r; 4435 } 4436 4437 /* check for errors on warm reset edc persisant supported ASIC */ 4438 amdgpu_persistent_edc_harvesting(adev, ras_block); 4439 4440 /* in resume phase, no need to create ras fs node */ 4441 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4442 return 0; 4443 4444 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4445 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4446 (ras_obj->hw_ops->query_poison_status || 4447 ras_obj->hw_ops->handle_poison_consumption))) { 4448 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4449 if (r) 4450 goto cleanup; 4451 } 4452 4453 if (ras_obj->hw_ops && 4454 (ras_obj->hw_ops->query_ras_error_count || 4455 ras_obj->hw_ops->query_ras_error_status)) { 4456 r = amdgpu_ras_sysfs_create(adev, ras_block); 4457 if (r) 4458 goto interrupt; 4459 4460 /* Those are the cached values at init. 4461 */ 4462 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4463 if (!query_info) 4464 return -ENOMEM; 4465 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4466 4467 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4468 atomic_set(&con->ras_ce_count, ce_count); 4469 atomic_set(&con->ras_ue_count, ue_count); 4470 } 4471 4472 kfree(query_info); 4473 } 4474 4475 return 0; 4476 4477 interrupt: 4478 if (ras_obj->ras_cb) 4479 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4480 cleanup: 4481 amdgpu_ras_feature_enable(adev, ras_block, 0); 4482 return r; 4483 } 4484 4485 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4486 struct ras_common_if *ras_block) 4487 { 4488 return amdgpu_ras_block_late_init(adev, ras_block); 4489 } 4490 4491 /* helper function to remove ras fs node and interrupt handler */ 4492 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4493 struct ras_common_if *ras_block) 4494 { 4495 struct amdgpu_ras_block_object *ras_obj; 4496 if (!ras_block) 4497 return; 4498 4499 amdgpu_ras_sysfs_remove(adev, ras_block); 4500 4501 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4502 if (ras_obj->ras_cb) 4503 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4504 } 4505 4506 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4507 struct ras_common_if *ras_block) 4508 { 4509 return amdgpu_ras_block_late_fini(adev, ras_block); 4510 } 4511 4512 /* do some init work after IP late init as dependence. 4513 * and it runs in resume/gpu reset/booting up cases. 4514 */ 4515 void amdgpu_ras_resume(struct amdgpu_device *adev) 4516 { 4517 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4518 struct ras_manager *obj, *tmp; 4519 4520 if (!adev->ras_enabled || !con) { 4521 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4522 amdgpu_release_ras_context(adev); 4523 4524 return; 4525 } 4526 4527 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4528 /* Set up all other IPs which are not implemented. There is a 4529 * tricky thing that IP's actual ras error type should be 4530 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4531 * ERROR_NONE make sense anyway. 4532 */ 4533 amdgpu_ras_enable_all_features(adev, 1); 4534 4535 /* We enable ras on all hw_supported block, but as boot 4536 * parameter might disable some of them and one or more IP has 4537 * not implemented yet. So we disable them on behalf. 4538 */ 4539 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4540 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4541 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4542 /* there should be no any reference. */ 4543 WARN_ON(alive_obj(obj)); 4544 } 4545 } 4546 } 4547 } 4548 4549 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4550 { 4551 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4552 4553 if (!adev->ras_enabled || !con) 4554 return; 4555 4556 amdgpu_ras_disable_all_features(adev, 0); 4557 /* Make sure all ras objects are disabled. */ 4558 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4559 amdgpu_ras_disable_all_features(adev, 1); 4560 } 4561 4562 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4563 { 4564 struct amdgpu_ras_block_list *node, *tmp; 4565 struct amdgpu_ras_block_object *obj; 4566 int r; 4567 4568 amdgpu_ras_event_mgr_init(adev); 4569 4570 if (amdgpu_ras_aca_is_supported(adev)) { 4571 if (amdgpu_reset_in_recovery(adev)) { 4572 if (amdgpu_aca_is_enabled(adev)) 4573 r = amdgpu_aca_reset(adev); 4574 else 4575 r = amdgpu_mca_reset(adev); 4576 if (r) 4577 return r; 4578 } 4579 4580 if (!amdgpu_sriov_vf(adev)) { 4581 if (amdgpu_aca_is_enabled(adev)) 4582 amdgpu_ras_set_aca_debug_mode(adev, false); 4583 else 4584 amdgpu_ras_set_mca_debug_mode(adev, false); 4585 } 4586 } 4587 4588 /* Guest side doesn't need init ras feature */ 4589 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4590 return 0; 4591 4592 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4593 obj = node->ras_obj; 4594 if (!obj) { 4595 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4596 continue; 4597 } 4598 4599 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4600 continue; 4601 4602 if (obj->ras_late_init) { 4603 r = obj->ras_late_init(adev, &obj->ras_comm); 4604 if (r) { 4605 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4606 obj->ras_comm.name, r); 4607 return r; 4608 } 4609 } else 4610 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4611 } 4612 4613 return 0; 4614 } 4615 4616 /* do some fini work before IP fini as dependence */ 4617 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4618 { 4619 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4620 4621 if (!adev->ras_enabled || !con) 4622 return 0; 4623 4624 4625 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4626 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4627 amdgpu_ras_disable_all_features(adev, 0); 4628 amdgpu_ras_recovery_fini(adev); 4629 return 0; 4630 } 4631 4632 int amdgpu_ras_fini(struct amdgpu_device *adev) 4633 { 4634 struct amdgpu_ras_block_list *ras_node, *tmp; 4635 struct amdgpu_ras_block_object *obj = NULL; 4636 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4637 4638 if (!adev->ras_enabled || !con) 4639 return 0; 4640 4641 amdgpu_ras_critical_region_fini(adev); 4642 mutex_destroy(&con->critical_region_lock); 4643 4644 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4645 if (ras_node->ras_obj) { 4646 obj = ras_node->ras_obj; 4647 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4648 obj->ras_fini) 4649 obj->ras_fini(adev, &obj->ras_comm); 4650 else 4651 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4652 } 4653 4654 /* Clear ras blocks from ras_list and free ras block list node */ 4655 list_del(&ras_node->node); 4656 kfree(ras_node); 4657 } 4658 4659 amdgpu_ras_fs_fini(adev); 4660 amdgpu_ras_interrupt_remove_all(adev); 4661 4662 if (amdgpu_ras_aca_is_supported(adev)) { 4663 if (amdgpu_aca_is_enabled(adev)) 4664 amdgpu_aca_fini(adev); 4665 else 4666 amdgpu_mca_fini(adev); 4667 } 4668 4669 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4670 4671 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4672 amdgpu_ras_disable_all_features(adev, 0); 4673 4674 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4675 4676 amdgpu_ras_set_context(adev, NULL); 4677 kfree(con); 4678 4679 return 0; 4680 } 4681 4682 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4683 { 4684 struct amdgpu_ras *ras; 4685 4686 ras = amdgpu_ras_get_context(adev); 4687 if (!ras) 4688 return false; 4689 4690 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4691 } 4692 4693 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4694 { 4695 struct amdgpu_ras *ras; 4696 4697 ras = amdgpu_ras_get_context(adev); 4698 if (ras) { 4699 if (status) 4700 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4701 else 4702 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4703 } 4704 } 4705 4706 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4707 { 4708 struct amdgpu_ras *ras; 4709 4710 ras = amdgpu_ras_get_context(adev); 4711 if (ras) { 4712 ras->ras_err_state = 0; 4713 ras->gpu_reset_flags = 0; 4714 } 4715 } 4716 4717 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4718 enum amdgpu_ras_block block) 4719 { 4720 struct amdgpu_ras *ras; 4721 4722 ras = amdgpu_ras_get_context(adev); 4723 if (ras) 4724 set_bit(block, &ras->ras_err_state); 4725 } 4726 4727 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4728 { 4729 struct amdgpu_ras *ras; 4730 4731 ras = amdgpu_ras_get_context(adev); 4732 if (ras) { 4733 if (block == AMDGPU_RAS_BLOCK__ANY) 4734 return (ras->ras_err_state != 0); 4735 else 4736 return test_bit(block, &ras->ras_err_state) || 4737 test_bit(AMDGPU_RAS_BLOCK__LAST, 4738 &ras->ras_err_state); 4739 } 4740 4741 return false; 4742 } 4743 4744 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4745 { 4746 struct amdgpu_ras *ras; 4747 4748 ras = amdgpu_ras_get_context(adev); 4749 if (!ras) 4750 return NULL; 4751 4752 return ras->event_mgr; 4753 } 4754 4755 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4756 const void *caller) 4757 { 4758 struct ras_event_manager *event_mgr; 4759 struct ras_event_state *event_state; 4760 int ret = 0; 4761 4762 if (amdgpu_uniras_enabled(adev)) 4763 return 0; 4764 4765 if (type >= RAS_EVENT_TYPE_COUNT) { 4766 ret = -EINVAL; 4767 goto out; 4768 } 4769 4770 event_mgr = __get_ras_event_mgr(adev); 4771 if (!event_mgr) { 4772 ret = -EINVAL; 4773 goto out; 4774 } 4775 4776 event_state = &event_mgr->event_state[type]; 4777 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4778 atomic64_inc(&event_state->count); 4779 4780 out: 4781 if (ret && caller) 4782 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4783 (int)type, caller, ret); 4784 4785 return ret; 4786 } 4787 4788 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4789 { 4790 struct ras_event_manager *event_mgr; 4791 u64 id; 4792 4793 if (type >= RAS_EVENT_TYPE_COUNT) 4794 return RAS_EVENT_INVALID_ID; 4795 4796 switch (type) { 4797 case RAS_EVENT_TYPE_FATAL: 4798 case RAS_EVENT_TYPE_POISON_CREATION: 4799 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4800 event_mgr = __get_ras_event_mgr(adev); 4801 if (!event_mgr) 4802 return RAS_EVENT_INVALID_ID; 4803 4804 id = event_mgr->event_state[type].last_seqno; 4805 break; 4806 case RAS_EVENT_TYPE_INVALID: 4807 default: 4808 id = RAS_EVENT_INVALID_ID; 4809 break; 4810 } 4811 4812 return id; 4813 } 4814 4815 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4816 { 4817 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4818 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4819 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4820 u64 event_id = RAS_EVENT_INVALID_ID; 4821 4822 if (amdgpu_uniras_enabled(adev)) 4823 return 0; 4824 4825 if (!amdgpu_ras_mark_ras_event(adev, type)) 4826 event_id = amdgpu_ras_acquire_event_id(adev, type); 4827 4828 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4829 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4830 4831 amdgpu_ras_set_fed(adev, true); 4832 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4833 amdgpu_ras_reset_gpu(adev); 4834 } 4835 4836 return -EBUSY; 4837 } 4838 4839 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4840 { 4841 if (adev->asic_type == CHIP_VEGA20 && 4842 adev->pm.fw_version <= 0x283400) { 4843 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4844 amdgpu_ras_intr_triggered(); 4845 } 4846 4847 return false; 4848 } 4849 4850 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4851 { 4852 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4853 4854 if (!con) 4855 return; 4856 4857 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4858 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4859 amdgpu_ras_set_context(adev, NULL); 4860 kfree(con); 4861 } 4862 } 4863 4864 #ifdef CONFIG_X86_MCE_AMD 4865 static struct amdgpu_device *find_adev(uint32_t node_id) 4866 { 4867 int i; 4868 struct amdgpu_device *adev = NULL; 4869 4870 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4871 adev = mce_adev_list.devs[i]; 4872 4873 if (adev && adev->gmc.xgmi.connected_to_cpu && 4874 adev->gmc.xgmi.physical_node_id == node_id) 4875 break; 4876 adev = NULL; 4877 } 4878 4879 return adev; 4880 } 4881 4882 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4883 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4884 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4885 #define GPU_ID_OFFSET 8 4886 4887 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4888 unsigned long val, void *data) 4889 { 4890 struct mce *m = (struct mce *)data; 4891 struct amdgpu_device *adev = NULL; 4892 uint32_t gpu_id = 0; 4893 uint32_t umc_inst = 0, ch_inst = 0; 4894 4895 /* 4896 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4897 * and error occurred in DramECC (Extended error code = 0) then only 4898 * process the error, else bail out. 4899 */ 4900 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4901 (XEC(m->status, 0x3f) == 0x0))) 4902 return NOTIFY_DONE; 4903 4904 /* 4905 * If it is correctable error, return. 4906 */ 4907 if (mce_is_correctable(m)) 4908 return NOTIFY_OK; 4909 4910 /* 4911 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4912 */ 4913 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4914 4915 adev = find_adev(gpu_id); 4916 if (!adev) { 4917 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4918 gpu_id); 4919 return NOTIFY_DONE; 4920 } 4921 4922 /* 4923 * If it is uncorrectable error, then find out UMC instance and 4924 * channel index. 4925 */ 4926 umc_inst = GET_UMC_INST(m->ipid); 4927 ch_inst = GET_CHAN_INDEX(m->ipid); 4928 4929 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4930 umc_inst, ch_inst); 4931 4932 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4933 return NOTIFY_OK; 4934 else 4935 return NOTIFY_DONE; 4936 } 4937 4938 static struct notifier_block amdgpu_bad_page_nb = { 4939 .notifier_call = amdgpu_bad_page_notifier, 4940 .priority = MCE_PRIO_UC, 4941 }; 4942 4943 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4944 { 4945 /* 4946 * Add the adev to the mce_adev_list. 4947 * During mode2 reset, amdgpu device is temporarily 4948 * removed from the mgpu_info list which can cause 4949 * page retirement to fail. 4950 * Use this list instead of mgpu_info to find the amdgpu 4951 * device on which the UMC error was reported. 4952 */ 4953 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4954 4955 /* 4956 * Register the x86 notifier only once 4957 * with MCE subsystem. 4958 */ 4959 if (notifier_registered == false) { 4960 mce_register_decode_chain(&amdgpu_bad_page_nb); 4961 notifier_registered = true; 4962 } 4963 } 4964 #endif 4965 4966 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4967 { 4968 if (!adev) 4969 return NULL; 4970 4971 return adev->psp.ras_context.ras; 4972 } 4973 4974 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4975 { 4976 if (!adev) 4977 return -EINVAL; 4978 4979 adev->psp.ras_context.ras = ras_con; 4980 return 0; 4981 } 4982 4983 /* check if ras is supported on block, say, sdma, gfx */ 4984 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4985 unsigned int block) 4986 { 4987 int ret = 0; 4988 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4989 4990 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4991 return 0; 4992 4993 ret = ras && (adev->ras_enabled & (1 << block)); 4994 4995 /* For the special asic with mem ecc enabled but sram ecc 4996 * not enabled, even if the ras block is not supported on 4997 * .ras_enabled, if the asic supports poison mode and the 4998 * ras block has ras configuration, it can be considered 4999 * that the ras block supports ras function. 5000 */ 5001 if (!ret && 5002 (block == AMDGPU_RAS_BLOCK__GFX || 5003 block == AMDGPU_RAS_BLOCK__SDMA || 5004 block == AMDGPU_RAS_BLOCK__VCN || 5005 block == AMDGPU_RAS_BLOCK__JPEG) && 5006 (amdgpu_ras_mask & (1 << block)) && 5007 amdgpu_ras_is_poison_mode_supported(adev) && 5008 amdgpu_ras_get_ras_block(adev, block, 0)) 5009 ret = 1; 5010 5011 return ret; 5012 } 5013 5014 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5015 { 5016 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5017 5018 /* mode1 is the only selection for RMA status */ 5019 if (amdgpu_ras_is_rma(adev)) { 5020 ras->gpu_reset_flags = 0; 5021 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5022 } 5023 5024 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5025 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5026 int hive_ras_recovery = 0; 5027 5028 if (hive) { 5029 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5030 amdgpu_put_xgmi_hive(hive); 5031 } 5032 /* In the case of multiple GPUs, after a GPU has started 5033 * resetting all GPUs on hive, other GPUs do not need to 5034 * trigger GPU reset again. 5035 */ 5036 if (!hive_ras_recovery) 5037 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5038 else 5039 atomic_set(&ras->in_recovery, 0); 5040 } else { 5041 flush_work(&ras->recovery_work); 5042 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5043 } 5044 5045 return 0; 5046 } 5047 5048 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5049 { 5050 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5051 int ret = 0; 5052 5053 if (con) { 5054 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5055 if (!ret) 5056 con->is_aca_debug_mode = enable; 5057 } 5058 5059 return ret; 5060 } 5061 5062 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5063 { 5064 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5065 int ret = 0; 5066 5067 if (con) { 5068 if (amdgpu_aca_is_enabled(adev)) 5069 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5070 else 5071 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5072 if (!ret) 5073 con->is_aca_debug_mode = enable; 5074 } 5075 5076 return ret; 5077 } 5078 5079 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5080 { 5081 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5082 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5083 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5084 5085 if (!con) 5086 return false; 5087 5088 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5089 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5090 return con->is_aca_debug_mode; 5091 else 5092 return true; 5093 } 5094 5095 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5096 unsigned int *error_query_mode) 5097 { 5098 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5099 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5100 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5101 5102 if (!con) { 5103 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5104 return false; 5105 } 5106 5107 if (amdgpu_sriov_vf(adev)) { 5108 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5109 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5110 *error_query_mode = 5111 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5112 } else { 5113 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5114 } 5115 5116 return true; 5117 } 5118 5119 /* Register each ip ras block into amdgpu ras */ 5120 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5121 struct amdgpu_ras_block_object *ras_block_obj) 5122 { 5123 struct amdgpu_ras_block_list *ras_node; 5124 if (!adev || !ras_block_obj) 5125 return -EINVAL; 5126 5127 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5128 if (!ras_node) 5129 return -ENOMEM; 5130 5131 INIT_LIST_HEAD(&ras_node->node); 5132 ras_node->ras_obj = ras_block_obj; 5133 list_add_tail(&ras_node->node, &adev->ras_list); 5134 5135 return 0; 5136 } 5137 5138 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5139 { 5140 if (!err_type_name) 5141 return; 5142 5143 switch (err_type) { 5144 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5145 sprintf(err_type_name, "correctable"); 5146 break; 5147 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5148 sprintf(err_type_name, "uncorrectable"); 5149 break; 5150 default: 5151 sprintf(err_type_name, "unknown"); 5152 break; 5153 } 5154 } 5155 5156 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5157 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5158 uint32_t instance, 5159 uint32_t *memory_id) 5160 { 5161 uint32_t err_status_lo_data, err_status_lo_offset; 5162 5163 if (!reg_entry) 5164 return false; 5165 5166 err_status_lo_offset = 5167 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5168 reg_entry->seg_lo, reg_entry->reg_lo); 5169 err_status_lo_data = RREG32(err_status_lo_offset); 5170 5171 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5172 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5173 return false; 5174 5175 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5176 5177 return true; 5178 } 5179 5180 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5181 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5182 uint32_t instance, 5183 unsigned long *err_cnt) 5184 { 5185 uint32_t err_status_hi_data, err_status_hi_offset; 5186 5187 if (!reg_entry) 5188 return false; 5189 5190 err_status_hi_offset = 5191 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5192 reg_entry->seg_hi, reg_entry->reg_hi); 5193 err_status_hi_data = RREG32(err_status_hi_offset); 5194 5195 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5196 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5197 /* keep the check here in case we need to refer to the result later */ 5198 dev_dbg(adev->dev, "Invalid err_info field\n"); 5199 5200 /* read err count */ 5201 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5202 5203 return true; 5204 } 5205 5206 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5207 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5208 uint32_t reg_list_size, 5209 const struct amdgpu_ras_memory_id_entry *mem_list, 5210 uint32_t mem_list_size, 5211 uint32_t instance, 5212 uint32_t err_type, 5213 unsigned long *err_count) 5214 { 5215 uint32_t memory_id; 5216 unsigned long err_cnt; 5217 char err_type_name[16]; 5218 uint32_t i, j; 5219 5220 for (i = 0; i < reg_list_size; i++) { 5221 /* query memory_id from err_status_lo */ 5222 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5223 instance, &memory_id)) 5224 continue; 5225 5226 /* query err_cnt from err_status_hi */ 5227 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5228 instance, &err_cnt) || 5229 !err_cnt) 5230 continue; 5231 5232 *err_count += err_cnt; 5233 5234 /* log the errors */ 5235 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5236 if (!mem_list) { 5237 /* memory_list is not supported */ 5238 dev_info(adev->dev, 5239 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5240 err_cnt, err_type_name, 5241 reg_list[i].block_name, 5242 instance, memory_id); 5243 } else { 5244 for (j = 0; j < mem_list_size; j++) { 5245 if (memory_id == mem_list[j].memory_id) { 5246 dev_info(adev->dev, 5247 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5248 err_cnt, err_type_name, 5249 reg_list[i].block_name, 5250 instance, mem_list[j].name); 5251 break; 5252 } 5253 } 5254 } 5255 } 5256 } 5257 5258 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5259 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5260 uint32_t reg_list_size, 5261 uint32_t instance) 5262 { 5263 uint32_t err_status_lo_offset, err_status_hi_offset; 5264 uint32_t i; 5265 5266 for (i = 0; i < reg_list_size; i++) { 5267 err_status_lo_offset = 5268 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5269 reg_list[i].seg_lo, reg_list[i].reg_lo); 5270 err_status_hi_offset = 5271 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5272 reg_list[i].seg_hi, reg_list[i].reg_hi); 5273 WREG32(err_status_lo_offset, 0); 5274 WREG32(err_status_hi_offset, 0); 5275 } 5276 } 5277 5278 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5279 { 5280 memset(err_data, 0, sizeof(*err_data)); 5281 5282 INIT_LIST_HEAD(&err_data->err_node_list); 5283 5284 return 0; 5285 } 5286 5287 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5288 { 5289 if (!err_node) 5290 return; 5291 5292 list_del(&err_node->node); 5293 kvfree(err_node); 5294 } 5295 5296 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5297 { 5298 struct ras_err_node *err_node, *tmp; 5299 5300 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5301 amdgpu_ras_error_node_release(err_node); 5302 } 5303 5304 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5305 struct amdgpu_smuio_mcm_config_info *mcm_info) 5306 { 5307 struct ras_err_node *err_node; 5308 struct amdgpu_smuio_mcm_config_info *ref_id; 5309 5310 if (!err_data || !mcm_info) 5311 return NULL; 5312 5313 for_each_ras_error(err_node, err_data) { 5314 ref_id = &err_node->err_info.mcm_info; 5315 5316 if (mcm_info->socket_id == ref_id->socket_id && 5317 mcm_info->die_id == ref_id->die_id) 5318 return err_node; 5319 } 5320 5321 return NULL; 5322 } 5323 5324 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5325 { 5326 struct ras_err_node *err_node; 5327 5328 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5329 if (!err_node) 5330 return NULL; 5331 5332 INIT_LIST_HEAD(&err_node->node); 5333 5334 return err_node; 5335 } 5336 5337 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5338 { 5339 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5340 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5341 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5342 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5343 5344 if (unlikely(infoa->socket_id != infob->socket_id)) 5345 return infoa->socket_id - infob->socket_id; 5346 else 5347 return infoa->die_id - infob->die_id; 5348 5349 return 0; 5350 } 5351 5352 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5353 struct amdgpu_smuio_mcm_config_info *mcm_info) 5354 { 5355 struct ras_err_node *err_node; 5356 5357 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5358 if (err_node) 5359 return &err_node->err_info; 5360 5361 err_node = amdgpu_ras_error_node_new(); 5362 if (!err_node) 5363 return NULL; 5364 5365 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5366 5367 err_data->err_list_count++; 5368 list_add_tail(&err_node->node, &err_data->err_node_list); 5369 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5370 5371 return &err_node->err_info; 5372 } 5373 5374 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5375 struct amdgpu_smuio_mcm_config_info *mcm_info, 5376 u64 count) 5377 { 5378 struct ras_err_info *err_info; 5379 5380 if (!err_data || !mcm_info) 5381 return -EINVAL; 5382 5383 if (!count) 5384 return 0; 5385 5386 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5387 if (!err_info) 5388 return -EINVAL; 5389 5390 err_info->ue_count += count; 5391 err_data->ue_count += count; 5392 5393 return 0; 5394 } 5395 5396 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5397 struct amdgpu_smuio_mcm_config_info *mcm_info, 5398 u64 count) 5399 { 5400 struct ras_err_info *err_info; 5401 5402 if (!err_data || !mcm_info) 5403 return -EINVAL; 5404 5405 if (!count) 5406 return 0; 5407 5408 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5409 if (!err_info) 5410 return -EINVAL; 5411 5412 err_info->ce_count += count; 5413 err_data->ce_count += count; 5414 5415 return 0; 5416 } 5417 5418 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5419 struct amdgpu_smuio_mcm_config_info *mcm_info, 5420 u64 count) 5421 { 5422 struct ras_err_info *err_info; 5423 5424 if (!err_data || !mcm_info) 5425 return -EINVAL; 5426 5427 if (!count) 5428 return 0; 5429 5430 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5431 if (!err_info) 5432 return -EINVAL; 5433 5434 err_info->de_count += count; 5435 err_data->de_count += count; 5436 5437 return 0; 5438 } 5439 5440 #define mmMP0_SMN_C2PMSG_92 0x1609C 5441 #define mmMP0_SMN_C2PMSG_126 0x160BE 5442 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5443 u32 instance) 5444 { 5445 u32 socket_id, aid_id, hbm_id; 5446 u32 fw_status; 5447 u32 boot_error; 5448 u64 reg_addr; 5449 5450 /* The pattern for smn addressing in other SOC could be different from 5451 * the one for aqua_vanjaram. We should revisit the code if the pattern 5452 * is changed. In such case, replace the aqua_vanjaram implementation 5453 * with more common helper */ 5454 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5455 aqua_vanjaram_encode_ext_smn_addressing(instance); 5456 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5457 5458 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5459 aqua_vanjaram_encode_ext_smn_addressing(instance); 5460 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5461 5462 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5463 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5464 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5465 5466 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5467 dev_info(adev->dev, 5468 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5469 socket_id, aid_id, hbm_id, fw_status); 5470 5471 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5472 dev_info(adev->dev, 5473 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5474 socket_id, aid_id, fw_status); 5475 5476 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5477 dev_info(adev->dev, 5478 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5479 socket_id, aid_id, fw_status); 5480 5481 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5482 dev_info(adev->dev, 5483 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5484 socket_id, aid_id, fw_status); 5485 5486 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5487 dev_info(adev->dev, 5488 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5489 socket_id, aid_id, fw_status); 5490 5491 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5492 dev_info(adev->dev, 5493 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5494 socket_id, aid_id, fw_status); 5495 5496 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5497 dev_info(adev->dev, 5498 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5499 socket_id, aid_id, hbm_id, fw_status); 5500 5501 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5502 dev_info(adev->dev, 5503 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5504 socket_id, aid_id, hbm_id, fw_status); 5505 5506 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5507 dev_info(adev->dev, 5508 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5509 socket_id, aid_id, fw_status); 5510 5511 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5512 dev_info(adev->dev, 5513 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5514 socket_id, aid_id, fw_status); 5515 } 5516 5517 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5518 u32 instance) 5519 { 5520 u64 reg_addr; 5521 u32 reg_data; 5522 int retry_loop; 5523 5524 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5525 aqua_vanjaram_encode_ext_smn_addressing(instance); 5526 5527 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5528 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5529 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5530 return false; 5531 else 5532 msleep(1); 5533 } 5534 5535 return true; 5536 } 5537 5538 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5539 { 5540 u32 i; 5541 5542 for (i = 0; i < num_instances; i++) { 5543 if (amdgpu_ras_boot_error_detected(adev, i)) 5544 amdgpu_ras_boot_time_error_reporting(adev, i); 5545 } 5546 } 5547 5548 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5549 { 5550 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5551 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5552 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5553 int ret = 0; 5554 5555 if (amdgpu_ras_check_critical_address(adev, start)) 5556 return 0; 5557 5558 mutex_lock(&con->page_rsv_lock); 5559 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5560 if (ret == -ENOENT) 5561 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5562 mutex_unlock(&con->page_rsv_lock); 5563 5564 return ret; 5565 } 5566 5567 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5568 const char *fmt, ...) 5569 { 5570 struct va_format vaf; 5571 va_list args; 5572 5573 va_start(args, fmt); 5574 vaf.fmt = fmt; 5575 vaf.va = &args; 5576 5577 if (RAS_EVENT_ID_IS_VALID(event_id)) 5578 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5579 else 5580 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5581 5582 va_end(args); 5583 } 5584 5585 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5586 { 5587 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5588 5589 if (amdgpu_uniras_enabled(adev)) 5590 return amdgpu_ras_mgr_is_rma(adev); 5591 5592 if (!con) 5593 return false; 5594 5595 return con->is_rma; 5596 } 5597 5598 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5599 struct amdgpu_bo *bo) 5600 { 5601 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5602 struct amdgpu_vram_mgr_resource *vres; 5603 struct ras_critical_region *region; 5604 struct drm_buddy_block *block; 5605 int ret = 0; 5606 5607 if (!bo || !bo->tbo.resource) 5608 return -EINVAL; 5609 5610 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5611 5612 mutex_lock(&con->critical_region_lock); 5613 5614 /* Check if the bo had been recorded */ 5615 list_for_each_entry(region, &con->critical_region_head, node) 5616 if (region->bo == bo) 5617 goto out; 5618 5619 /* Record new critical amdgpu bo */ 5620 list_for_each_entry(block, &vres->blocks, link) { 5621 region = kzalloc(sizeof(*region), GFP_KERNEL); 5622 if (!region) { 5623 ret = -ENOMEM; 5624 goto out; 5625 } 5626 region->bo = bo; 5627 region->start = amdgpu_vram_mgr_block_start(block); 5628 region->size = amdgpu_vram_mgr_block_size(block); 5629 list_add_tail(®ion->node, &con->critical_region_head); 5630 } 5631 5632 out: 5633 mutex_unlock(&con->critical_region_lock); 5634 5635 return ret; 5636 } 5637 5638 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5639 { 5640 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5641 } 5642 5643 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5644 { 5645 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5646 struct ras_critical_region *region, *tmp; 5647 5648 mutex_lock(&con->critical_region_lock); 5649 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5650 list_del(®ion->node); 5651 kfree(region); 5652 } 5653 mutex_unlock(&con->critical_region_lock); 5654 } 5655 5656 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5657 { 5658 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5659 struct ras_critical_region *region; 5660 bool ret = false; 5661 5662 mutex_lock(&con->critical_region_lock); 5663 list_for_each_entry(region, &con->critical_region_head, node) { 5664 if ((region->start <= addr) && 5665 (addr < (region->start + region->size))) { 5666 ret = true; 5667 break; 5668 } 5669 } 5670 mutex_unlock(&con->critical_region_lock); 5671 5672 return ret; 5673 } 5674