1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 #include "amdgpu_ras_mgr.h" 45 46 #ifdef CONFIG_X86_MCE_AMD 47 #include <asm/mce.h> 48 49 static bool notifier_registered; 50 #endif 51 static const char *RAS_FS_NAME = "ras"; 52 53 const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59 }; 60 61 const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82 }; 83 84 const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89 }; 90 91 struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96 }; 97 98 const char *get_ras_block_str(struct ras_common_if *ras_block) 99 { 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111 } 112 113 #define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116 #define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120 /* inject address is 52 bits */ 121 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132 #define BYPASS_ALLOCATED_ADDRESS 0x0 133 #define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135 enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139 }; 140 141 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151 #ifdef CONFIG_X86_MCE_AMD 152 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153 struct mce_notifier_adev_list { 154 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 155 int num_gpu; 156 }; 157 static struct mce_notifier_adev_list mce_adev_list; 158 #endif 159 160 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 161 { 162 if (adev && amdgpu_ras_get_context(adev)) 163 amdgpu_ras_get_context(adev)->error_query_ready = ready; 164 } 165 166 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 167 { 168 if (adev && amdgpu_ras_get_context(adev)) 169 return amdgpu_ras_get_context(adev)->error_query_ready; 170 171 return false; 172 } 173 174 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 175 { 176 struct ras_err_data err_data; 177 struct eeprom_table_record err_rec; 178 int ret; 179 180 ret = amdgpu_ras_check_bad_page(adev, address); 181 if (ret == -EINVAL) { 182 dev_warn(adev->dev, 183 "RAS WARN: input address 0x%llx is invalid.\n", 184 address); 185 return -EINVAL; 186 } else if (ret == 1) { 187 dev_warn(adev->dev, 188 "RAS WARN: 0x%llx has already been marked as bad page!\n", 189 address); 190 return 0; 191 } 192 193 ret = amdgpu_ras_error_data_init(&err_data); 194 if (ret) 195 return ret; 196 197 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 198 err_data.err_addr = &err_rec; 199 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 200 201 if (amdgpu_bad_page_threshold != 0) { 202 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 203 err_data.err_addr_cnt, false); 204 amdgpu_ras_save_bad_pages(adev, NULL); 205 } 206 207 amdgpu_ras_error_data_fini(&err_data); 208 209 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 210 dev_warn(adev->dev, "Clear EEPROM:\n"); 211 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 212 213 return 0; 214 } 215 216 static int amdgpu_check_address_validity(struct amdgpu_device *adev, 217 uint64_t address, uint64_t flags) 218 { 219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 220 struct amdgpu_vram_block_info blk_info; 221 uint64_t page_pfns[32] = {0}; 222 int i, ret, count; 223 bool hit = false; 224 225 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 226 return 0; 227 228 if (amdgpu_sriov_vf(adev)) { 229 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 230 return -EPERM; 231 return hit ? -EACCES : 0; 232 } 233 234 if ((address >= adev->gmc.mc_vram_size) || 235 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 236 return -EFAULT; 237 238 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 239 address, page_pfns, ARRAY_SIZE(page_pfns)); 240 if (count <= 0) 241 return -EPERM; 242 243 for (i = 0; i < count; i++) { 244 memset(&blk_info, 0, sizeof(blk_info)); 245 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 246 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 247 if (!ret) { 248 /* The input address that needs to be checked is allocated by 249 * current calling process, so it is necessary to exclude 250 * the calling process. 251 */ 252 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 253 ((blk_info.task.pid != task_pid_nr(current)) || 254 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 255 return -EACCES; 256 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 257 (blk_info.task.pid == con->init_task_pid) && 258 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 259 return -EACCES; 260 } 261 } 262 263 return 0; 264 } 265 266 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 267 size_t size, loff_t *pos) 268 { 269 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 270 struct ras_query_if info = { 271 .head = obj->head, 272 }; 273 ssize_t s; 274 char val[128]; 275 276 if (amdgpu_ras_query_error_status(obj->adev, &info)) 277 return -EINVAL; 278 279 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 280 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 281 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 282 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 283 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 284 } 285 286 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 287 "ue", info.ue_count, 288 "ce", info.ce_count); 289 if (*pos >= s) 290 return 0; 291 292 s -= *pos; 293 s = min_t(u64, s, size); 294 295 296 if (copy_to_user(buf, &val[*pos], s)) 297 return -EINVAL; 298 299 *pos += s; 300 301 return s; 302 } 303 304 static const struct file_operations amdgpu_ras_debugfs_ops = { 305 .owner = THIS_MODULE, 306 .read = amdgpu_ras_debugfs_read, 307 .write = NULL, 308 .llseek = default_llseek 309 }; 310 311 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 316 *block_id = i; 317 if (strcmp(name, ras_block_string[i]) == 0) 318 return 0; 319 } 320 return -EINVAL; 321 } 322 323 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 324 const char __user *buf, size_t size, 325 loff_t *pos, struct ras_debug_if *data) 326 { 327 ssize_t s = min_t(u64, 64, size); 328 char str[65]; 329 char block_name[33]; 330 char err[9] = "ue"; 331 int op = -1; 332 int block_id; 333 uint32_t sub_block; 334 u64 address, value; 335 /* default value is 0 if the mask is not set by user */ 336 u32 instance_mask = 0; 337 338 if (*pos) 339 return -EINVAL; 340 *pos = size; 341 342 memset(str, 0, sizeof(str)); 343 memset(data, 0, sizeof(*data)); 344 345 if (copy_from_user(str, buf, s)) 346 return -EINVAL; 347 348 if (sscanf(str, "disable %32s", block_name) == 1) 349 op = 0; 350 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 351 op = 1; 352 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 353 op = 2; 354 else if (strstr(str, "retire_page") != NULL) 355 op = 3; 356 else if (strstr(str, "check_address") != NULL) 357 op = 4; 358 else if (str[0] && str[1] && str[2] && str[3]) 359 /* ascii string, but commands are not matched. */ 360 return -EINVAL; 361 362 if (op != -1) { 363 if (op == 3) { 364 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 365 sscanf(str, "%*s %llu", &address) != 1) 366 return -EINVAL; 367 368 data->op = op; 369 data->inject.address = address; 370 371 return 0; 372 } else if (op == 4) { 373 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 374 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 375 return -EINVAL; 376 377 data->op = op; 378 data->inject.address = address; 379 data->inject.value = value; 380 return 0; 381 } 382 383 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 384 return -EINVAL; 385 386 data->head.block = block_id; 387 /* only ue, ce and poison errors are supported */ 388 if (!memcmp("ue", err, 2)) 389 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 390 else if (!memcmp("ce", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 392 else if (!memcmp("poison", err, 6)) 393 data->head.type = AMDGPU_RAS_ERROR__POISON; 394 else 395 return -EINVAL; 396 397 data->op = op; 398 399 if (op == 2) { 400 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 401 &sub_block, &address, &value, &instance_mask) != 4 && 402 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 405 &sub_block, &address, &value) != 3 && 406 sscanf(str, "%*s %*s %*s %u %llu %llu", 407 &sub_block, &address, &value) != 3) 408 return -EINVAL; 409 data->head.sub_block_index = sub_block; 410 data->inject.address = address; 411 data->inject.value = value; 412 data->inject.instance_mask = instance_mask; 413 } 414 } else { 415 if (size < sizeof(*data)) 416 return -EINVAL; 417 418 if (copy_from_user(data, buf, sizeof(*data))) 419 return -EINVAL; 420 } 421 422 return 0; 423 } 424 425 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 426 struct ras_debug_if *data) 427 { 428 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 429 uint32_t mask, inst_mask = data->inject.instance_mask; 430 431 /* no need to set instance mask if there is only one instance */ 432 if (num_xcc <= 1 && inst_mask) { 433 data->inject.instance_mask = 0; 434 dev_dbg(adev->dev, 435 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 436 inst_mask); 437 438 return; 439 } 440 441 switch (data->head.block) { 442 case AMDGPU_RAS_BLOCK__GFX: 443 mask = GENMASK(num_xcc - 1, 0); 444 break; 445 case AMDGPU_RAS_BLOCK__SDMA: 446 mask = GENMASK(adev->sdma.num_instances - 1, 0); 447 break; 448 case AMDGPU_RAS_BLOCK__VCN: 449 case AMDGPU_RAS_BLOCK__JPEG: 450 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 451 break; 452 default: 453 mask = inst_mask; 454 break; 455 } 456 457 /* remove invalid bits in instance mask */ 458 data->inject.instance_mask &= mask; 459 if (inst_mask != data->inject.instance_mask) 460 dev_dbg(adev->dev, 461 "Adjust RAS inject mask 0x%x to 0x%x\n", 462 inst_mask, data->inject.instance_mask); 463 } 464 465 /** 466 * DOC: AMDGPU RAS debugfs control interface 467 * 468 * The control interface accepts struct ras_debug_if which has two members. 469 * 470 * First member: ras_debug_if::head or ras_debug_if::inject. 471 * 472 * head is used to indicate which IP block will be under control. 473 * 474 * head has four members, they are block, type, sub_block_index, name. 475 * block: which IP will be under control. 476 * type: what kind of error will be enabled/disabled/injected. 477 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 478 * name: the name of IP. 479 * 480 * inject has three more members than head, they are address, value and mask. 481 * As their names indicate, inject operation will write the 482 * value to the address. 483 * 484 * The second member: struct ras_debug_if::op. 485 * It has three kinds of operations. 486 * 487 * - 0: disable RAS on the block. Take ::head as its data. 488 * - 1: enable RAS on the block. Take ::head as its data. 489 * - 2: inject errors on the block. Take ::inject as its data. 490 * 491 * How to use the interface? 492 * 493 * In a program 494 * 495 * Copy the struct ras_debug_if in your code and initialize it. 496 * Write the struct to the control interface. 497 * 498 * From shell 499 * 500 * .. code-block:: bash 501 * 502 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 503 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 504 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * 506 * Where N, is the card which you want to affect. 507 * 508 * "disable" requires only the block. 509 * "enable" requires the block and error type. 510 * "inject" requires the block, error type, address, and value. 511 * 512 * The block is one of: umc, sdma, gfx, etc. 513 * see ras_block_string[] for details 514 * 515 * The error type is one of: ue, ce and poison where, 516 * ue is multi-uncorrectable 517 * ce is single-correctable 518 * poison is poison 519 * 520 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 521 * The address and value are hexadecimal numbers, leading 0x is optional. 522 * The mask means instance mask, is optional, default value is 0x1. 523 * 524 * For instance, 525 * 526 * .. code-block:: bash 527 * 528 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 529 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 530 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * 532 * How to check the result of the operation? 533 * 534 * To check disable/enable, see "ras" features at, 535 * /sys/class/drm/card[0/1/2...]/device/ras/features 536 * 537 * To check inject, see the corresponding error count at, 538 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 539 * 540 * .. note:: 541 * Operations are only allowed on blocks which are supported. 542 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 543 * to see which blocks support RAS on a particular asic. 544 * 545 */ 546 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 547 const char __user *buf, 548 size_t size, loff_t *pos) 549 { 550 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 551 struct ras_debug_if data; 552 int ret = 0; 553 554 if (!amdgpu_ras_get_error_query_ready(adev)) { 555 dev_warn(adev->dev, "RAS WARN: error injection " 556 "currently inaccessible\n"); 557 return size; 558 } 559 560 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 561 if (ret) 562 return ret; 563 564 if (data.op == 3) { 565 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 566 if (!ret) 567 return size; 568 else 569 return ret; 570 } else if (data.op == 4) { 571 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 572 return ret ? ret : size; 573 } 574 575 if (!amdgpu_ras_is_supported(adev, data.head.block)) 576 return -EINVAL; 577 578 switch (data.op) { 579 case 0: 580 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 581 break; 582 case 1: 583 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 584 break; 585 case 2: 586 /* umc ce/ue error injection for a bad page is not allowed */ 587 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 588 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 589 if (ret == -EINVAL) { 590 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 591 data.inject.address); 592 break; 593 } else if (ret == 1) { 594 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 595 data.inject.address); 596 break; 597 } 598 599 amdgpu_ras_instance_mask_check(adev, &data); 600 601 /* data.inject.address is offset instead of absolute gpu address */ 602 ret = amdgpu_ras_error_inject(adev, &data.inject); 603 break; 604 default: 605 ret = -EINVAL; 606 break; 607 } 608 609 if (ret) 610 return ret; 611 612 return size; 613 } 614 615 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 616 617 /** 618 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 619 * 620 * Some boards contain an EEPROM which is used to persistently store a list of 621 * bad pages which experiences ECC errors in vram. This interface provides 622 * a way to reset the EEPROM, e.g., after testing error injection. 623 * 624 * Usage: 625 * 626 * .. code-block:: bash 627 * 628 * echo 1 > ../ras/ras_eeprom_reset 629 * 630 * will reset EEPROM table to 0 entries. 631 * 632 */ 633 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 634 const char __user *buf, 635 size_t size, loff_t *pos) 636 { 637 struct amdgpu_device *adev = 638 (struct amdgpu_device *)file_inode(f)->i_private; 639 int ret; 640 641 if (amdgpu_uniras_enabled(adev)) { 642 ret = amdgpu_uniras_clear_badpages_info(adev); 643 return ret ? ret : size; 644 } 645 646 ret = amdgpu_ras_eeprom_reset_table( 647 &(amdgpu_ras_get_context(adev)->eeprom_control)); 648 649 if (!ret) { 650 /* Something was written to EEPROM. 651 */ 652 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 653 return size; 654 } else { 655 return ret; 656 } 657 } 658 659 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 660 .owner = THIS_MODULE, 661 .read = NULL, 662 .write = amdgpu_ras_debugfs_ctrl_write, 663 .llseek = default_llseek 664 }; 665 666 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 667 .owner = THIS_MODULE, 668 .read = NULL, 669 .write = amdgpu_ras_debugfs_eeprom_write, 670 .llseek = default_llseek 671 }; 672 673 /** 674 * DOC: AMDGPU RAS sysfs Error Count Interface 675 * 676 * It allows the user to read the error count for each IP block on the gpu through 677 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 678 * 679 * It outputs the multiple lines which report the uncorrected (ue) and corrected 680 * (ce) error counts. 681 * 682 * The format of one line is below, 683 * 684 * [ce|ue]: count 685 * 686 * Example: 687 * 688 * .. code-block:: bash 689 * 690 * ue: 0 691 * ce: 1 692 * 693 */ 694 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 695 struct device_attribute *attr, char *buf) 696 { 697 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 698 struct ras_query_if info = { 699 .head = obj->head, 700 }; 701 702 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 703 return sysfs_emit(buf, "Query currently inaccessible\n"); 704 705 if (amdgpu_ras_query_error_status(obj->adev, &info)) 706 return -EINVAL; 707 708 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 709 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 710 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 711 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 712 } 713 714 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 715 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 716 "ce", info.ce_count, "de", info.de_count); 717 else 718 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 719 "ce", info.ce_count); 720 } 721 722 /* obj begin */ 723 724 #define get_obj(obj) do { (obj)->use++; } while (0) 725 #define alive_obj(obj) ((obj)->use) 726 727 static inline void put_obj(struct ras_manager *obj) 728 { 729 if (obj && (--obj->use == 0)) { 730 list_del(&obj->node); 731 amdgpu_ras_error_data_fini(&obj->err_data); 732 } 733 734 if (obj && (obj->use < 0)) 735 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 736 } 737 738 /* make one obj and return it. */ 739 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 740 struct ras_common_if *head) 741 { 742 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 743 struct ras_manager *obj; 744 745 if (!adev->ras_enabled || !con) 746 return NULL; 747 748 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 749 return NULL; 750 751 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 752 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 753 return NULL; 754 755 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 756 } else 757 obj = &con->objs[head->block]; 758 759 /* already exist. return obj? */ 760 if (alive_obj(obj)) 761 return NULL; 762 763 if (amdgpu_ras_error_data_init(&obj->err_data)) 764 return NULL; 765 766 obj->head = *head; 767 obj->adev = adev; 768 list_add(&obj->node, &con->head); 769 get_obj(obj); 770 771 return obj; 772 } 773 774 /* return an obj equal to head, or the first when head is NULL */ 775 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 776 struct ras_common_if *head) 777 { 778 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 779 struct ras_manager *obj; 780 int i; 781 782 if (!adev->ras_enabled || !con) 783 return NULL; 784 785 if (head) { 786 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 787 return NULL; 788 789 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 790 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 791 return NULL; 792 793 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 794 } else 795 obj = &con->objs[head->block]; 796 797 if (alive_obj(obj)) 798 return obj; 799 } else { 800 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 801 obj = &con->objs[i]; 802 if (alive_obj(obj)) 803 return obj; 804 } 805 } 806 807 return NULL; 808 } 809 /* obj end */ 810 811 /* feature ctl begin */ 812 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 813 struct ras_common_if *head) 814 { 815 return adev->ras_hw_enabled & BIT(head->block); 816 } 817 818 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 819 struct ras_common_if *head) 820 { 821 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 822 823 return con->features & BIT(head->block); 824 } 825 826 /* 827 * if obj is not created, then create one. 828 * set feature enable flag. 829 */ 830 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 831 struct ras_common_if *head, int enable) 832 { 833 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 834 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 835 836 /* If hardware does not support ras, then do not create obj. 837 * But if hardware support ras, we can create the obj. 838 * Ras framework checks con->hw_supported to see if it need do 839 * corresponding initialization. 840 * IP checks con->support to see if it need disable ras. 841 */ 842 if (!amdgpu_ras_is_feature_allowed(adev, head)) 843 return 0; 844 845 if (enable) { 846 if (!obj) { 847 obj = amdgpu_ras_create_obj(adev, head); 848 if (!obj) 849 return -EINVAL; 850 } else { 851 /* In case we create obj somewhere else */ 852 get_obj(obj); 853 } 854 con->features |= BIT(head->block); 855 } else { 856 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 857 con->features &= ~BIT(head->block); 858 put_obj(obj); 859 } 860 } 861 862 return 0; 863 } 864 865 /* wrapper of psp_ras_enable_features */ 866 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 867 struct ras_common_if *head, bool enable) 868 { 869 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 870 union ta_ras_cmd_input *info; 871 int ret; 872 873 if (!con) 874 return -EINVAL; 875 876 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 877 /* For gfx ip, regardless of feature support status, */ 878 /* Force issue enable or disable ras feature commands */ 879 if (head->block != AMDGPU_RAS_BLOCK__GFX && 880 !amdgpu_ras_is_feature_allowed(adev, head)) 881 return 0; 882 883 /* Only enable gfx ras feature from host side */ 884 if (head->block == AMDGPU_RAS_BLOCK__GFX && 885 !amdgpu_sriov_vf(adev) && 886 !amdgpu_ras_intr_triggered()) { 887 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 888 if (!info) 889 return -ENOMEM; 890 891 if (!enable) { 892 info->disable_features = (struct ta_ras_disable_features_input) { 893 .block_id = amdgpu_ras_block_to_ta(head->block), 894 .error_type = amdgpu_ras_error_to_ta(head->type), 895 }; 896 } else { 897 info->enable_features = (struct ta_ras_enable_features_input) { 898 .block_id = amdgpu_ras_block_to_ta(head->block), 899 .error_type = amdgpu_ras_error_to_ta(head->type), 900 }; 901 } 902 903 ret = psp_ras_enable_features(&adev->psp, info, enable); 904 if (ret) { 905 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 906 enable ? "enable":"disable", 907 get_ras_block_str(head), 908 amdgpu_ras_is_poison_mode_supported(adev), ret); 909 kfree(info); 910 return ret; 911 } 912 913 kfree(info); 914 } 915 916 /* setup the obj */ 917 __amdgpu_ras_feature_enable(adev, head, enable); 918 919 return 0; 920 } 921 922 /* Only used in device probe stage and called only once. */ 923 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 924 struct ras_common_if *head, bool enable) 925 { 926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 927 int ret; 928 929 if (!con) 930 return -EINVAL; 931 932 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 933 if (enable) { 934 /* There is no harm to issue a ras TA cmd regardless of 935 * the currecnt ras state. 936 * If current state == target state, it will do nothing 937 * But sometimes it requests driver to reset and repost 938 * with error code -EAGAIN. 939 */ 940 ret = amdgpu_ras_feature_enable(adev, head, 1); 941 /* With old ras TA, we might fail to enable ras. 942 * Log it and just setup the object. 943 * TODO need remove this WA in the future. 944 */ 945 if (ret == -EINVAL) { 946 ret = __amdgpu_ras_feature_enable(adev, head, 1); 947 if (!ret) 948 dev_info(adev->dev, 949 "RAS INFO: %s setup object\n", 950 get_ras_block_str(head)); 951 } 952 } else { 953 /* setup the object then issue a ras TA disable cmd.*/ 954 ret = __amdgpu_ras_feature_enable(adev, head, 1); 955 if (ret) 956 return ret; 957 958 /* gfx block ras disable cmd must send to ras-ta */ 959 if (head->block == AMDGPU_RAS_BLOCK__GFX) 960 con->features |= BIT(head->block); 961 962 ret = amdgpu_ras_feature_enable(adev, head, 0); 963 964 /* clean gfx block ras features flag */ 965 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 966 con->features &= ~BIT(head->block); 967 } 968 } else 969 ret = amdgpu_ras_feature_enable(adev, head, enable); 970 971 return ret; 972 } 973 974 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 975 bool bypass) 976 { 977 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 978 struct ras_manager *obj, *tmp; 979 980 list_for_each_entry_safe(obj, tmp, &con->head, node) { 981 /* bypass psp. 982 * aka just release the obj and corresponding flags 983 */ 984 if (bypass) { 985 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 986 break; 987 } else { 988 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 989 break; 990 } 991 } 992 993 return con->features; 994 } 995 996 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 997 bool bypass) 998 { 999 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1000 int i; 1001 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1002 1003 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1004 struct ras_common_if head = { 1005 .block = i, 1006 .type = default_ras_type, 1007 .sub_block_index = 0, 1008 }; 1009 1010 if (i == AMDGPU_RAS_BLOCK__MCA) 1011 continue; 1012 1013 if (bypass) { 1014 /* 1015 * bypass psp. vbios enable ras for us. 1016 * so just create the obj 1017 */ 1018 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1019 break; 1020 } else { 1021 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1022 break; 1023 } 1024 } 1025 1026 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1027 struct ras_common_if head = { 1028 .block = AMDGPU_RAS_BLOCK__MCA, 1029 .type = default_ras_type, 1030 .sub_block_index = i, 1031 }; 1032 1033 if (bypass) { 1034 /* 1035 * bypass psp. vbios enable ras for us. 1036 * so just create the obj 1037 */ 1038 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1039 break; 1040 } else { 1041 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1042 break; 1043 } 1044 } 1045 1046 return con->features; 1047 } 1048 /* feature ctl end */ 1049 1050 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1051 enum amdgpu_ras_block block) 1052 { 1053 if (!block_obj) 1054 return -EINVAL; 1055 1056 if (block_obj->ras_comm.block == block) 1057 return 0; 1058 1059 return -EINVAL; 1060 } 1061 1062 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1063 enum amdgpu_ras_block block, uint32_t sub_block_index) 1064 { 1065 struct amdgpu_ras_block_list *node, *tmp; 1066 struct amdgpu_ras_block_object *obj; 1067 1068 if (block >= AMDGPU_RAS_BLOCK__LAST) 1069 return NULL; 1070 1071 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1072 if (!node->ras_obj) { 1073 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1074 continue; 1075 } 1076 1077 obj = node->ras_obj; 1078 if (obj->ras_block_match) { 1079 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1080 return obj; 1081 } else { 1082 if (amdgpu_ras_block_match_default(obj, block) == 0) 1083 return obj; 1084 } 1085 } 1086 1087 return NULL; 1088 } 1089 1090 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1091 { 1092 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1093 int ret = 0; 1094 1095 /* 1096 * choosing right query method according to 1097 * whether smu support query error information 1098 */ 1099 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1100 if (ret == -EOPNOTSUPP) { 1101 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1102 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1103 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1104 1105 /* umc query_ras_error_address is also responsible for clearing 1106 * error status 1107 */ 1108 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1109 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1110 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1111 } else if (!ret) { 1112 if (adev->umc.ras && 1113 adev->umc.ras->ecc_info_query_ras_error_count) 1114 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1115 1116 if (adev->umc.ras && 1117 adev->umc.ras->ecc_info_query_ras_error_address) 1118 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1119 } 1120 } 1121 1122 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1123 struct ras_manager *ras_mgr, 1124 struct ras_err_data *err_data, 1125 struct ras_query_context *qctx, 1126 const char *blk_name, 1127 bool is_ue, 1128 bool is_de) 1129 { 1130 struct amdgpu_smuio_mcm_config_info *mcm_info; 1131 struct ras_err_node *err_node; 1132 struct ras_err_info *err_info; 1133 u64 event_id = qctx->evid.event_id; 1134 1135 if (is_ue) { 1136 for_each_ras_error(err_node, err_data) { 1137 err_info = &err_node->err_info; 1138 mcm_info = &err_info->mcm_info; 1139 if (err_info->ue_count) { 1140 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1141 "%lld new uncorrectable hardware errors detected in %s block\n", 1142 mcm_info->socket_id, 1143 mcm_info->die_id, 1144 err_info->ue_count, 1145 blk_name); 1146 } 1147 } 1148 1149 for_each_ras_error(err_node, &ras_mgr->err_data) { 1150 err_info = &err_node->err_info; 1151 mcm_info = &err_info->mcm_info; 1152 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1153 "%lld uncorrectable hardware errors detected in total in %s block\n", 1154 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1155 } 1156 1157 } else { 1158 if (is_de) { 1159 for_each_ras_error(err_node, err_data) { 1160 err_info = &err_node->err_info; 1161 mcm_info = &err_info->mcm_info; 1162 if (err_info->de_count) { 1163 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1164 "%lld new deferred hardware errors detected in %s block\n", 1165 mcm_info->socket_id, 1166 mcm_info->die_id, 1167 err_info->de_count, 1168 blk_name); 1169 } 1170 } 1171 1172 for_each_ras_error(err_node, &ras_mgr->err_data) { 1173 err_info = &err_node->err_info; 1174 mcm_info = &err_info->mcm_info; 1175 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1176 "%lld deferred hardware errors detected in total in %s block\n", 1177 mcm_info->socket_id, mcm_info->die_id, 1178 err_info->de_count, blk_name); 1179 } 1180 } else { 1181 if (adev->debug_disable_ce_logs) 1182 return; 1183 1184 for_each_ras_error(err_node, err_data) { 1185 err_info = &err_node->err_info; 1186 mcm_info = &err_info->mcm_info; 1187 if (err_info->ce_count) { 1188 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1189 "%lld new correctable hardware errors detected in %s block\n", 1190 mcm_info->socket_id, 1191 mcm_info->die_id, 1192 err_info->ce_count, 1193 blk_name); 1194 } 1195 } 1196 1197 for_each_ras_error(err_node, &ras_mgr->err_data) { 1198 err_info = &err_node->err_info; 1199 mcm_info = &err_info->mcm_info; 1200 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1201 "%lld correctable hardware errors detected in total in %s block\n", 1202 mcm_info->socket_id, mcm_info->die_id, 1203 err_info->ce_count, blk_name); 1204 } 1205 } 1206 } 1207 } 1208 1209 static inline bool err_data_has_source_info(struct ras_err_data *data) 1210 { 1211 return !list_empty(&data->err_node_list); 1212 } 1213 1214 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1215 struct ras_query_if *query_if, 1216 struct ras_err_data *err_data, 1217 struct ras_query_context *qctx) 1218 { 1219 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1220 const char *blk_name = get_ras_block_str(&query_if->head); 1221 u64 event_id = qctx->evid.event_id; 1222 1223 if (err_data->ce_count) { 1224 if (err_data_has_source_info(err_data)) { 1225 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1226 blk_name, false, false); 1227 } else if (!adev->aid_mask && 1228 adev->smuio.funcs && 1229 adev->smuio.funcs->get_socket_id && 1230 adev->smuio.funcs->get_die_id) { 1231 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1232 "%ld correctable hardware errors " 1233 "detected in %s block\n", 1234 adev->smuio.funcs->get_socket_id(adev), 1235 adev->smuio.funcs->get_die_id(adev), 1236 ras_mgr->err_data.ce_count, 1237 blk_name); 1238 } else { 1239 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1240 "detected in %s block\n", 1241 ras_mgr->err_data.ce_count, 1242 blk_name); 1243 } 1244 } 1245 1246 if (err_data->ue_count) { 1247 if (err_data_has_source_info(err_data)) { 1248 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1249 blk_name, true, false); 1250 } else if (!adev->aid_mask && 1251 adev->smuio.funcs && 1252 adev->smuio.funcs->get_socket_id && 1253 adev->smuio.funcs->get_die_id) { 1254 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1255 "%ld uncorrectable hardware errors " 1256 "detected in %s block\n", 1257 adev->smuio.funcs->get_socket_id(adev), 1258 adev->smuio.funcs->get_die_id(adev), 1259 ras_mgr->err_data.ue_count, 1260 blk_name); 1261 } else { 1262 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1263 "detected in %s block\n", 1264 ras_mgr->err_data.ue_count, 1265 blk_name); 1266 } 1267 } 1268 1269 if (err_data->de_count) { 1270 if (err_data_has_source_info(err_data)) { 1271 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1272 blk_name, false, true); 1273 } else if (!adev->aid_mask && 1274 adev->smuio.funcs && 1275 adev->smuio.funcs->get_socket_id && 1276 adev->smuio.funcs->get_die_id) { 1277 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1278 "%ld deferred hardware errors " 1279 "detected in %s block\n", 1280 adev->smuio.funcs->get_socket_id(adev), 1281 adev->smuio.funcs->get_die_id(adev), 1282 ras_mgr->err_data.de_count, 1283 blk_name); 1284 } else { 1285 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1286 "detected in %s block\n", 1287 ras_mgr->err_data.de_count, 1288 blk_name); 1289 } 1290 } 1291 } 1292 1293 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1294 struct ras_query_if *query_if, 1295 struct ras_err_data *err_data, 1296 struct ras_query_context *qctx) 1297 { 1298 unsigned long new_ue, new_ce, new_de; 1299 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1300 const char *blk_name = get_ras_block_str(&query_if->head); 1301 u64 event_id = qctx->evid.event_id; 1302 1303 new_ce = err_data->ce_count - obj->err_data.ce_count; 1304 new_ue = err_data->ue_count - obj->err_data.ue_count; 1305 new_de = err_data->de_count - obj->err_data.de_count; 1306 1307 if (new_ce) { 1308 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1309 "detected in %s block\n", 1310 new_ce, 1311 blk_name); 1312 } 1313 1314 if (new_ue) { 1315 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1316 "detected in %s block\n", 1317 new_ue, 1318 blk_name); 1319 } 1320 1321 if (new_de) { 1322 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1323 "detected in %s block\n", 1324 new_de, 1325 blk_name); 1326 } 1327 } 1328 1329 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1330 { 1331 struct ras_err_node *err_node; 1332 struct ras_err_info *err_info; 1333 1334 if (err_data_has_source_info(err_data)) { 1335 for_each_ras_error(err_node, err_data) { 1336 err_info = &err_node->err_info; 1337 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1338 &err_info->mcm_info, err_info->de_count); 1339 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->ce_count); 1341 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ue_count); 1343 } 1344 } else { 1345 /* for legacy asic path which doesn't has error source info */ 1346 obj->err_data.ue_count += err_data->ue_count; 1347 obj->err_data.ce_count += err_data->ce_count; 1348 obj->err_data.de_count += err_data->de_count; 1349 } 1350 } 1351 1352 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1353 struct ras_err_data *err_data) 1354 { 1355 /* Host reports absolute counts */ 1356 obj->err_data.ue_count = err_data->ue_count; 1357 obj->err_data.ce_count = err_data->ce_count; 1358 obj->err_data.de_count = err_data->de_count; 1359 } 1360 1361 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1362 { 1363 struct ras_common_if head; 1364 1365 memset(&head, 0, sizeof(head)); 1366 head.block = blk; 1367 1368 return amdgpu_ras_find_obj(adev, &head); 1369 } 1370 1371 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1372 const struct aca_info *aca_info, void *data) 1373 { 1374 struct ras_manager *obj; 1375 1376 /* in resume phase, no need to create aca fs node */ 1377 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1378 return 0; 1379 1380 obj = get_ras_manager(adev, blk); 1381 if (!obj) 1382 return -EINVAL; 1383 1384 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1385 } 1386 1387 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1388 { 1389 struct ras_manager *obj; 1390 1391 obj = get_ras_manager(adev, blk); 1392 if (!obj) 1393 return -EINVAL; 1394 1395 amdgpu_aca_remove_handle(&obj->aca_handle); 1396 1397 return 0; 1398 } 1399 1400 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1401 enum aca_error_type type, struct ras_err_data *err_data, 1402 struct ras_query_context *qctx) 1403 { 1404 struct ras_manager *obj; 1405 1406 obj = get_ras_manager(adev, blk); 1407 if (!obj) 1408 return -EINVAL; 1409 1410 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1411 } 1412 1413 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1414 struct aca_handle *handle, char *buf, void *data) 1415 { 1416 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1417 struct ras_query_if info = { 1418 .head = obj->head, 1419 }; 1420 1421 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1422 return sysfs_emit(buf, "Query currently inaccessible\n"); 1423 1424 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1425 return -EINVAL; 1426 1427 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1428 "ce", info.ce_count, "de", info.de_count); 1429 } 1430 1431 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1432 struct ras_query_if *info, 1433 struct ras_err_data *err_data, 1434 struct ras_query_context *qctx, 1435 unsigned int error_query_mode) 1436 { 1437 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1438 struct amdgpu_ras_block_object *block_obj = NULL; 1439 int ret; 1440 1441 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1442 return -EINVAL; 1443 1444 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1445 return -EINVAL; 1446 1447 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1448 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1449 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1450 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1451 amdgpu_ras_get_ecc_info(adev, err_data); 1452 } else { 1453 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1454 if (!block_obj || !block_obj->hw_ops) { 1455 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1456 get_ras_block_str(&info->head)); 1457 return -EINVAL; 1458 } 1459 1460 if (block_obj->hw_ops->query_ras_error_count) 1461 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1462 1463 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1464 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1465 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1466 if (block_obj->hw_ops->query_ras_error_status) 1467 block_obj->hw_ops->query_ras_error_status(adev); 1468 } 1469 } 1470 } else { 1471 if (amdgpu_aca_is_enabled(adev)) { 1472 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1473 if (ret) 1474 return ret; 1475 1476 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1477 if (ret) 1478 return ret; 1479 1480 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1481 if (ret) 1482 return ret; 1483 } else { 1484 /* FIXME: add code to check return value later */ 1485 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1486 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1487 } 1488 } 1489 1490 return 0; 1491 } 1492 1493 /* query/inject/cure begin */ 1494 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1495 struct ras_query_if *info, 1496 enum ras_event_type type) 1497 { 1498 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1499 struct ras_err_data err_data; 1500 struct ras_query_context qctx; 1501 unsigned int error_query_mode; 1502 int ret; 1503 1504 if (!obj) 1505 return -EINVAL; 1506 1507 ret = amdgpu_ras_error_data_init(&err_data); 1508 if (ret) 1509 return ret; 1510 1511 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1512 return -EINVAL; 1513 1514 memset(&qctx, 0, sizeof(qctx)); 1515 qctx.evid.type = type; 1516 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1517 1518 if (!down_read_trylock(&adev->reset_domain->sem)) { 1519 ret = -EIO; 1520 goto out_fini_err_data; 1521 } 1522 1523 ret = amdgpu_ras_query_error_status_helper(adev, info, 1524 &err_data, 1525 &qctx, 1526 error_query_mode); 1527 up_read(&adev->reset_domain->sem); 1528 if (ret) 1529 goto out_fini_err_data; 1530 1531 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1532 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1533 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1534 } else { 1535 /* Host provides absolute error counts. First generate the report 1536 * using the previous VF internal count against new host count. 1537 * Then Update VF internal count. 1538 */ 1539 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1540 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1541 } 1542 1543 info->ue_count = obj->err_data.ue_count; 1544 info->ce_count = obj->err_data.ce_count; 1545 info->de_count = obj->err_data.de_count; 1546 1547 out_fini_err_data: 1548 amdgpu_ras_error_data_fini(&err_data); 1549 1550 return ret; 1551 } 1552 1553 static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1554 { 1555 struct ras_cmd_dev_handle req = {0}; 1556 int ret; 1557 1558 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1559 &req, sizeof(req), NULL, 0); 1560 if (ret) { 1561 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1562 return ret; 1563 } 1564 1565 return 0; 1566 } 1567 1568 static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1569 struct ras_query_if *info) 1570 { 1571 struct ras_cmd_block_ecc_info_req req = {0}; 1572 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1573 int ret; 1574 1575 if (!info) 1576 return -EINVAL; 1577 1578 req.block_id = info->head.block; 1579 req.subblock_id = info->head.sub_block_index; 1580 1581 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1582 &req, sizeof(req), &rsp, sizeof(rsp)); 1583 if (!ret) { 1584 info->ce_count = rsp.ce_count; 1585 info->ue_count = rsp.ue_count; 1586 info->de_count = rsp.de_count; 1587 } 1588 1589 return ret; 1590 } 1591 1592 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1593 { 1594 if (amdgpu_uniras_enabled(adev)) 1595 return amdgpu_uniras_query_block_ecc(adev, info); 1596 else 1597 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1598 } 1599 1600 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1601 enum amdgpu_ras_block block) 1602 { 1603 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1604 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1605 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1606 1607 if (!block_obj || !block_obj->hw_ops) { 1608 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1609 ras_block_str(block)); 1610 return -EOPNOTSUPP; 1611 } 1612 1613 if (!amdgpu_ras_is_supported(adev, block) || 1614 !amdgpu_ras_get_aca_debug_mode(adev)) 1615 return -EOPNOTSUPP; 1616 1617 if (amdgpu_sriov_vf(adev)) 1618 return -EOPNOTSUPP; 1619 1620 /* skip ras error reset in gpu reset */ 1621 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1622 ((smu_funcs && smu_funcs->set_debug_mode) || 1623 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1624 return -EOPNOTSUPP; 1625 1626 if (block_obj->hw_ops->reset_ras_error_count) 1627 block_obj->hw_ops->reset_ras_error_count(adev); 1628 1629 return 0; 1630 } 1631 1632 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1633 enum amdgpu_ras_block block) 1634 { 1635 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1636 1637 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1638 return 0; 1639 1640 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1641 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1642 if (block_obj->hw_ops->reset_ras_error_status) 1643 block_obj->hw_ops->reset_ras_error_status(adev); 1644 } 1645 1646 return 0; 1647 } 1648 1649 static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1650 struct ras_inject_if *info) 1651 { 1652 struct ras_cmd_inject_error_req inject_req; 1653 struct ras_cmd_inject_error_rsp rsp; 1654 1655 if (!info) 1656 return -EINVAL; 1657 1658 memset(&inject_req, 0, sizeof(inject_req)); 1659 inject_req.block_id = info->head.block; 1660 inject_req.subblock_id = info->head.sub_block_index; 1661 inject_req.address = info->address; 1662 inject_req.error_type = info->head.type; 1663 inject_req.instance_mask = info->instance_mask; 1664 inject_req.method = info->value; 1665 1666 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1667 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1668 } 1669 1670 /* wrapper of psp_ras_trigger_error */ 1671 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1672 struct ras_inject_if *info) 1673 { 1674 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1675 struct ta_ras_trigger_error_input block_info = { 1676 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1677 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1678 .sub_block_index = info->head.sub_block_index, 1679 .address = info->address, 1680 .value = info->value, 1681 }; 1682 int ret = -EINVAL; 1683 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1684 info->head.block, 1685 info->head.sub_block_index); 1686 1687 if (amdgpu_uniras_enabled(adev)) 1688 return amdgpu_uniras_error_inject(adev, info); 1689 1690 /* inject on guest isn't allowed, return success directly */ 1691 if (amdgpu_sriov_vf(adev)) 1692 return 0; 1693 1694 if (!obj) 1695 return -EINVAL; 1696 1697 if (!block_obj || !block_obj->hw_ops) { 1698 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1699 get_ras_block_str(&info->head)); 1700 return -EINVAL; 1701 } 1702 1703 /* Calculate XGMI relative offset */ 1704 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1705 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1706 block_info.address = 1707 amdgpu_xgmi_get_relative_phy_addr(adev, 1708 block_info.address); 1709 } 1710 1711 if (block_obj->hw_ops->ras_error_inject) { 1712 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1713 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1714 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1715 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1716 info->instance_mask); 1717 } else { 1718 /* default path */ 1719 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1720 } 1721 1722 if (ret) 1723 dev_err(adev->dev, "ras inject %s failed %d\n", 1724 get_ras_block_str(&info->head), ret); 1725 1726 return ret; 1727 } 1728 1729 /** 1730 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1731 * @adev: pointer to AMD GPU device 1732 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1733 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1734 * @query_info: pointer to ras_query_if 1735 * 1736 * Return 0 for query success or do nothing, otherwise return an error 1737 * on failures 1738 */ 1739 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1740 unsigned long *ce_count, 1741 unsigned long *ue_count, 1742 struct ras_query_if *query_info) 1743 { 1744 int ret; 1745 1746 if (!query_info) 1747 /* do nothing if query_info is not specified */ 1748 return 0; 1749 1750 ret = amdgpu_ras_query_error_status(adev, query_info); 1751 if (ret) 1752 return ret; 1753 1754 *ce_count += query_info->ce_count; 1755 *ue_count += query_info->ue_count; 1756 1757 /* some hardware/IP supports read to clear 1758 * no need to explictly reset the err status after the query call */ 1759 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1760 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1761 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1762 dev_warn(adev->dev, 1763 "Failed to reset error counter and error status\n"); 1764 } 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1771 * @adev: pointer to AMD GPU device 1772 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1773 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1774 * errors. 1775 * @query_info: pointer to ras_query_if if the query request is only for 1776 * specific ip block; if info is NULL, then the qurey request is for 1777 * all the ip blocks that support query ras error counters/status 1778 * 1779 * If set, @ce_count or @ue_count, count and return the corresponding 1780 * error counts in those integer pointers. Return 0 if the device 1781 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1782 */ 1783 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1784 unsigned long *ce_count, 1785 unsigned long *ue_count, 1786 struct ras_query_if *query_info) 1787 { 1788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1789 struct ras_manager *obj; 1790 unsigned long ce, ue; 1791 int ret; 1792 1793 if (!adev->ras_enabled || !con) 1794 return -EOPNOTSUPP; 1795 1796 /* Don't count since no reporting. 1797 */ 1798 if (!ce_count && !ue_count) 1799 return 0; 1800 1801 ce = 0; 1802 ue = 0; 1803 if (!query_info) { 1804 /* query all the ip blocks that support ras query interface */ 1805 list_for_each_entry(obj, &con->head, node) { 1806 struct ras_query_if info = { 1807 .head = obj->head, 1808 }; 1809 1810 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1811 } 1812 } else { 1813 /* query specific ip block */ 1814 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1815 } 1816 1817 if (ret) 1818 return ret; 1819 1820 if (ce_count) 1821 *ce_count = ce; 1822 1823 if (ue_count) 1824 *ue_count = ue; 1825 1826 return 0; 1827 } 1828 /* query/inject/cure end */ 1829 1830 1831 /* sysfs begin */ 1832 1833 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1834 struct ras_badpage *bps, uint32_t count, uint32_t start); 1835 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837 1838 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1839 { 1840 switch (flags) { 1841 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1842 return "R"; 1843 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1844 return "P"; 1845 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1846 default: 1847 return "F"; 1848 } 1849 } 1850 1851 /** 1852 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1853 * 1854 * It allows user to read the bad pages of vram on the gpu through 1855 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1856 * 1857 * It outputs multiple lines, and each line stands for one gpu page. 1858 * 1859 * The format of one line is below, 1860 * gpu pfn : gpu page size : flags 1861 * 1862 * gpu pfn and gpu page size are printed in hex format. 1863 * flags can be one of below character, 1864 * 1865 * R: reserved, this gpu page is reserved and not able to use. 1866 * 1867 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1868 * in next window of page_reserve. 1869 * 1870 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1871 * 1872 * Examples: 1873 * 1874 * .. code-block:: bash 1875 * 1876 * 0x00000001 : 0x00001000 : R 1877 * 0x00000002 : 0x00001000 : P 1878 * 1879 */ 1880 1881 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1882 struct kobject *kobj, const struct bin_attribute *attr, 1883 char *buf, loff_t ppos, size_t count) 1884 { 1885 struct amdgpu_ras *con = 1886 container_of(attr, struct amdgpu_ras, badpages_attr); 1887 struct amdgpu_device *adev = con->adev; 1888 const unsigned int element_size = 1889 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1890 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1891 unsigned int end = div64_ul(ppos + count - 1, element_size); 1892 ssize_t s = 0; 1893 struct ras_badpage *bps = NULL; 1894 int bps_count = 0, i, status; 1895 uint64_t address; 1896 1897 memset(buf, 0, count); 1898 1899 bps_count = end - start; 1900 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1901 if (!bps) 1902 return 0; 1903 1904 memset(bps, 0, sizeof(*bps) * bps_count); 1905 1906 if (amdgpu_uniras_enabled(adev)) 1907 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1908 else 1909 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1910 1911 if (bps_count <= 0) { 1912 kfree(bps); 1913 return 0; 1914 } 1915 1916 for (i = 0; i < bps_count; i++) { 1917 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1918 if (amdgpu_ras_check_critical_address(adev, address)) 1919 continue; 1920 1921 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1922 1923 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1924 address); 1925 if (status == -EBUSY) 1926 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1927 else if (status == -ENOENT) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1929 else 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1931 1932 s += scnprintf(&buf[s], element_size + 1, 1933 "0x%08x : 0x%08x : %1s\n", 1934 bps[i].bp, 1935 bps[i].size, 1936 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1937 } 1938 1939 kfree(bps); 1940 1941 return s; 1942 } 1943 1944 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1945 struct device_attribute *attr, char *buf) 1946 { 1947 struct amdgpu_ras *con = 1948 container_of(attr, struct amdgpu_ras, features_attr); 1949 1950 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1951 } 1952 1953 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1954 u32 *minor, u32 *rev) 1955 { 1956 int i; 1957 1958 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1959 return false; 1960 1961 for (i = 0; i < adev->num_ip_blocks; i++) { 1962 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1963 *major = adev->ip_blocks[i].version->major; 1964 *minor = adev->ip_blocks[i].version->minor; 1965 *rev = adev->ip_blocks[i].version->rev; 1966 return true; 1967 } 1968 } 1969 1970 return false; 1971 } 1972 1973 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1974 struct device_attribute *attr, char *buf) 1975 { 1976 struct amdgpu_ras *con = 1977 container_of(attr, struct amdgpu_ras, version_attr); 1978 u32 major, minor, rev; 1979 ssize_t size = 0; 1980 1981 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1982 con->eeprom_control.tbl_hdr.version); 1983 1984 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1985 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1986 major, minor, rev); 1987 1988 return size; 1989 } 1990 1991 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1992 struct device_attribute *attr, char *buf) 1993 { 1994 struct amdgpu_ras *con = 1995 container_of(attr, struct amdgpu_ras, schema_attr); 1996 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1997 } 1998 1999 static struct { 2000 enum ras_event_type type; 2001 const char *name; 2002 } dump_event[] = { 2003 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2004 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2005 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2006 }; 2007 2008 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2009 struct device_attribute *attr, char *buf) 2010 { 2011 struct amdgpu_ras *con = 2012 container_of(attr, struct amdgpu_ras, event_state_attr); 2013 struct ras_event_manager *event_mgr = con->event_mgr; 2014 struct ras_event_state *event_state; 2015 int i, size = 0; 2016 2017 if (!event_mgr) 2018 return -EINVAL; 2019 2020 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2021 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2022 event_state = &event_mgr->event_state[dump_event[i].type]; 2023 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2024 dump_event[i].name, 2025 atomic64_read(&event_state->count), 2026 event_state->last_seqno); 2027 } 2028 2029 return (ssize_t)size; 2030 } 2031 2032 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2033 { 2034 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2035 2036 if (adev->dev->kobj.sd) 2037 sysfs_remove_file_from_group(&adev->dev->kobj, 2038 &con->badpages_attr.attr, 2039 RAS_FS_NAME); 2040 } 2041 2042 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2043 { 2044 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2045 struct attribute *attrs[] = { 2046 &con->features_attr.attr, 2047 &con->version_attr.attr, 2048 &con->schema_attr.attr, 2049 &con->event_state_attr.attr, 2050 NULL 2051 }; 2052 struct attribute_group group = { 2053 .name = RAS_FS_NAME, 2054 .attrs = attrs, 2055 }; 2056 2057 if (adev->dev->kobj.sd) 2058 sysfs_remove_group(&adev->dev->kobj, &group); 2059 2060 return 0; 2061 } 2062 2063 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2064 struct ras_common_if *head) 2065 { 2066 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2067 2068 if (amdgpu_aca_is_enabled(adev)) 2069 return 0; 2070 2071 if (!obj || obj->attr_inuse) 2072 return -EINVAL; 2073 2074 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2075 return 0; 2076 2077 get_obj(obj); 2078 2079 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2080 "%s_err_count", head->name); 2081 2082 obj->sysfs_attr = (struct device_attribute){ 2083 .attr = { 2084 .name = obj->fs_data.sysfs_name, 2085 .mode = S_IRUGO, 2086 }, 2087 .show = amdgpu_ras_sysfs_read, 2088 }; 2089 sysfs_attr_init(&obj->sysfs_attr.attr); 2090 2091 if (sysfs_add_file_to_group(&adev->dev->kobj, 2092 &obj->sysfs_attr.attr, 2093 RAS_FS_NAME)) { 2094 put_obj(obj); 2095 return -EINVAL; 2096 } 2097 2098 obj->attr_inuse = 1; 2099 2100 return 0; 2101 } 2102 2103 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2104 struct ras_common_if *head) 2105 { 2106 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2107 2108 if (amdgpu_aca_is_enabled(adev)) 2109 return 0; 2110 2111 if (!obj || !obj->attr_inuse) 2112 return -EINVAL; 2113 2114 if (adev->dev->kobj.sd) 2115 sysfs_remove_file_from_group(&adev->dev->kobj, 2116 &obj->sysfs_attr.attr, 2117 RAS_FS_NAME); 2118 obj->attr_inuse = 0; 2119 put_obj(obj); 2120 2121 return 0; 2122 } 2123 2124 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2125 { 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2127 struct ras_manager *obj, *tmp; 2128 2129 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2130 amdgpu_ras_sysfs_remove(adev, &obj->head); 2131 } 2132 2133 if (amdgpu_bad_page_threshold != 0) 2134 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2135 2136 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2137 2138 return 0; 2139 } 2140 /* sysfs end */ 2141 2142 /** 2143 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2144 * 2145 * Normally when there is an uncorrectable error, the driver will reset 2146 * the GPU to recover. However, in the event of an unrecoverable error, 2147 * the driver provides an interface to reboot the system automatically 2148 * in that event. 2149 * 2150 * The following file in debugfs provides that interface: 2151 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2152 * 2153 * Usage: 2154 * 2155 * .. code-block:: bash 2156 * 2157 * echo true > .../ras/auto_reboot 2158 * 2159 */ 2160 /* debugfs begin */ 2161 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2162 { 2163 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2164 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2165 struct drm_minor *minor = adev_to_drm(adev)->primary; 2166 struct dentry *dir; 2167 2168 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2169 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2170 &amdgpu_ras_debugfs_ctrl_ops); 2171 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_eeprom_ops); 2173 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2174 &con->bad_page_cnt_threshold); 2175 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2176 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2177 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2178 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2179 &amdgpu_ras_debugfs_eeprom_size_ops); 2180 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2181 S_IRUGO, dir, adev, 2182 &amdgpu_ras_debugfs_eeprom_table_ops); 2183 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2184 2185 /* 2186 * After one uncorrectable error happens, usually GPU recovery will 2187 * be scheduled. But due to the known problem in GPU recovery failing 2188 * to bring GPU back, below interface provides one direct way to 2189 * user to reboot system automatically in such case within 2190 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2191 * will never be called. 2192 */ 2193 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2194 2195 /* 2196 * User could set this not to clean up hardware's error count register 2197 * of RAS IPs during ras recovery. 2198 */ 2199 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2200 &con->disable_ras_err_cnt_harvest); 2201 return dir; 2202 } 2203 2204 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2205 struct ras_fs_if *head, 2206 struct dentry *dir) 2207 { 2208 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2209 2210 if (!obj || !dir) 2211 return; 2212 2213 get_obj(obj); 2214 2215 memcpy(obj->fs_data.debugfs_name, 2216 head->debugfs_name, 2217 sizeof(obj->fs_data.debugfs_name)); 2218 2219 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2220 obj, &amdgpu_ras_debugfs_ops); 2221 } 2222 2223 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2224 { 2225 bool ret; 2226 2227 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2228 case IP_VERSION(13, 0, 6): 2229 case IP_VERSION(13, 0, 12): 2230 case IP_VERSION(13, 0, 14): 2231 ret = true; 2232 break; 2233 default: 2234 ret = false; 2235 break; 2236 } 2237 2238 return ret; 2239 } 2240 2241 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2242 { 2243 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2244 struct dentry *dir; 2245 struct ras_manager *obj; 2246 struct ras_fs_if fs_info; 2247 2248 /* 2249 * it won't be called in resume path, no need to check 2250 * suspend and gpu reset status 2251 */ 2252 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2253 return; 2254 2255 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2256 2257 list_for_each_entry(obj, &con->head, node) { 2258 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2259 (obj->attr_inuse == 1)) { 2260 sprintf(fs_info.debugfs_name, "%s_err_inject", 2261 get_ras_block_str(&obj->head)); 2262 fs_info.head = obj->head; 2263 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2264 } 2265 } 2266 2267 if (amdgpu_ras_aca_is_supported(adev)) { 2268 if (amdgpu_aca_is_enabled(adev)) 2269 amdgpu_aca_smu_debugfs_init(adev, dir); 2270 else 2271 amdgpu_mca_smu_debugfs_init(adev, dir); 2272 } 2273 } 2274 2275 /* debugfs end */ 2276 2277 /* ras fs */ 2278 static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2279 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2280 static DEVICE_ATTR(features, S_IRUGO, 2281 amdgpu_ras_sysfs_features_read, NULL); 2282 static DEVICE_ATTR(version, 0444, 2283 amdgpu_ras_sysfs_version_show, NULL); 2284 static DEVICE_ATTR(schema, 0444, 2285 amdgpu_ras_sysfs_schema_show, NULL); 2286 static DEVICE_ATTR(event_state, 0444, 2287 amdgpu_ras_sysfs_event_state_show, NULL); 2288 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2289 { 2290 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2291 struct attribute_group group = { 2292 .name = RAS_FS_NAME, 2293 }; 2294 struct attribute *attrs[] = { 2295 &con->features_attr.attr, 2296 &con->version_attr.attr, 2297 &con->schema_attr.attr, 2298 &con->event_state_attr.attr, 2299 NULL 2300 }; 2301 const struct bin_attribute *bin_attrs[] = { 2302 NULL, 2303 NULL, 2304 }; 2305 int r; 2306 2307 group.attrs = attrs; 2308 2309 /* add features entry */ 2310 con->features_attr = dev_attr_features; 2311 sysfs_attr_init(attrs[0]); 2312 2313 /* add version entry */ 2314 con->version_attr = dev_attr_version; 2315 sysfs_attr_init(attrs[1]); 2316 2317 /* add schema entry */ 2318 con->schema_attr = dev_attr_schema; 2319 sysfs_attr_init(attrs[2]); 2320 2321 /* add event_state entry */ 2322 con->event_state_attr = dev_attr_event_state; 2323 sysfs_attr_init(attrs[3]); 2324 2325 if (amdgpu_bad_page_threshold != 0) { 2326 /* add bad_page_features entry */ 2327 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2328 sysfs_bin_attr_init(&con->badpages_attr); 2329 bin_attrs[0] = &con->badpages_attr; 2330 group.bin_attrs = bin_attrs; 2331 } 2332 2333 r = sysfs_create_group(&adev->dev->kobj, &group); 2334 if (r) 2335 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2336 2337 return 0; 2338 } 2339 2340 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2341 { 2342 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2343 struct ras_manager *con_obj, *ip_obj, *tmp; 2344 2345 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2346 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2347 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2348 if (ip_obj) 2349 put_obj(ip_obj); 2350 } 2351 } 2352 2353 amdgpu_ras_sysfs_remove_all(adev); 2354 return 0; 2355 } 2356 /* ras fs end */ 2357 2358 /* ih begin */ 2359 2360 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2361 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2362 * register to check whether the interrupt is triggered or not, and properly 2363 * ack the interrupt if it is there 2364 */ 2365 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2366 { 2367 /* Fatal error events are handled on host side */ 2368 if (amdgpu_sriov_vf(adev)) 2369 return; 2370 /* 2371 * If the current interrupt is caused by a non-fatal RAS error, skip 2372 * check for fatal error. For fatal errors, FED status of all devices 2373 * in XGMI hive gets set when the first device gets fatal error 2374 * interrupt. The error gets propagated to other devices as well, so 2375 * make sure to ack the interrupt regardless of FED status. 2376 */ 2377 if (!amdgpu_ras_get_fed_status(adev) && 2378 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2379 return; 2380 2381 if (amdgpu_uniras_enabled(adev)) { 2382 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2383 return; 2384 } 2385 2386 if (adev->nbio.ras && 2387 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2388 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2389 2390 if (adev->nbio.ras && 2391 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2392 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2393 } 2394 2395 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2396 struct amdgpu_iv_entry *entry) 2397 { 2398 bool poison_stat = false; 2399 struct amdgpu_device *adev = obj->adev; 2400 struct amdgpu_ras_block_object *block_obj = 2401 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2402 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2403 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2404 u64 event_id; 2405 int ret; 2406 2407 if (!block_obj || !con) 2408 return; 2409 2410 ret = amdgpu_ras_mark_ras_event(adev, type); 2411 if (ret) 2412 return; 2413 2414 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2415 /* both query_poison_status and handle_poison_consumption are optional, 2416 * but at least one of them should be implemented if we need poison 2417 * consumption handler 2418 */ 2419 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2420 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2421 if (!poison_stat) { 2422 /* Not poison consumption interrupt, no need to handle it */ 2423 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2424 block_obj->ras_comm.name); 2425 2426 return; 2427 } 2428 } 2429 2430 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2431 2432 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2433 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2434 2435 /* gpu reset is fallback for failed and default cases. 2436 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2437 */ 2438 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2439 event_id = amdgpu_ras_acquire_event_id(adev, type); 2440 RAS_EVENT_LOG(adev, event_id, 2441 "GPU reset for %s RAS poison consumption is issued!\n", 2442 block_obj->ras_comm.name); 2443 amdgpu_ras_reset_gpu(adev); 2444 } 2445 2446 if (!poison_stat) 2447 amdgpu_gfx_poison_consumption_handler(adev, entry); 2448 } 2449 2450 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2451 struct amdgpu_iv_entry *entry) 2452 { 2453 struct amdgpu_device *adev = obj->adev; 2454 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2455 u64 event_id; 2456 int ret; 2457 2458 ret = amdgpu_ras_mark_ras_event(adev, type); 2459 if (ret) 2460 return; 2461 2462 event_id = amdgpu_ras_acquire_event_id(adev, type); 2463 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2464 2465 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2466 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2467 2468 atomic_inc(&con->page_retirement_req_cnt); 2469 atomic_inc(&con->poison_creation_count); 2470 2471 wake_up(&con->page_retirement_wq); 2472 } 2473 } 2474 2475 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2476 struct amdgpu_iv_entry *entry) 2477 { 2478 struct ras_ih_data *data = &obj->ih_data; 2479 struct ras_err_data err_data; 2480 int ret; 2481 2482 if (!data->cb) 2483 return; 2484 2485 ret = amdgpu_ras_error_data_init(&err_data); 2486 if (ret) 2487 return; 2488 2489 /* Let IP handle its data, maybe we need get the output 2490 * from the callback to update the error type/count, etc 2491 */ 2492 amdgpu_ras_set_fed(obj->adev, true); 2493 ret = data->cb(obj->adev, &err_data, entry); 2494 /* ue will trigger an interrupt, and in that case 2495 * we need do a reset to recovery the whole system. 2496 * But leave IP do that recovery, here we just dispatch 2497 * the error. 2498 */ 2499 if (ret == AMDGPU_RAS_SUCCESS) { 2500 /* these counts could be left as 0 if 2501 * some blocks do not count error number 2502 */ 2503 obj->err_data.ue_count += err_data.ue_count; 2504 obj->err_data.ce_count += err_data.ce_count; 2505 obj->err_data.de_count += err_data.de_count; 2506 } 2507 2508 amdgpu_ras_error_data_fini(&err_data); 2509 } 2510 2511 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2512 { 2513 struct ras_ih_data *data = &obj->ih_data; 2514 struct amdgpu_iv_entry entry; 2515 2516 while (data->rptr != data->wptr) { 2517 rmb(); 2518 memcpy(&entry, &data->ring[data->rptr], 2519 data->element_size); 2520 2521 wmb(); 2522 data->rptr = (data->aligned_element_size + 2523 data->rptr) % data->ring_size; 2524 2525 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2526 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2527 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2528 else 2529 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2530 } else { 2531 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2532 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2533 else 2534 dev_warn(obj->adev->dev, 2535 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2536 } 2537 } 2538 } 2539 2540 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2541 { 2542 struct ras_ih_data *data = 2543 container_of(work, struct ras_ih_data, ih_work); 2544 struct ras_manager *obj = 2545 container_of(data, struct ras_manager, ih_data); 2546 2547 amdgpu_ras_interrupt_handler(obj); 2548 } 2549 2550 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2551 struct ras_dispatch_if *info) 2552 { 2553 struct ras_manager *obj; 2554 struct ras_ih_data *data; 2555 2556 if (amdgpu_uniras_enabled(adev)) { 2557 struct ras_ih_info ih_info; 2558 2559 memset(&ih_info, 0, sizeof(ih_info)); 2560 ih_info.block = info->head.block; 2561 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2562 2563 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2564 } 2565 2566 obj = amdgpu_ras_find_obj(adev, &info->head); 2567 if (!obj) 2568 return -EINVAL; 2569 2570 data = &obj->ih_data; 2571 2572 if (data->inuse == 0) 2573 return 0; 2574 2575 /* Might be overflow... */ 2576 memcpy(&data->ring[data->wptr], info->entry, 2577 data->element_size); 2578 2579 wmb(); 2580 data->wptr = (data->aligned_element_size + 2581 data->wptr) % data->ring_size; 2582 2583 schedule_work(&data->ih_work); 2584 2585 return 0; 2586 } 2587 2588 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2589 struct ras_common_if *head) 2590 { 2591 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2592 struct ras_ih_data *data; 2593 2594 if (!obj) 2595 return -EINVAL; 2596 2597 data = &obj->ih_data; 2598 if (data->inuse == 0) 2599 return 0; 2600 2601 cancel_work_sync(&data->ih_work); 2602 2603 kfree(data->ring); 2604 memset(data, 0, sizeof(*data)); 2605 put_obj(obj); 2606 2607 return 0; 2608 } 2609 2610 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2611 struct ras_common_if *head) 2612 { 2613 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2614 struct ras_ih_data *data; 2615 struct amdgpu_ras_block_object *ras_obj; 2616 2617 if (!obj) { 2618 /* in case we registe the IH before enable ras feature */ 2619 obj = amdgpu_ras_create_obj(adev, head); 2620 if (!obj) 2621 return -EINVAL; 2622 } else 2623 get_obj(obj); 2624 2625 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2626 2627 data = &obj->ih_data; 2628 /* add the callback.etc */ 2629 *data = (struct ras_ih_data) { 2630 .inuse = 0, 2631 .cb = ras_obj->ras_cb, 2632 .element_size = sizeof(struct amdgpu_iv_entry), 2633 .rptr = 0, 2634 .wptr = 0, 2635 }; 2636 2637 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2638 2639 data->aligned_element_size = ALIGN(data->element_size, 8); 2640 /* the ring can store 64 iv entries. */ 2641 data->ring_size = 64 * data->aligned_element_size; 2642 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2643 if (!data->ring) { 2644 put_obj(obj); 2645 return -ENOMEM; 2646 } 2647 2648 /* IH is ready */ 2649 data->inuse = 1; 2650 2651 return 0; 2652 } 2653 2654 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2655 { 2656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2657 struct ras_manager *obj, *tmp; 2658 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2660 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2661 } 2662 2663 return 0; 2664 } 2665 /* ih end */ 2666 2667 /* traversal all IPs except NBIO to query error counter */ 2668 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2669 { 2670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2671 struct ras_manager *obj; 2672 2673 if (!adev->ras_enabled || !con) 2674 return; 2675 2676 list_for_each_entry(obj, &con->head, node) { 2677 struct ras_query_if info = { 2678 .head = obj->head, 2679 }; 2680 2681 /* 2682 * PCIE_BIF IP has one different isr by ras controller 2683 * interrupt, the specific ras counter query will be 2684 * done in that isr. So skip such block from common 2685 * sync flood interrupt isr calling. 2686 */ 2687 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2688 continue; 2689 2690 /* 2691 * this is a workaround for aldebaran, skip send msg to 2692 * smu to get ecc_info table due to smu handle get ecc 2693 * info table failed temporarily. 2694 * should be removed until smu fix handle ecc_info table. 2695 */ 2696 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2697 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2698 IP_VERSION(13, 0, 2))) 2699 continue; 2700 2701 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2702 2703 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2704 IP_VERSION(11, 0, 2) && 2705 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 4) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(13, 0, 0)) { 2709 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2710 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2711 } 2712 } 2713 } 2714 2715 /* Parse RdRspStatus and WrRspStatus */ 2716 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2717 struct ras_query_if *info) 2718 { 2719 struct amdgpu_ras_block_object *block_obj; 2720 /* 2721 * Only two block need to query read/write 2722 * RspStatus at current state 2723 */ 2724 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2725 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2726 return; 2727 2728 block_obj = amdgpu_ras_get_ras_block(adev, 2729 info->head.block, 2730 info->head.sub_block_index); 2731 2732 if (!block_obj || !block_obj->hw_ops) { 2733 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2734 get_ras_block_str(&info->head)); 2735 return; 2736 } 2737 2738 if (block_obj->hw_ops->query_ras_error_status) 2739 block_obj->hw_ops->query_ras_error_status(adev); 2740 2741 } 2742 2743 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2744 { 2745 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2746 struct ras_manager *obj; 2747 2748 if (!adev->ras_enabled || !con) 2749 return; 2750 2751 list_for_each_entry(obj, &con->head, node) { 2752 struct ras_query_if info = { 2753 .head = obj->head, 2754 }; 2755 2756 amdgpu_ras_error_status_query(adev, &info); 2757 } 2758 } 2759 2760 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2761 struct ras_badpage *bps, uint32_t count, uint32_t start) 2762 { 2763 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2764 struct ras_err_handler_data *data; 2765 int r = 0; 2766 uint32_t i; 2767 2768 if (!con || !con->eh_data || !bps || !count) 2769 return -EINVAL; 2770 2771 mutex_lock(&con->recovery_lock); 2772 data = con->eh_data; 2773 if (start < data->count) { 2774 for (i = start; i < data->count; i++) { 2775 if (!data->bps[i].ts) 2776 continue; 2777 2778 bps[r].bp = data->bps[i].retired_page; 2779 r++; 2780 if (r >= count) 2781 break; 2782 } 2783 } 2784 mutex_unlock(&con->recovery_lock); 2785 2786 return r; 2787 } 2788 2789 static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2790 struct ras_badpage *bps, uint32_t count, uint32_t start) 2791 { 2792 struct ras_cmd_bad_pages_info_req cmd_input; 2793 struct ras_cmd_bad_pages_info_rsp *output; 2794 uint32_t group, start_group, end_group; 2795 uint32_t pos, pos_in_group; 2796 int r = 0, i; 2797 2798 if (!bps || !count) 2799 return -EINVAL; 2800 2801 output = kmalloc(sizeof(*output), GFP_KERNEL); 2802 if (!output) 2803 return -ENOMEM; 2804 2805 memset(&cmd_input, 0, sizeof(cmd_input)); 2806 2807 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2808 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2809 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2810 2811 pos = start; 2812 for (group = start_group; group < end_group; group++) { 2813 memset(output, 0, sizeof(*output)); 2814 cmd_input.group_index = group; 2815 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2816 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2817 goto out; 2818 2819 if (pos >= output->bp_total_cnt) 2820 goto out; 2821 2822 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2823 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2824 if (!output->records[i].ts) 2825 continue; 2826 2827 bps[r].bp = output->records[i].retired_page; 2828 r++; 2829 if (r >= count) 2830 goto out; 2831 } 2832 } 2833 2834 out: 2835 kfree(output); 2836 return r; 2837 } 2838 2839 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2840 struct amdgpu_hive_info *hive, bool status) 2841 { 2842 struct amdgpu_device *tmp_adev; 2843 2844 if (hive) { 2845 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2846 amdgpu_ras_set_fed(tmp_adev, status); 2847 } else { 2848 amdgpu_ras_set_fed(adev, status); 2849 } 2850 } 2851 2852 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2853 { 2854 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2856 int hive_ras_recovery = 0; 2857 2858 if (hive) { 2859 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2860 amdgpu_put_xgmi_hive(hive); 2861 } 2862 2863 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2864 return true; 2865 2866 return false; 2867 } 2868 2869 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2870 { 2871 if (amdgpu_ras_intr_triggered()) 2872 return RAS_EVENT_TYPE_FATAL; 2873 else 2874 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2875 } 2876 2877 static void amdgpu_ras_do_recovery(struct work_struct *work) 2878 { 2879 struct amdgpu_ras *ras = 2880 container_of(work, struct amdgpu_ras, recovery_work); 2881 struct amdgpu_device *remote_adev = NULL; 2882 struct amdgpu_device *adev = ras->adev; 2883 struct list_head device_list, *device_list_handle = NULL; 2884 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2885 unsigned int error_query_mode; 2886 enum ras_event_type type; 2887 2888 if (hive) { 2889 atomic_set(&hive->ras_recovery, 1); 2890 2891 /* If any device which is part of the hive received RAS fatal 2892 * error interrupt, set fatal error status on all. This 2893 * condition will need a recovery, and flag will be cleared 2894 * as part of recovery. 2895 */ 2896 list_for_each_entry(remote_adev, &hive->device_list, 2897 gmc.xgmi.head) 2898 if (amdgpu_ras_get_fed_status(remote_adev)) { 2899 amdgpu_ras_set_fed_all(adev, hive, true); 2900 break; 2901 } 2902 } 2903 if (!ras->disable_ras_err_cnt_harvest) { 2904 2905 /* Build list of devices to query RAS related errors */ 2906 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2907 device_list_handle = &hive->device_list; 2908 } else { 2909 INIT_LIST_HEAD(&device_list); 2910 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2911 device_list_handle = &device_list; 2912 } 2913 2914 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2915 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2916 /* wait 500ms to ensure pmfw polling mca bank info done */ 2917 msleep(500); 2918 } 2919 } 2920 2921 type = amdgpu_ras_get_fatal_error_event(adev); 2922 list_for_each_entry(remote_adev, 2923 device_list_handle, gmc.xgmi.head) { 2924 if (amdgpu_uniras_enabled(remote_adev)) { 2925 amdgpu_ras_mgr_update_ras_ecc(remote_adev); 2926 } else { 2927 amdgpu_ras_query_err_status(remote_adev); 2928 amdgpu_ras_log_on_err_counter(remote_adev, type); 2929 } 2930 } 2931 2932 } 2933 2934 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2935 struct amdgpu_reset_context reset_context; 2936 memset(&reset_context, 0, sizeof(reset_context)); 2937 2938 reset_context.method = AMD_RESET_METHOD_NONE; 2939 reset_context.reset_req_dev = adev; 2940 reset_context.src = AMDGPU_RESET_SRC_RAS; 2941 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2942 2943 /* Perform full reset in fatal error mode */ 2944 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2945 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2946 else { 2947 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2948 2949 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2950 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2951 reset_context.method = AMD_RESET_METHOD_MODE2; 2952 } 2953 2954 /* Fatal error occurs in poison mode, mode1 reset is used to 2955 * recover gpu. 2956 */ 2957 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2958 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2959 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2960 2961 psp_fatal_error_recovery_quirk(&adev->psp); 2962 } 2963 } 2964 2965 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2966 } 2967 atomic_set(&ras->in_recovery, 0); 2968 if (hive) { 2969 atomic_set(&hive->ras_recovery, 0); 2970 amdgpu_put_xgmi_hive(hive); 2971 } 2972 } 2973 2974 /* alloc/realloc bps array */ 2975 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2976 struct ras_err_handler_data *data, int pages) 2977 { 2978 unsigned int old_space = data->count + data->space_left; 2979 unsigned int new_space = old_space + pages; 2980 unsigned int align_space = ALIGN(new_space, 512); 2981 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2982 2983 if (!bps) { 2984 return -ENOMEM; 2985 } 2986 2987 if (data->bps) { 2988 memcpy(bps, data->bps, 2989 data->count * sizeof(*data->bps)); 2990 kfree(data->bps); 2991 } 2992 2993 data->bps = bps; 2994 data->space_left += align_space - old_space; 2995 return 0; 2996 } 2997 2998 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2999 struct eeprom_table_record *bps, 3000 struct ras_err_data *err_data) 3001 { 3002 struct ta_ras_query_address_input addr_in; 3003 uint32_t socket = 0; 3004 int ret = 0; 3005 3006 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3007 socket = adev->smuio.funcs->get_socket_id(adev); 3008 3009 /* reinit err_data */ 3010 err_data->err_addr_cnt = 0; 3011 err_data->err_addr_len = adev->umc.retire_unit; 3012 3013 memset(&addr_in, 0, sizeof(addr_in)); 3014 addr_in.ma.err_addr = bps->address; 3015 addr_in.ma.socket_id = socket; 3016 addr_in.ma.ch_inst = bps->mem_channel; 3017 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3018 /* tell RAS TA the node instance is not used */ 3019 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3020 } else { 3021 addr_in.ma.umc_inst = bps->mcumc_id; 3022 addr_in.ma.node_inst = bps->cu; 3023 } 3024 3025 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3026 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3027 &addr_in, NULL, false); 3028 3029 return ret; 3030 } 3031 3032 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3033 struct eeprom_table_record *bps, 3034 struct ras_err_data *err_data) 3035 { 3036 struct ta_ras_query_address_input addr_in; 3037 uint32_t die_id, socket = 0; 3038 3039 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3040 socket = adev->smuio.funcs->get_socket_id(adev); 3041 3042 /* although die id is gotten from PA in nps1 mode, the id is 3043 * fitable for any nps mode 3044 */ 3045 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3046 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3047 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3048 else 3049 return -EINVAL; 3050 3051 /* reinit err_data */ 3052 err_data->err_addr_cnt = 0; 3053 err_data->err_addr_len = adev->umc.retire_unit; 3054 3055 memset(&addr_in, 0, sizeof(addr_in)); 3056 addr_in.ma.err_addr = bps->address; 3057 addr_in.ma.ch_inst = bps->mem_channel; 3058 addr_in.ma.umc_inst = bps->mcumc_id; 3059 addr_in.ma.node_inst = die_id; 3060 addr_in.ma.socket_id = socket; 3061 3062 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3063 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3064 &addr_in, NULL, false); 3065 else 3066 return -EINVAL; 3067 } 3068 3069 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3070 struct eeprom_table_record *bps, int count) 3071 { 3072 int j; 3073 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3074 struct ras_err_handler_data *data = con->eh_data; 3075 3076 for (j = 0; j < count; j++) { 3077 if (amdgpu_ras_check_bad_page_unlock(con, 3078 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3079 data->count++; 3080 data->space_left--; 3081 continue; 3082 } 3083 3084 if (!data->space_left && 3085 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3086 return -ENOMEM; 3087 } 3088 3089 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3090 3091 memcpy(&data->bps[data->count], &(bps[j]), 3092 sizeof(struct eeprom_table_record)); 3093 data->count++; 3094 data->space_left--; 3095 con->bad_page_num++; 3096 } 3097 3098 return 0; 3099 } 3100 3101 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3102 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3103 enum amdgpu_memory_partition nps) 3104 { 3105 int i = 0; 3106 enum amdgpu_memory_partition save_nps; 3107 3108 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3109 3110 /*old asics just have pa in eeprom*/ 3111 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3112 memcpy(err_data->err_addr, bps, 3113 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3114 goto out; 3115 } 3116 3117 for (i = 0; i < adev->umc.retire_unit; i++) 3118 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3119 3120 if (save_nps) { 3121 if (save_nps == nps) { 3122 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3123 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3124 return -EINVAL; 3125 for (i = 0; i < adev->umc.retire_unit; i++) { 3126 err_data->err_addr[i].address = bps[0].address; 3127 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3128 err_data->err_addr[i].bank = bps[0].bank; 3129 err_data->err_addr[i].err_type = bps[0].err_type; 3130 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3131 } 3132 } else { 3133 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3134 return -EINVAL; 3135 } 3136 } else { 3137 if (bps[0].address == 0) { 3138 /* for specific old eeprom data, mca address is not stored, 3139 * calc it from pa 3140 */ 3141 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3142 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3143 return -EINVAL; 3144 } 3145 3146 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3147 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3148 memcpy(err_data->err_addr, bps, 3149 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3150 else 3151 return -EOPNOTSUPP; 3152 } 3153 } 3154 3155 out: 3156 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3157 } 3158 3159 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3160 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3161 enum amdgpu_memory_partition nps) 3162 { 3163 int i = 0; 3164 enum amdgpu_memory_partition save_nps; 3165 3166 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3167 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3168 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3169 } else { 3170 /* if pmfw manages eeprom, save_nps is not stored on eeprom, 3171 * we should always convert mca address into physical address, 3172 * make save_nps different from nps 3173 */ 3174 save_nps = nps + 1; 3175 } 3176 3177 if (save_nps == nps) { 3178 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3179 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3180 return -EINVAL; 3181 for (i = 0; i < adev->umc.retire_unit; i++) { 3182 err_data->err_addr[i].address = bps->address; 3183 err_data->err_addr[i].mem_channel = bps->mem_channel; 3184 err_data->err_addr[i].bank = bps->bank; 3185 err_data->err_addr[i].err_type = bps->err_type; 3186 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3187 } 3188 } else { 3189 if (bps->address) { 3190 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3191 return -EINVAL; 3192 } else { 3193 /* for specific old eeprom data, mca address is not stored, 3194 * calc it from pa 3195 */ 3196 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3197 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3198 return -EINVAL; 3199 3200 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3201 return -EOPNOTSUPP; 3202 } 3203 } 3204 3205 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3206 adev->umc.retire_unit); 3207 } 3208 3209 /* it deal with vram only. */ 3210 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3211 struct eeprom_table_record *bps, int pages, bool from_rom) 3212 { 3213 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3214 struct ras_err_data err_data; 3215 struct amdgpu_ras_eeprom_control *control = 3216 &adev->psp.ras_context.ras->eeprom_control; 3217 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3218 int ret = 0; 3219 uint32_t i = 0; 3220 3221 if (!con || !con->eh_data || !bps || pages <= 0) 3222 return 0; 3223 3224 if (from_rom) { 3225 err_data.err_addr = 3226 kcalloc(adev->umc.retire_unit, 3227 sizeof(struct eeprom_table_record), GFP_KERNEL); 3228 if (!err_data.err_addr) { 3229 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3230 return -ENOMEM; 3231 } 3232 3233 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3234 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3235 } 3236 3237 mutex_lock(&con->recovery_lock); 3238 3239 if (from_rom) { 3240 /* there is no pa recs in V3, so skip pa recs processing */ 3241 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3242 !amdgpu_ras_smu_eeprom_supported(adev)) { 3243 for (i = 0; i < pages; i++) { 3244 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3245 if ((bps[i].address == bps[i + 1].address) && 3246 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3247 /* deal with retire_unit records a time */ 3248 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3249 &bps[i], &err_data, nps); 3250 if (ret) 3251 con->bad_page_num -= adev->umc.retire_unit; 3252 i += (adev->umc.retire_unit - 1); 3253 } else { 3254 break; 3255 } 3256 } else { 3257 break; 3258 } 3259 } 3260 } 3261 for (; i < pages; i++) { 3262 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3263 &bps[i], &err_data, nps); 3264 if (ret) 3265 con->bad_page_num -= adev->umc.retire_unit; 3266 } 3267 3268 con->eh_data->count_saved = con->eh_data->count; 3269 } else { 3270 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3271 } 3272 3273 if (from_rom) 3274 kfree(err_data.err_addr); 3275 mutex_unlock(&con->recovery_lock); 3276 3277 return ret; 3278 } 3279 3280 /* 3281 * write error record array to eeprom, the function should be 3282 * protected by recovery_lock 3283 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3284 */ 3285 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3286 unsigned long *new_cnt) 3287 { 3288 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3289 struct ras_err_handler_data *data; 3290 struct amdgpu_ras_eeprom_control *control; 3291 int save_count, unit_num, i; 3292 3293 if (!con || !con->eh_data) { 3294 if (new_cnt) 3295 *new_cnt = 0; 3296 3297 return 0; 3298 } 3299 3300 if (!con->eeprom_control.is_eeprom_valid) { 3301 dev_warn(adev->dev, 3302 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3303 if (new_cnt) 3304 *new_cnt = 0; 3305 3306 return 0; 3307 } 3308 3309 mutex_lock(&con->recovery_lock); 3310 control = &con->eeprom_control; 3311 data = con->eh_data; 3312 if (amdgpu_ras_smu_eeprom_supported(adev)) 3313 unit_num = control->ras_num_recs - 3314 control->ras_num_recs_old; 3315 else 3316 unit_num = data->count / adev->umc.retire_unit - 3317 control->ras_num_recs; 3318 3319 save_count = con->bad_page_num - control->ras_num_bad_pages; 3320 mutex_unlock(&con->recovery_lock); 3321 3322 if (new_cnt) 3323 *new_cnt = unit_num; 3324 3325 /* only new entries are saved */ 3326 if (unit_num && save_count) { 3327 /*old asics only save pa to eeprom like before*/ 3328 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3329 if (amdgpu_ras_eeprom_append(control, 3330 &data->bps[data->count_saved], unit_num)) { 3331 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3332 return -EIO; 3333 } 3334 } else { 3335 for (i = 0; i < unit_num; i++) { 3336 if (amdgpu_ras_eeprom_append(control, 3337 &data->bps[data->count_saved + 3338 i * adev->umc.retire_unit], 1)) { 3339 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3340 return -EIO; 3341 } 3342 } 3343 } 3344 3345 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3346 data->count_saved = data->count; 3347 } 3348 3349 return 0; 3350 } 3351 3352 /* 3353 * read error record array in eeprom and reserve enough space for 3354 * storing new bad pages 3355 */ 3356 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3357 { 3358 struct amdgpu_ras_eeprom_control *control = 3359 &adev->psp.ras_context.ras->eeprom_control; 3360 struct eeprom_table_record *bps; 3361 int ret, i = 0; 3362 3363 /* no bad page record, skip eeprom access */ 3364 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3365 return 0; 3366 3367 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3368 if (!bps) 3369 return -ENOMEM; 3370 3371 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3372 if (ret) { 3373 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3374 } else { 3375 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3376 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3377 as pa recs, so add verion check to avoid it. 3378 */ 3379 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3380 !amdgpu_ras_smu_eeprom_supported(adev)) { 3381 for (i = 0; i < control->ras_num_recs; i++) { 3382 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3383 if ((bps[i].address == bps[i + 1].address) && 3384 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3385 control->ras_num_pa_recs += adev->umc.retire_unit; 3386 i += (adev->umc.retire_unit - 1); 3387 } else { 3388 control->ras_num_mca_recs += 3389 (control->ras_num_recs - i); 3390 break; 3391 } 3392 } else { 3393 control->ras_num_mca_recs += (control->ras_num_recs - i); 3394 break; 3395 } 3396 } 3397 } else { 3398 control->ras_num_mca_recs = control->ras_num_recs; 3399 } 3400 } 3401 3402 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3403 if (ret) 3404 goto out; 3405 3406 ret = amdgpu_ras_eeprom_check(control); 3407 if (ret) 3408 goto out; 3409 3410 /* HW not usable */ 3411 if (amdgpu_ras_is_rma(adev)) 3412 ret = -EHWPOISON; 3413 } 3414 3415 out: 3416 kfree(bps); 3417 return ret; 3418 } 3419 3420 static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3421 uint64_t addr) 3422 { 3423 struct ras_err_handler_data *data = con->eh_data; 3424 struct amdgpu_device *adev = con->adev; 3425 int i; 3426 3427 if ((addr >= adev->gmc.mc_vram_size && 3428 adev->gmc.mc_vram_size) || 3429 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3430 return -EINVAL; 3431 3432 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3433 for (i = 0; i < data->count; i++) 3434 if (addr == data->bps[i].retired_page) 3435 return 1; 3436 3437 return 0; 3438 } 3439 3440 /* 3441 * check if an address belongs to bad page 3442 * 3443 * Note: this check is only for umc block 3444 */ 3445 static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3446 uint64_t addr) 3447 { 3448 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3449 int ret = 0; 3450 3451 if (!con || !con->eh_data) 3452 return ret; 3453 3454 mutex_lock(&con->recovery_lock); 3455 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3456 mutex_unlock(&con->recovery_lock); 3457 return ret; 3458 } 3459 3460 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3461 uint32_t max_count) 3462 { 3463 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3464 3465 /* 3466 * amdgpu_bad_page_threshold is used to config 3467 * the threshold for the number of bad pages. 3468 * -1: Threshold is set to default value 3469 * Driver will issue a warning message when threshold is reached 3470 * and continue runtime services. 3471 * 0: Disable bad page retirement 3472 * Driver will not retire bad pages 3473 * which is intended for debugging purpose. 3474 * -2: Threshold is determined by a formula 3475 * that assumes 1 bad page per 100M of local memory. 3476 * Driver will continue runtime services when threhold is reached. 3477 * 0 < threshold < max number of bad page records in EEPROM, 3478 * A user-defined threshold is set 3479 * Driver will halt runtime services when this custom threshold is reached. 3480 */ 3481 if (amdgpu_bad_page_threshold == -2) { 3482 u64 val = adev->gmc.mc_vram_size; 3483 3484 do_div(val, RAS_BAD_PAGE_COVER); 3485 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3486 max_count); 3487 } else if (amdgpu_bad_page_threshold == -1) { 3488 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3489 } else { 3490 con->bad_page_cnt_threshold = min_t(int, max_count, 3491 amdgpu_bad_page_threshold); 3492 } 3493 } 3494 3495 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3496 enum amdgpu_ras_block block, uint16_t pasid, 3497 pasid_notify pasid_fn, void *data, uint32_t reset) 3498 { 3499 int ret = 0; 3500 struct ras_poison_msg poison_msg; 3501 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3502 3503 memset(&poison_msg, 0, sizeof(poison_msg)); 3504 poison_msg.block = block; 3505 poison_msg.pasid = pasid; 3506 poison_msg.reset = reset; 3507 poison_msg.pasid_fn = pasid_fn; 3508 poison_msg.data = data; 3509 3510 ret = kfifo_put(&con->poison_fifo, poison_msg); 3511 if (!ret) { 3512 dev_err(adev->dev, "Poison message fifo is full!\n"); 3513 return -ENOSPC; 3514 } 3515 3516 return 0; 3517 } 3518 3519 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3520 struct ras_poison_msg *poison_msg) 3521 { 3522 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3523 3524 return kfifo_get(&con->poison_fifo, poison_msg); 3525 } 3526 3527 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3528 { 3529 mutex_init(&ecc_log->lock); 3530 3531 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3532 ecc_log->de_queried_count = 0; 3533 ecc_log->consumption_q_count = 0; 3534 } 3535 3536 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3537 { 3538 struct radix_tree_iter iter; 3539 void __rcu **slot; 3540 struct ras_ecc_err *ecc_err; 3541 3542 mutex_lock(&ecc_log->lock); 3543 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3544 ecc_err = radix_tree_deref_slot(slot); 3545 kfree(ecc_err->err_pages.pfn); 3546 kfree(ecc_err); 3547 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3548 } 3549 mutex_unlock(&ecc_log->lock); 3550 3551 mutex_destroy(&ecc_log->lock); 3552 ecc_log->de_queried_count = 0; 3553 ecc_log->consumption_q_count = 0; 3554 } 3555 3556 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3557 uint32_t delayed_ms) 3558 { 3559 int ret; 3560 3561 mutex_lock(&con->umc_ecc_log.lock); 3562 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3563 UMC_ECC_NEW_DETECTED_TAG); 3564 mutex_unlock(&con->umc_ecc_log.lock); 3565 3566 if (ret) 3567 schedule_delayed_work(&con->page_retirement_dwork, 3568 msecs_to_jiffies(delayed_ms)); 3569 3570 return ret ? true : false; 3571 } 3572 3573 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3574 { 3575 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3576 page_retirement_dwork.work); 3577 struct amdgpu_device *adev = con->adev; 3578 struct ras_err_data err_data; 3579 3580 /* If gpu reset is ongoing, delay retiring the bad pages */ 3581 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3582 amdgpu_ras_schedule_retirement_dwork(con, 3583 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3584 return; 3585 } 3586 3587 amdgpu_ras_error_data_init(&err_data); 3588 3589 amdgpu_umc_handle_bad_pages(adev, &err_data); 3590 3591 amdgpu_ras_error_data_fini(&err_data); 3592 3593 amdgpu_ras_schedule_retirement_dwork(con, 3594 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3595 } 3596 3597 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3598 uint32_t poison_creation_count) 3599 { 3600 int ret = 0; 3601 struct ras_ecc_log_info *ecc_log; 3602 struct ras_query_if info; 3603 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3604 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3605 u64 de_queried_count; 3606 u64 consumption_q_count; 3607 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3608 3609 memset(&info, 0, sizeof(info)); 3610 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3611 3612 ecc_log = &ras->umc_ecc_log; 3613 ecc_log->de_queried_count = 0; 3614 ecc_log->consumption_q_count = 0; 3615 3616 do { 3617 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3618 if (ret) 3619 return ret; 3620 3621 de_queried_count = ecc_log->de_queried_count; 3622 consumption_q_count = ecc_log->consumption_q_count; 3623 3624 if (de_queried_count && consumption_q_count) 3625 break; 3626 3627 msleep(100); 3628 } while (--timeout); 3629 3630 if (de_queried_count) 3631 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3632 3633 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3634 amdgpu_ras_reset_gpu(adev); 3635 3636 return 0; 3637 } 3638 3639 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3640 { 3641 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3642 struct ras_poison_msg msg; 3643 int ret; 3644 3645 do { 3646 ret = kfifo_get(&con->poison_fifo, &msg); 3647 } while (ret); 3648 } 3649 3650 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3651 uint32_t msg_count, uint32_t *gpu_reset) 3652 { 3653 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3654 uint32_t reset_flags = 0, reset = 0; 3655 struct ras_poison_msg msg; 3656 int ret, i; 3657 3658 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3659 3660 for (i = 0; i < msg_count; i++) { 3661 ret = amdgpu_ras_get_poison_req(adev, &msg); 3662 if (!ret) 3663 continue; 3664 3665 if (msg.pasid_fn) 3666 msg.pasid_fn(adev, msg.pasid, msg.data); 3667 3668 reset_flags |= msg.reset; 3669 } 3670 3671 /* 3672 * Try to ensure poison creation handler is completed first 3673 * to set rma if bad page exceed threshold. 3674 */ 3675 flush_delayed_work(&con->page_retirement_dwork); 3676 3677 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3678 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3679 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3680 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3681 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3682 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3683 else 3684 reset = reset_flags; 3685 3686 con->gpu_reset_flags |= reset; 3687 amdgpu_ras_reset_gpu(adev); 3688 3689 *gpu_reset = reset; 3690 3691 /* Wait for gpu recovery to complete */ 3692 flush_work(&con->recovery_work); 3693 } 3694 3695 return 0; 3696 } 3697 3698 static int amdgpu_ras_page_retirement_thread(void *param) 3699 { 3700 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3701 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3702 uint32_t poison_creation_count, msg_count; 3703 uint32_t gpu_reset; 3704 int ret; 3705 3706 while (!kthread_should_stop()) { 3707 3708 wait_event_interruptible(con->page_retirement_wq, 3709 kthread_should_stop() || 3710 atomic_read(&con->page_retirement_req_cnt)); 3711 3712 if (kthread_should_stop()) 3713 break; 3714 3715 mutex_lock(&con->poison_lock); 3716 gpu_reset = 0; 3717 3718 do { 3719 poison_creation_count = atomic_read(&con->poison_creation_count); 3720 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3721 if (ret == -EIO) 3722 break; 3723 3724 if (poison_creation_count) { 3725 atomic_sub(poison_creation_count, &con->poison_creation_count); 3726 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3727 } 3728 } while (atomic_read(&con->poison_creation_count) && 3729 !atomic_read(&con->poison_consumption_count)); 3730 3731 if (ret != -EIO) { 3732 msg_count = kfifo_len(&con->poison_fifo); 3733 if (msg_count) { 3734 ret = amdgpu_ras_poison_consumption_handler(adev, 3735 msg_count, &gpu_reset); 3736 if ((ret != -EIO) && 3737 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3738 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3739 } 3740 } 3741 3742 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3743 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3744 /* Clear poison creation request */ 3745 atomic_set(&con->poison_creation_count, 0); 3746 atomic_set(&con->poison_consumption_count, 0); 3747 3748 /* Clear poison fifo */ 3749 amdgpu_ras_clear_poison_fifo(adev); 3750 3751 /* Clear all poison requests */ 3752 atomic_set(&con->page_retirement_req_cnt, 0); 3753 3754 if (ret == -EIO) { 3755 /* Wait for mode-1 reset to complete */ 3756 down_read(&adev->reset_domain->sem); 3757 up_read(&adev->reset_domain->sem); 3758 } 3759 3760 /* Wake up work to save bad pages to eeprom */ 3761 schedule_delayed_work(&con->page_retirement_dwork, 0); 3762 } else if (gpu_reset) { 3763 /* gpu just completed mode-2 reset or other reset */ 3764 /* Clear poison consumption messages cached in fifo */ 3765 msg_count = kfifo_len(&con->poison_fifo); 3766 if (msg_count) { 3767 amdgpu_ras_clear_poison_fifo(adev); 3768 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3769 } 3770 3771 atomic_set(&con->poison_consumption_count, 0); 3772 3773 /* Wake up work to save bad pages to eeprom */ 3774 schedule_delayed_work(&con->page_retirement_dwork, 0); 3775 } 3776 mutex_unlock(&con->poison_lock); 3777 } 3778 3779 return 0; 3780 } 3781 3782 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3783 { 3784 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3785 struct amdgpu_ras_eeprom_control *control; 3786 int ret; 3787 3788 if (!con || amdgpu_sriov_vf(adev)) 3789 return 0; 3790 3791 if (amdgpu_uniras_enabled(adev)) 3792 return 0; 3793 3794 control = &con->eeprom_control; 3795 con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev); 3796 3797 ret = amdgpu_ras_eeprom_init(control); 3798 control->is_eeprom_valid = !ret; 3799 3800 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3801 control->ras_num_pa_recs = control->ras_num_recs; 3802 3803 if (adev->umc.ras && 3804 adev->umc.ras->get_retire_flip_bits) 3805 adev->umc.ras->get_retire_flip_bits(adev); 3806 3807 if (control->ras_num_recs && control->is_eeprom_valid) { 3808 ret = amdgpu_ras_load_bad_pages(adev); 3809 if (ret) { 3810 control->is_eeprom_valid = false; 3811 return 0; 3812 } 3813 3814 amdgpu_dpm_send_hbm_bad_pages_num( 3815 adev, control->ras_num_bad_pages); 3816 3817 if (con->update_channel_flag == true) { 3818 amdgpu_dpm_send_hbm_bad_channel_flag( 3819 adev, control->bad_channel_bitmap); 3820 con->update_channel_flag = false; 3821 } 3822 3823 /* The format action is only applied to new ASICs */ 3824 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3825 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3826 if (!amdgpu_ras_eeprom_reset_table(control)) 3827 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3828 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3829 } 3830 3831 return 0; 3832 } 3833 3834 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3835 { 3836 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3837 struct ras_err_handler_data **data; 3838 u32 max_eeprom_records_count = 0; 3839 int ret; 3840 3841 if (!con || amdgpu_sriov_vf(adev)) 3842 return 0; 3843 3844 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3845 * supports RAS and debugfs is enabled, but when 3846 * adev->ras_enabled is unset, i.e. when "ras_enable" 3847 * module parameter is set to 0. 3848 */ 3849 con->adev = adev; 3850 3851 if (!adev->ras_enabled) 3852 return 0; 3853 3854 data = &con->eh_data; 3855 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3856 if (!*data) { 3857 ret = -ENOMEM; 3858 goto out; 3859 } 3860 3861 mutex_init(&con->recovery_lock); 3862 mutex_init(&con->poison_lock); 3863 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3864 atomic_set(&con->in_recovery, 0); 3865 atomic_set(&con->rma_in_recovery, 0); 3866 con->eeprom_control.bad_channel_bitmap = 0; 3867 3868 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3869 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3870 3871 if (init_bp_info) { 3872 ret = amdgpu_ras_init_badpage_info(adev); 3873 if (ret) 3874 goto free; 3875 } 3876 3877 mutex_init(&con->page_rsv_lock); 3878 INIT_KFIFO(con->poison_fifo); 3879 mutex_init(&con->page_retirement_lock); 3880 init_waitqueue_head(&con->page_retirement_wq); 3881 atomic_set(&con->page_retirement_req_cnt, 0); 3882 atomic_set(&con->poison_creation_count, 0); 3883 atomic_set(&con->poison_consumption_count, 0); 3884 con->page_retirement_thread = 3885 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3886 if (IS_ERR(con->page_retirement_thread)) { 3887 con->page_retirement_thread = NULL; 3888 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3889 } 3890 3891 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3892 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3893 #ifdef CONFIG_X86_MCE_AMD 3894 if ((adev->asic_type == CHIP_ALDEBARAN) && 3895 (adev->gmc.xgmi.connected_to_cpu)) 3896 amdgpu_register_bad_pages_mca_notifier(adev); 3897 #endif 3898 return 0; 3899 3900 free: 3901 kfree((*data)->bps); 3902 kfree(*data); 3903 con->eh_data = NULL; 3904 out: 3905 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3906 3907 /* 3908 * Except error threshold exceeding case, other failure cases in this 3909 * function would not fail amdgpu driver init. 3910 */ 3911 if (!amdgpu_ras_is_rma(adev)) 3912 ret = 0; 3913 else 3914 ret = -EINVAL; 3915 3916 return ret; 3917 } 3918 3919 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3920 { 3921 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3922 struct ras_err_handler_data *data = con->eh_data; 3923 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3924 bool ret; 3925 3926 /* recovery_init failed to init it, fini is useless */ 3927 if (!data) 3928 return 0; 3929 3930 /* Save all cached bad pages to eeprom */ 3931 do { 3932 flush_delayed_work(&con->page_retirement_dwork); 3933 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3934 } while (ret && max_flush_timeout--); 3935 3936 if (con->page_retirement_thread) 3937 kthread_stop(con->page_retirement_thread); 3938 3939 atomic_set(&con->page_retirement_req_cnt, 0); 3940 atomic_set(&con->poison_creation_count, 0); 3941 3942 mutex_destroy(&con->page_rsv_lock); 3943 3944 cancel_work_sync(&con->recovery_work); 3945 3946 cancel_delayed_work_sync(&con->page_retirement_dwork); 3947 3948 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3949 3950 mutex_lock(&con->recovery_lock); 3951 con->eh_data = NULL; 3952 kfree(data->bps); 3953 kfree(data); 3954 mutex_unlock(&con->recovery_lock); 3955 3956 amdgpu_ras_critical_region_init(adev); 3957 3958 return 0; 3959 } 3960 /* recovery end */ 3961 3962 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3963 { 3964 if (amdgpu_sriov_vf(adev)) { 3965 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3966 case IP_VERSION(13, 0, 2): 3967 case IP_VERSION(13, 0, 6): 3968 case IP_VERSION(13, 0, 12): 3969 case IP_VERSION(13, 0, 14): 3970 return true; 3971 default: 3972 return false; 3973 } 3974 } 3975 3976 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3977 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3978 case IP_VERSION(13, 0, 0): 3979 case IP_VERSION(13, 0, 6): 3980 case IP_VERSION(13, 0, 10): 3981 case IP_VERSION(13, 0, 12): 3982 case IP_VERSION(13, 0, 14): 3983 case IP_VERSION(14, 0, 3): 3984 return true; 3985 default: 3986 return false; 3987 } 3988 } 3989 3990 return adev->asic_type == CHIP_VEGA10 || 3991 adev->asic_type == CHIP_VEGA20 || 3992 adev->asic_type == CHIP_ARCTURUS || 3993 adev->asic_type == CHIP_ALDEBARAN || 3994 adev->asic_type == CHIP_SIENNA_CICHLID; 3995 } 3996 3997 /* 3998 * this is workaround for vega20 workstation sku, 3999 * force enable gfx ras, ignore vbios gfx ras flag 4000 * due to GC EDC can not write 4001 */ 4002 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 4003 { 4004 struct atom_context *ctx = adev->mode_info.atom_context; 4005 4006 if (!ctx) 4007 return; 4008 4009 if (strnstr(ctx->vbios_pn, "D16406", 4010 sizeof(ctx->vbios_pn)) || 4011 strnstr(ctx->vbios_pn, "D36002", 4012 sizeof(ctx->vbios_pn))) 4013 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 4014 } 4015 4016 /* Query ras capablity via atomfirmware interface */ 4017 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 4018 { 4019 /* mem_ecc cap */ 4020 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 4021 dev_info(adev->dev, "MEM ECC is active.\n"); 4022 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 4023 1 << AMDGPU_RAS_BLOCK__DF); 4024 } else { 4025 dev_info(adev->dev, "MEM ECC is not presented.\n"); 4026 } 4027 4028 /* sram_ecc cap */ 4029 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4030 dev_info(adev->dev, "SRAM ECC is active.\n"); 4031 if (!amdgpu_sriov_vf(adev)) 4032 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4033 1 << AMDGPU_RAS_BLOCK__DF); 4034 else 4035 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4036 1 << AMDGPU_RAS_BLOCK__SDMA | 4037 1 << AMDGPU_RAS_BLOCK__GFX); 4038 4039 /* 4040 * VCN/JPEG RAS can be supported on both bare metal and 4041 * SRIOV environment 4042 */ 4043 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4044 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4045 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4046 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4047 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4048 1 << AMDGPU_RAS_BLOCK__JPEG); 4049 else 4050 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4051 1 << AMDGPU_RAS_BLOCK__JPEG); 4052 4053 /* 4054 * XGMI RAS is not supported if xgmi num physical nodes 4055 * is zero 4056 */ 4057 if (!adev->gmc.xgmi.num_physical_nodes) 4058 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4059 } else { 4060 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4061 } 4062 } 4063 4064 /* Query poison mode from umc/df IP callbacks */ 4065 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4066 { 4067 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4068 bool df_poison, umc_poison; 4069 4070 /* poison setting is useless on SRIOV guest */ 4071 if (amdgpu_sriov_vf(adev) || !con) 4072 return; 4073 4074 /* Init poison supported flag, the default value is false */ 4075 if (adev->gmc.xgmi.connected_to_cpu || 4076 adev->gmc.is_app_apu) { 4077 /* enabled by default when GPU is connected to CPU */ 4078 con->poison_supported = true; 4079 } else if (adev->df.funcs && 4080 adev->df.funcs->query_ras_poison_mode && 4081 adev->umc.ras && 4082 adev->umc.ras->query_ras_poison_mode) { 4083 df_poison = 4084 adev->df.funcs->query_ras_poison_mode(adev); 4085 umc_poison = 4086 adev->umc.ras->query_ras_poison_mode(adev); 4087 4088 /* Only poison is set in both DF and UMC, we can support it */ 4089 if (df_poison && umc_poison) 4090 con->poison_supported = true; 4091 else if (df_poison != umc_poison) 4092 dev_warn(adev->dev, 4093 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4094 df_poison, umc_poison); 4095 } 4096 } 4097 4098 /* 4099 * check hardware's ras ability which will be saved in hw_supported. 4100 * if hardware does not support ras, we can skip some ras initializtion and 4101 * forbid some ras operations from IP. 4102 * if software itself, say boot parameter, limit the ras ability. We still 4103 * need allow IP do some limited operations, like disable. In such case, 4104 * we have to initialize ras as normal. but need check if operation is 4105 * allowed or not in each function. 4106 */ 4107 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4108 { 4109 adev->ras_hw_enabled = adev->ras_enabled = 0; 4110 4111 if (!amdgpu_ras_asic_supported(adev)) 4112 return; 4113 4114 if (amdgpu_sriov_vf(adev)) { 4115 if (amdgpu_virt_get_ras_capability(adev)) 4116 goto init_ras_enabled_flag; 4117 } 4118 4119 /* query ras capability from psp */ 4120 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4121 goto init_ras_enabled_flag; 4122 4123 /* query ras capablity from bios */ 4124 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4125 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4126 } else { 4127 /* driver only manages a few IP blocks RAS feature 4128 * when GPU is connected cpu through XGMI */ 4129 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4130 1 << AMDGPU_RAS_BLOCK__SDMA | 4131 1 << AMDGPU_RAS_BLOCK__MMHUB); 4132 } 4133 4134 /* apply asic specific settings (vega20 only for now) */ 4135 amdgpu_ras_get_quirks(adev); 4136 4137 /* query poison mode from umc/df ip callback */ 4138 amdgpu_ras_query_poison_mode(adev); 4139 4140 init_ras_enabled_flag: 4141 /* hw_supported needs to be aligned with RAS block mask. */ 4142 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4143 4144 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4145 adev->ras_hw_enabled & amdgpu_ras_mask; 4146 4147 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4148 if (!amdgpu_sriov_vf(adev)) { 4149 adev->aca.is_enabled = 4150 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4151 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4152 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4153 } 4154 4155 /* bad page feature is not applicable to specific app platform */ 4156 if (adev->gmc.is_app_apu && 4157 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4158 amdgpu_bad_page_threshold = 0; 4159 } 4160 4161 static void amdgpu_ras_counte_dw(struct work_struct *work) 4162 { 4163 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4164 ras_counte_delay_work.work); 4165 struct amdgpu_device *adev = con->adev; 4166 struct drm_device *dev = adev_to_drm(adev); 4167 unsigned long ce_count, ue_count; 4168 int res; 4169 4170 res = pm_runtime_get_sync(dev->dev); 4171 if (res < 0) 4172 goto Out; 4173 4174 /* Cache new values. 4175 */ 4176 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4177 atomic_set(&con->ras_ce_count, ce_count); 4178 atomic_set(&con->ras_ue_count, ue_count); 4179 } 4180 4181 Out: 4182 pm_runtime_put_autosuspend(dev->dev); 4183 } 4184 4185 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4186 { 4187 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4188 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4189 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4190 AMDGPU_RAS_ERROR__PARITY; 4191 } 4192 4193 static void ras_event_mgr_init(struct ras_event_manager *mgr) 4194 { 4195 struct ras_event_state *event_state; 4196 int i; 4197 4198 memset(mgr, 0, sizeof(*mgr)); 4199 atomic64_set(&mgr->seqno, 0); 4200 4201 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4202 event_state = &mgr->event_state[i]; 4203 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4204 atomic64_set(&event_state->count, 0); 4205 } 4206 } 4207 4208 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4209 { 4210 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4211 struct amdgpu_hive_info *hive; 4212 4213 if (!ras) 4214 return; 4215 4216 hive = amdgpu_get_xgmi_hive(adev); 4217 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4218 4219 /* init event manager with node 0 on xgmi system */ 4220 if (!amdgpu_reset_in_recovery(adev)) { 4221 if (!hive || adev->gmc.xgmi.node_id == 0) 4222 ras_event_mgr_init(ras->event_mgr); 4223 } 4224 4225 if (hive) 4226 amdgpu_put_xgmi_hive(hive); 4227 } 4228 4229 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4230 { 4231 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4232 4233 if (!con || (adev->flags & AMD_IS_APU)) 4234 return; 4235 4236 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4237 case IP_VERSION(13, 0, 2): 4238 case IP_VERSION(13, 0, 6): 4239 case IP_VERSION(13, 0, 12): 4240 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4241 break; 4242 case IP_VERSION(13, 0, 14): 4243 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4244 break; 4245 default: 4246 break; 4247 } 4248 } 4249 4250 int amdgpu_ras_init(struct amdgpu_device *adev) 4251 { 4252 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4253 int r; 4254 4255 if (con) 4256 return 0; 4257 4258 con = kzalloc(sizeof(*con) + 4259 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4260 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4261 GFP_KERNEL); 4262 if (!con) 4263 return -ENOMEM; 4264 4265 con->adev = adev; 4266 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4267 atomic_set(&con->ras_ce_count, 0); 4268 atomic_set(&con->ras_ue_count, 0); 4269 4270 con->objs = (struct ras_manager *)(con + 1); 4271 4272 amdgpu_ras_set_context(adev, con); 4273 4274 amdgpu_ras_check_supported(adev); 4275 4276 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4277 /* set gfx block ras context feature for VEGA20 Gaming 4278 * send ras disable cmd to ras ta during ras late init. 4279 */ 4280 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4281 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4282 4283 return 0; 4284 } 4285 4286 r = 0; 4287 goto release_con; 4288 } 4289 4290 con->update_channel_flag = false; 4291 con->features = 0; 4292 con->schema = 0; 4293 INIT_LIST_HEAD(&con->head); 4294 /* Might need get this flag from vbios. */ 4295 con->flags = RAS_DEFAULT_FLAGS; 4296 4297 /* initialize nbio ras function ahead of any other 4298 * ras functions so hardware fatal error interrupt 4299 * can be enabled as early as possible */ 4300 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4301 case IP_VERSION(7, 4, 0): 4302 case IP_VERSION(7, 4, 1): 4303 case IP_VERSION(7, 4, 4): 4304 if (!adev->gmc.xgmi.connected_to_cpu) 4305 adev->nbio.ras = &nbio_v7_4_ras; 4306 break; 4307 case IP_VERSION(4, 3, 0): 4308 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4309 /* unlike other generation of nbio ras, 4310 * nbio v4_3 only support fatal error interrupt 4311 * to inform software that DF is freezed due to 4312 * system fatal error event. driver should not 4313 * enable nbio ras in such case. Instead, 4314 * check DF RAS */ 4315 adev->nbio.ras = &nbio_v4_3_ras; 4316 break; 4317 case IP_VERSION(6, 3, 1): 4318 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4319 /* unlike other generation of nbio ras, 4320 * nbif v6_3_1 only support fatal error interrupt 4321 * to inform software that DF is freezed due to 4322 * system fatal error event. driver should not 4323 * enable nbio ras in such case. Instead, 4324 * check DF RAS 4325 */ 4326 adev->nbio.ras = &nbif_v6_3_1_ras; 4327 break; 4328 case IP_VERSION(7, 9, 0): 4329 case IP_VERSION(7, 9, 1): 4330 if (!adev->gmc.is_app_apu) 4331 adev->nbio.ras = &nbio_v7_9_ras; 4332 break; 4333 default: 4334 /* nbio ras is not available */ 4335 break; 4336 } 4337 4338 /* nbio ras block needs to be enabled ahead of other ras blocks 4339 * to handle fatal error */ 4340 r = amdgpu_nbio_ras_sw_init(adev); 4341 if (r) 4342 return r; 4343 4344 if (adev->nbio.ras && 4345 adev->nbio.ras->init_ras_controller_interrupt) { 4346 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4347 if (r) 4348 goto release_con; 4349 } 4350 4351 if (adev->nbio.ras && 4352 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4353 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4354 if (r) 4355 goto release_con; 4356 } 4357 4358 /* Packed socket_id to ras feature mask bits[31:29] */ 4359 if (adev->smuio.funcs && 4360 adev->smuio.funcs->get_socket_id) 4361 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4362 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4363 4364 /* Get RAS schema for particular SOC */ 4365 con->schema = amdgpu_get_ras_schema(adev); 4366 4367 amdgpu_ras_init_reserved_vram_size(adev); 4368 4369 if (amdgpu_ras_fs_init(adev)) { 4370 r = -EINVAL; 4371 goto release_con; 4372 } 4373 4374 if (amdgpu_ras_aca_is_supported(adev)) { 4375 if (amdgpu_aca_is_enabled(adev)) 4376 r = amdgpu_aca_init(adev); 4377 else 4378 r = amdgpu_mca_init(adev); 4379 if (r) 4380 goto release_con; 4381 } 4382 4383 con->init_task_pid = task_pid_nr(current); 4384 get_task_comm(con->init_task_comm, current); 4385 4386 mutex_init(&con->critical_region_lock); 4387 INIT_LIST_HEAD(&con->critical_region_head); 4388 4389 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4390 "hardware ability[%x] ras_mask[%x]\n", 4391 adev->ras_hw_enabled, adev->ras_enabled); 4392 4393 return 0; 4394 release_con: 4395 amdgpu_ras_set_context(adev, NULL); 4396 kfree(con); 4397 4398 return r; 4399 } 4400 4401 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4402 { 4403 if (adev->gmc.xgmi.connected_to_cpu || 4404 adev->gmc.is_app_apu) 4405 return 1; 4406 return 0; 4407 } 4408 4409 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4410 struct ras_common_if *ras_block) 4411 { 4412 struct ras_query_if info = { 4413 .head = *ras_block, 4414 }; 4415 4416 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4417 return 0; 4418 4419 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4420 DRM_WARN("RAS init harvest failure"); 4421 4422 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4423 DRM_WARN("RAS init harvest reset failure"); 4424 4425 return 0; 4426 } 4427 4428 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4429 { 4430 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4431 4432 if (!con) 4433 return false; 4434 4435 return con->poison_supported; 4436 } 4437 4438 /* helper function to handle common stuff in ip late init phase */ 4439 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4440 struct ras_common_if *ras_block) 4441 { 4442 struct amdgpu_ras_block_object *ras_obj = NULL; 4443 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4444 struct ras_query_if *query_info; 4445 unsigned long ue_count, ce_count; 4446 int r; 4447 4448 /* disable RAS feature per IP block if it is not supported */ 4449 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4450 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4451 return 0; 4452 } 4453 4454 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4455 if (r) { 4456 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4457 /* in resume phase, if fail to enable ras, 4458 * clean up all ras fs nodes, and disable ras */ 4459 goto cleanup; 4460 } else 4461 return r; 4462 } 4463 4464 /* check for errors on warm reset edc persisant supported ASIC */ 4465 amdgpu_persistent_edc_harvesting(adev, ras_block); 4466 4467 /* in resume phase, no need to create ras fs node */ 4468 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4469 return 0; 4470 4471 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4472 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4473 (ras_obj->hw_ops->query_poison_status || 4474 ras_obj->hw_ops->handle_poison_consumption))) { 4475 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4476 if (r) 4477 goto cleanup; 4478 } 4479 4480 if (ras_obj->hw_ops && 4481 (ras_obj->hw_ops->query_ras_error_count || 4482 ras_obj->hw_ops->query_ras_error_status)) { 4483 r = amdgpu_ras_sysfs_create(adev, ras_block); 4484 if (r) 4485 goto interrupt; 4486 4487 /* Those are the cached values at init. 4488 */ 4489 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4490 if (!query_info) 4491 return -ENOMEM; 4492 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4493 4494 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4495 atomic_set(&con->ras_ce_count, ce_count); 4496 atomic_set(&con->ras_ue_count, ue_count); 4497 } 4498 4499 kfree(query_info); 4500 } 4501 4502 return 0; 4503 4504 interrupt: 4505 if (ras_obj->ras_cb) 4506 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4507 cleanup: 4508 amdgpu_ras_feature_enable(adev, ras_block, 0); 4509 return r; 4510 } 4511 4512 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4513 struct ras_common_if *ras_block) 4514 { 4515 return amdgpu_ras_block_late_init(adev, ras_block); 4516 } 4517 4518 /* helper function to remove ras fs node and interrupt handler */ 4519 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4520 struct ras_common_if *ras_block) 4521 { 4522 struct amdgpu_ras_block_object *ras_obj; 4523 if (!ras_block) 4524 return; 4525 4526 amdgpu_ras_sysfs_remove(adev, ras_block); 4527 4528 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4529 if (ras_obj->ras_cb) 4530 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4531 } 4532 4533 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4534 struct ras_common_if *ras_block) 4535 { 4536 return amdgpu_ras_block_late_fini(adev, ras_block); 4537 } 4538 4539 /* do some init work after IP late init as dependence. 4540 * and it runs in resume/gpu reset/booting up cases. 4541 */ 4542 void amdgpu_ras_resume(struct amdgpu_device *adev) 4543 { 4544 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4545 struct ras_manager *obj, *tmp; 4546 4547 if (!adev->ras_enabled || !con) { 4548 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4549 amdgpu_release_ras_context(adev); 4550 4551 return; 4552 } 4553 4554 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4555 /* Set up all other IPs which are not implemented. There is a 4556 * tricky thing that IP's actual ras error type should be 4557 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4558 * ERROR_NONE make sense anyway. 4559 */ 4560 amdgpu_ras_enable_all_features(adev, 1); 4561 4562 /* We enable ras on all hw_supported block, but as boot 4563 * parameter might disable some of them and one or more IP has 4564 * not implemented yet. So we disable them on behalf. 4565 */ 4566 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4567 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4568 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4569 /* there should be no any reference. */ 4570 WARN_ON(alive_obj(obj)); 4571 } 4572 } 4573 } 4574 } 4575 4576 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4577 { 4578 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4579 4580 if (!adev->ras_enabled || !con) 4581 return; 4582 4583 amdgpu_ras_disable_all_features(adev, 0); 4584 /* Make sure all ras objects are disabled. */ 4585 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4586 amdgpu_ras_disable_all_features(adev, 1); 4587 } 4588 4589 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4590 { 4591 struct amdgpu_ras_block_list *node, *tmp; 4592 struct amdgpu_ras_block_object *obj; 4593 int r; 4594 4595 amdgpu_ras_event_mgr_init(adev); 4596 4597 if (amdgpu_ras_aca_is_supported(adev)) { 4598 if (amdgpu_reset_in_recovery(adev)) { 4599 if (amdgpu_aca_is_enabled(adev)) 4600 r = amdgpu_aca_reset(adev); 4601 else 4602 r = amdgpu_mca_reset(adev); 4603 if (r) 4604 return r; 4605 } 4606 4607 if (!amdgpu_sriov_vf(adev)) { 4608 if (amdgpu_aca_is_enabled(adev)) 4609 amdgpu_ras_set_aca_debug_mode(adev, false); 4610 else 4611 amdgpu_ras_set_mca_debug_mode(adev, false); 4612 } 4613 } 4614 4615 /* Guest side doesn't need init ras feature */ 4616 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4617 return 0; 4618 4619 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4620 obj = node->ras_obj; 4621 if (!obj) { 4622 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4623 continue; 4624 } 4625 4626 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4627 continue; 4628 4629 if (obj->ras_late_init) { 4630 r = obj->ras_late_init(adev, &obj->ras_comm); 4631 if (r) { 4632 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4633 obj->ras_comm.name, r); 4634 return r; 4635 } 4636 } else 4637 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4638 } 4639 4640 return 0; 4641 } 4642 4643 /* do some fini work before IP fini as dependence */ 4644 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4645 { 4646 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4647 4648 if (!adev->ras_enabled || !con) 4649 return 0; 4650 4651 4652 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4653 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4654 amdgpu_ras_disable_all_features(adev, 0); 4655 amdgpu_ras_recovery_fini(adev); 4656 return 0; 4657 } 4658 4659 int amdgpu_ras_fini(struct amdgpu_device *adev) 4660 { 4661 struct amdgpu_ras_block_list *ras_node, *tmp; 4662 struct amdgpu_ras_block_object *obj = NULL; 4663 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4664 4665 if (!adev->ras_enabled || !con) 4666 return 0; 4667 4668 amdgpu_ras_critical_region_fini(adev); 4669 mutex_destroy(&con->critical_region_lock); 4670 4671 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4672 if (ras_node->ras_obj) { 4673 obj = ras_node->ras_obj; 4674 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4675 obj->ras_fini) 4676 obj->ras_fini(adev, &obj->ras_comm); 4677 else 4678 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4679 } 4680 4681 /* Clear ras blocks from ras_list and free ras block list node */ 4682 list_del(&ras_node->node); 4683 kfree(ras_node); 4684 } 4685 4686 amdgpu_ras_fs_fini(adev); 4687 amdgpu_ras_interrupt_remove_all(adev); 4688 4689 if (amdgpu_ras_aca_is_supported(adev)) { 4690 if (amdgpu_aca_is_enabled(adev)) 4691 amdgpu_aca_fini(adev); 4692 else 4693 amdgpu_mca_fini(adev); 4694 } 4695 4696 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4697 4698 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4699 amdgpu_ras_disable_all_features(adev, 0); 4700 4701 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4702 4703 amdgpu_ras_set_context(adev, NULL); 4704 kfree(con); 4705 4706 return 0; 4707 } 4708 4709 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4710 { 4711 struct amdgpu_ras *ras; 4712 4713 ras = amdgpu_ras_get_context(adev); 4714 if (!ras) 4715 return false; 4716 4717 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4718 } 4719 4720 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4721 { 4722 struct amdgpu_ras *ras; 4723 4724 ras = amdgpu_ras_get_context(adev); 4725 if (ras) { 4726 if (status) 4727 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4728 else 4729 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4730 } 4731 } 4732 4733 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4734 { 4735 struct amdgpu_ras *ras; 4736 4737 ras = amdgpu_ras_get_context(adev); 4738 if (ras) { 4739 ras->ras_err_state = 0; 4740 ras->gpu_reset_flags = 0; 4741 } 4742 } 4743 4744 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4745 enum amdgpu_ras_block block) 4746 { 4747 struct amdgpu_ras *ras; 4748 4749 ras = amdgpu_ras_get_context(adev); 4750 if (ras) 4751 set_bit(block, &ras->ras_err_state); 4752 } 4753 4754 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4755 { 4756 struct amdgpu_ras *ras; 4757 4758 ras = amdgpu_ras_get_context(adev); 4759 if (ras) { 4760 if (block == AMDGPU_RAS_BLOCK__ANY) 4761 return (ras->ras_err_state != 0); 4762 else 4763 return test_bit(block, &ras->ras_err_state) || 4764 test_bit(AMDGPU_RAS_BLOCK__LAST, 4765 &ras->ras_err_state); 4766 } 4767 4768 return false; 4769 } 4770 4771 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4772 { 4773 struct amdgpu_ras *ras; 4774 4775 ras = amdgpu_ras_get_context(adev); 4776 if (!ras) 4777 return NULL; 4778 4779 return ras->event_mgr; 4780 } 4781 4782 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4783 const void *caller) 4784 { 4785 struct ras_event_manager *event_mgr; 4786 struct ras_event_state *event_state; 4787 int ret = 0; 4788 4789 if (amdgpu_uniras_enabled(adev)) 4790 return 0; 4791 4792 if (type >= RAS_EVENT_TYPE_COUNT) { 4793 ret = -EINVAL; 4794 goto out; 4795 } 4796 4797 event_mgr = __get_ras_event_mgr(adev); 4798 if (!event_mgr) { 4799 ret = -EINVAL; 4800 goto out; 4801 } 4802 4803 event_state = &event_mgr->event_state[type]; 4804 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4805 atomic64_inc(&event_state->count); 4806 4807 out: 4808 if (ret && caller) 4809 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4810 (int)type, caller, ret); 4811 4812 return ret; 4813 } 4814 4815 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4816 { 4817 struct ras_event_manager *event_mgr; 4818 u64 id; 4819 4820 if (type >= RAS_EVENT_TYPE_COUNT) 4821 return RAS_EVENT_INVALID_ID; 4822 4823 switch (type) { 4824 case RAS_EVENT_TYPE_FATAL: 4825 case RAS_EVENT_TYPE_POISON_CREATION: 4826 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4827 event_mgr = __get_ras_event_mgr(adev); 4828 if (!event_mgr) 4829 return RAS_EVENT_INVALID_ID; 4830 4831 id = event_mgr->event_state[type].last_seqno; 4832 break; 4833 case RAS_EVENT_TYPE_INVALID: 4834 default: 4835 id = RAS_EVENT_INVALID_ID; 4836 break; 4837 } 4838 4839 return id; 4840 } 4841 4842 int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4843 { 4844 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4845 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4846 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4847 u64 event_id = RAS_EVENT_INVALID_ID; 4848 4849 if (amdgpu_uniras_enabled(adev)) 4850 return 0; 4851 4852 if (!amdgpu_ras_mark_ras_event(adev, type)) 4853 event_id = amdgpu_ras_acquire_event_id(adev, type); 4854 4855 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4856 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4857 4858 amdgpu_ras_set_fed(adev, true); 4859 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4860 amdgpu_ras_reset_gpu(adev); 4861 } 4862 4863 return -EBUSY; 4864 } 4865 4866 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4867 { 4868 if (adev->asic_type == CHIP_VEGA20 && 4869 adev->pm.fw_version <= 0x283400) { 4870 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4871 amdgpu_ras_intr_triggered(); 4872 } 4873 4874 return false; 4875 } 4876 4877 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4878 { 4879 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4880 4881 if (!con) 4882 return; 4883 4884 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4885 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4886 amdgpu_ras_set_context(adev, NULL); 4887 kfree(con); 4888 } 4889 } 4890 4891 #ifdef CONFIG_X86_MCE_AMD 4892 static struct amdgpu_device *find_adev(uint32_t node_id) 4893 { 4894 int i; 4895 struct amdgpu_device *adev = NULL; 4896 4897 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4898 adev = mce_adev_list.devs[i]; 4899 4900 if (adev && adev->gmc.xgmi.connected_to_cpu && 4901 adev->gmc.xgmi.physical_node_id == node_id) 4902 break; 4903 adev = NULL; 4904 } 4905 4906 return adev; 4907 } 4908 4909 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4910 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4911 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4912 #define GPU_ID_OFFSET 8 4913 4914 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4915 unsigned long val, void *data) 4916 { 4917 struct mce *m = (struct mce *)data; 4918 struct amdgpu_device *adev = NULL; 4919 uint32_t gpu_id = 0; 4920 uint32_t umc_inst = 0, ch_inst = 0; 4921 4922 /* 4923 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4924 * and error occurred in DramECC (Extended error code = 0) then only 4925 * process the error, else bail out. 4926 */ 4927 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4928 (XEC(m->status, 0x3f) == 0x0))) 4929 return NOTIFY_DONE; 4930 4931 /* 4932 * If it is correctable error, return. 4933 */ 4934 if (mce_is_correctable(m)) 4935 return NOTIFY_OK; 4936 4937 /* 4938 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4939 */ 4940 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4941 4942 adev = find_adev(gpu_id); 4943 if (!adev) { 4944 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4945 gpu_id); 4946 return NOTIFY_DONE; 4947 } 4948 4949 /* 4950 * If it is uncorrectable error, then find out UMC instance and 4951 * channel index. 4952 */ 4953 umc_inst = GET_UMC_INST(m->ipid); 4954 ch_inst = GET_CHAN_INDEX(m->ipid); 4955 4956 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4957 umc_inst, ch_inst); 4958 4959 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4960 return NOTIFY_OK; 4961 else 4962 return NOTIFY_DONE; 4963 } 4964 4965 static struct notifier_block amdgpu_bad_page_nb = { 4966 .notifier_call = amdgpu_bad_page_notifier, 4967 .priority = MCE_PRIO_UC, 4968 }; 4969 4970 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4971 { 4972 /* 4973 * Add the adev to the mce_adev_list. 4974 * During mode2 reset, amdgpu device is temporarily 4975 * removed from the mgpu_info list which can cause 4976 * page retirement to fail. 4977 * Use this list instead of mgpu_info to find the amdgpu 4978 * device on which the UMC error was reported. 4979 */ 4980 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4981 4982 /* 4983 * Register the x86 notifier only once 4984 * with MCE subsystem. 4985 */ 4986 if (notifier_registered == false) { 4987 mce_register_decode_chain(&amdgpu_bad_page_nb); 4988 notifier_registered = true; 4989 } 4990 } 4991 #endif 4992 4993 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4994 { 4995 if (!adev) 4996 return NULL; 4997 4998 return adev->psp.ras_context.ras; 4999 } 5000 5001 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 5002 { 5003 if (!adev) 5004 return -EINVAL; 5005 5006 adev->psp.ras_context.ras = ras_con; 5007 return 0; 5008 } 5009 5010 /* check if ras is supported on block, say, sdma, gfx */ 5011 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 5012 unsigned int block) 5013 { 5014 int ret = 0; 5015 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5016 5017 if (block >= AMDGPU_RAS_BLOCK_COUNT) 5018 return 0; 5019 5020 ret = ras && (adev->ras_enabled & (1 << block)); 5021 5022 /* For the special asic with mem ecc enabled but sram ecc 5023 * not enabled, even if the ras block is not supported on 5024 * .ras_enabled, if the asic supports poison mode and the 5025 * ras block has ras configuration, it can be considered 5026 * that the ras block supports ras function. 5027 */ 5028 if (!ret && 5029 (block == AMDGPU_RAS_BLOCK__GFX || 5030 block == AMDGPU_RAS_BLOCK__SDMA || 5031 block == AMDGPU_RAS_BLOCK__VCN || 5032 block == AMDGPU_RAS_BLOCK__JPEG) && 5033 (amdgpu_ras_mask & (1 << block)) && 5034 amdgpu_ras_is_poison_mode_supported(adev) && 5035 amdgpu_ras_get_ras_block(adev, block, 0)) 5036 ret = 1; 5037 5038 return ret; 5039 } 5040 5041 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5042 { 5043 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5044 5045 /* mode1 is the only selection for RMA status */ 5046 if (amdgpu_ras_is_rma(adev)) { 5047 ras->gpu_reset_flags = 0; 5048 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5049 } 5050 5051 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5052 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5053 int hive_ras_recovery = 0; 5054 5055 if (hive) { 5056 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5057 amdgpu_put_xgmi_hive(hive); 5058 } 5059 /* In the case of multiple GPUs, after a GPU has started 5060 * resetting all GPUs on hive, other GPUs do not need to 5061 * trigger GPU reset again. 5062 */ 5063 if (!hive_ras_recovery) 5064 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5065 else 5066 atomic_set(&ras->in_recovery, 0); 5067 } else { 5068 flush_work(&ras->recovery_work); 5069 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5070 } 5071 5072 return 0; 5073 } 5074 5075 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5076 { 5077 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5078 int ret = 0; 5079 5080 if (con) { 5081 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5082 if (!ret) 5083 con->is_aca_debug_mode = enable; 5084 } 5085 5086 return ret; 5087 } 5088 5089 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5090 { 5091 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5092 int ret = 0; 5093 5094 if (con) { 5095 if (amdgpu_aca_is_enabled(adev)) 5096 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5097 else 5098 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5099 if (!ret) 5100 con->is_aca_debug_mode = enable; 5101 } 5102 5103 return ret; 5104 } 5105 5106 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5107 { 5108 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5109 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5110 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5111 5112 if (!con) 5113 return false; 5114 5115 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5116 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5117 return con->is_aca_debug_mode; 5118 else 5119 return true; 5120 } 5121 5122 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5123 unsigned int *error_query_mode) 5124 { 5125 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5126 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5127 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5128 5129 if (!con) { 5130 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5131 return false; 5132 } 5133 5134 if (amdgpu_sriov_vf(adev)) { 5135 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5136 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5137 *error_query_mode = 5138 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5139 } else { 5140 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5141 } 5142 5143 return true; 5144 } 5145 5146 /* Register each ip ras block into amdgpu ras */ 5147 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5148 struct amdgpu_ras_block_object *ras_block_obj) 5149 { 5150 struct amdgpu_ras_block_list *ras_node; 5151 if (!adev || !ras_block_obj) 5152 return -EINVAL; 5153 5154 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5155 if (!ras_node) 5156 return -ENOMEM; 5157 5158 INIT_LIST_HEAD(&ras_node->node); 5159 ras_node->ras_obj = ras_block_obj; 5160 list_add_tail(&ras_node->node, &adev->ras_list); 5161 5162 return 0; 5163 } 5164 5165 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5166 { 5167 if (!err_type_name) 5168 return; 5169 5170 switch (err_type) { 5171 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5172 sprintf(err_type_name, "correctable"); 5173 break; 5174 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5175 sprintf(err_type_name, "uncorrectable"); 5176 break; 5177 default: 5178 sprintf(err_type_name, "unknown"); 5179 break; 5180 } 5181 } 5182 5183 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5184 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5185 uint32_t instance, 5186 uint32_t *memory_id) 5187 { 5188 uint32_t err_status_lo_data, err_status_lo_offset; 5189 5190 if (!reg_entry) 5191 return false; 5192 5193 err_status_lo_offset = 5194 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5195 reg_entry->seg_lo, reg_entry->reg_lo); 5196 err_status_lo_data = RREG32(err_status_lo_offset); 5197 5198 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5199 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5200 return false; 5201 5202 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5203 5204 return true; 5205 } 5206 5207 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5208 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5209 uint32_t instance, 5210 unsigned long *err_cnt) 5211 { 5212 uint32_t err_status_hi_data, err_status_hi_offset; 5213 5214 if (!reg_entry) 5215 return false; 5216 5217 err_status_hi_offset = 5218 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5219 reg_entry->seg_hi, reg_entry->reg_hi); 5220 err_status_hi_data = RREG32(err_status_hi_offset); 5221 5222 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5223 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5224 /* keep the check here in case we need to refer to the result later */ 5225 dev_dbg(adev->dev, "Invalid err_info field\n"); 5226 5227 /* read err count */ 5228 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5229 5230 return true; 5231 } 5232 5233 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5234 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5235 uint32_t reg_list_size, 5236 const struct amdgpu_ras_memory_id_entry *mem_list, 5237 uint32_t mem_list_size, 5238 uint32_t instance, 5239 uint32_t err_type, 5240 unsigned long *err_count) 5241 { 5242 uint32_t memory_id; 5243 unsigned long err_cnt; 5244 char err_type_name[16]; 5245 uint32_t i, j; 5246 5247 for (i = 0; i < reg_list_size; i++) { 5248 /* query memory_id from err_status_lo */ 5249 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5250 instance, &memory_id)) 5251 continue; 5252 5253 /* query err_cnt from err_status_hi */ 5254 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5255 instance, &err_cnt) || 5256 !err_cnt) 5257 continue; 5258 5259 *err_count += err_cnt; 5260 5261 /* log the errors */ 5262 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5263 if (!mem_list) { 5264 /* memory_list is not supported */ 5265 dev_info(adev->dev, 5266 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5267 err_cnt, err_type_name, 5268 reg_list[i].block_name, 5269 instance, memory_id); 5270 } else { 5271 for (j = 0; j < mem_list_size; j++) { 5272 if (memory_id == mem_list[j].memory_id) { 5273 dev_info(adev->dev, 5274 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5275 err_cnt, err_type_name, 5276 reg_list[i].block_name, 5277 instance, mem_list[j].name); 5278 break; 5279 } 5280 } 5281 } 5282 } 5283 } 5284 5285 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5286 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5287 uint32_t reg_list_size, 5288 uint32_t instance) 5289 { 5290 uint32_t err_status_lo_offset, err_status_hi_offset; 5291 uint32_t i; 5292 5293 for (i = 0; i < reg_list_size; i++) { 5294 err_status_lo_offset = 5295 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5296 reg_list[i].seg_lo, reg_list[i].reg_lo); 5297 err_status_hi_offset = 5298 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5299 reg_list[i].seg_hi, reg_list[i].reg_hi); 5300 WREG32(err_status_lo_offset, 0); 5301 WREG32(err_status_hi_offset, 0); 5302 } 5303 } 5304 5305 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5306 { 5307 memset(err_data, 0, sizeof(*err_data)); 5308 5309 INIT_LIST_HEAD(&err_data->err_node_list); 5310 5311 return 0; 5312 } 5313 5314 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5315 { 5316 if (!err_node) 5317 return; 5318 5319 list_del(&err_node->node); 5320 kvfree(err_node); 5321 } 5322 5323 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5324 { 5325 struct ras_err_node *err_node, *tmp; 5326 5327 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5328 amdgpu_ras_error_node_release(err_node); 5329 } 5330 5331 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5332 struct amdgpu_smuio_mcm_config_info *mcm_info) 5333 { 5334 struct ras_err_node *err_node; 5335 struct amdgpu_smuio_mcm_config_info *ref_id; 5336 5337 if (!err_data || !mcm_info) 5338 return NULL; 5339 5340 for_each_ras_error(err_node, err_data) { 5341 ref_id = &err_node->err_info.mcm_info; 5342 5343 if (mcm_info->socket_id == ref_id->socket_id && 5344 mcm_info->die_id == ref_id->die_id) 5345 return err_node; 5346 } 5347 5348 return NULL; 5349 } 5350 5351 static struct ras_err_node *amdgpu_ras_error_node_new(void) 5352 { 5353 struct ras_err_node *err_node; 5354 5355 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5356 if (!err_node) 5357 return NULL; 5358 5359 INIT_LIST_HEAD(&err_node->node); 5360 5361 return err_node; 5362 } 5363 5364 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5365 { 5366 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5367 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5368 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5369 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5370 5371 if (unlikely(infoa->socket_id != infob->socket_id)) 5372 return infoa->socket_id - infob->socket_id; 5373 else 5374 return infoa->die_id - infob->die_id; 5375 5376 return 0; 5377 } 5378 5379 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5380 struct amdgpu_smuio_mcm_config_info *mcm_info) 5381 { 5382 struct ras_err_node *err_node; 5383 5384 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5385 if (err_node) 5386 return &err_node->err_info; 5387 5388 err_node = amdgpu_ras_error_node_new(); 5389 if (!err_node) 5390 return NULL; 5391 5392 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5393 5394 err_data->err_list_count++; 5395 list_add_tail(&err_node->node, &err_data->err_node_list); 5396 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5397 5398 return &err_node->err_info; 5399 } 5400 5401 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5402 struct amdgpu_smuio_mcm_config_info *mcm_info, 5403 u64 count) 5404 { 5405 struct ras_err_info *err_info; 5406 5407 if (!err_data || !mcm_info) 5408 return -EINVAL; 5409 5410 if (!count) 5411 return 0; 5412 5413 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5414 if (!err_info) 5415 return -EINVAL; 5416 5417 err_info->ue_count += count; 5418 err_data->ue_count += count; 5419 5420 return 0; 5421 } 5422 5423 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5424 struct amdgpu_smuio_mcm_config_info *mcm_info, 5425 u64 count) 5426 { 5427 struct ras_err_info *err_info; 5428 5429 if (!err_data || !mcm_info) 5430 return -EINVAL; 5431 5432 if (!count) 5433 return 0; 5434 5435 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5436 if (!err_info) 5437 return -EINVAL; 5438 5439 err_info->ce_count += count; 5440 err_data->ce_count += count; 5441 5442 return 0; 5443 } 5444 5445 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5446 struct amdgpu_smuio_mcm_config_info *mcm_info, 5447 u64 count) 5448 { 5449 struct ras_err_info *err_info; 5450 5451 if (!err_data || !mcm_info) 5452 return -EINVAL; 5453 5454 if (!count) 5455 return 0; 5456 5457 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5458 if (!err_info) 5459 return -EINVAL; 5460 5461 err_info->de_count += count; 5462 err_data->de_count += count; 5463 5464 return 0; 5465 } 5466 5467 #define mmMP0_SMN_C2PMSG_92 0x1609C 5468 #define mmMP0_SMN_C2PMSG_126 0x160BE 5469 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5470 u32 instance) 5471 { 5472 u32 socket_id, aid_id, hbm_id; 5473 u32 fw_status; 5474 u32 boot_error; 5475 u64 reg_addr; 5476 5477 /* The pattern for smn addressing in other SOC could be different from 5478 * the one for aqua_vanjaram. We should revisit the code if the pattern 5479 * is changed. In such case, replace the aqua_vanjaram implementation 5480 * with more common helper */ 5481 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5482 aqua_vanjaram_encode_ext_smn_addressing(instance); 5483 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5484 5485 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5486 aqua_vanjaram_encode_ext_smn_addressing(instance); 5487 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5488 5489 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5490 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5491 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5492 5493 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5494 dev_info(adev->dev, 5495 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5496 socket_id, aid_id, hbm_id, fw_status); 5497 5498 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5499 dev_info(adev->dev, 5500 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5501 socket_id, aid_id, fw_status); 5502 5503 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5504 dev_info(adev->dev, 5505 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5506 socket_id, aid_id, fw_status); 5507 5508 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5509 dev_info(adev->dev, 5510 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5511 socket_id, aid_id, fw_status); 5512 5513 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5514 dev_info(adev->dev, 5515 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5516 socket_id, aid_id, fw_status); 5517 5518 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5519 dev_info(adev->dev, 5520 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5521 socket_id, aid_id, fw_status); 5522 5523 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5524 dev_info(adev->dev, 5525 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5526 socket_id, aid_id, hbm_id, fw_status); 5527 5528 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5529 dev_info(adev->dev, 5530 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5531 socket_id, aid_id, hbm_id, fw_status); 5532 5533 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5534 dev_info(adev->dev, 5535 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5536 socket_id, aid_id, fw_status); 5537 5538 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5539 dev_info(adev->dev, 5540 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5541 socket_id, aid_id, fw_status); 5542 } 5543 5544 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5545 u32 instance) 5546 { 5547 u64 reg_addr; 5548 u32 reg_data; 5549 int retry_loop; 5550 5551 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5552 aqua_vanjaram_encode_ext_smn_addressing(instance); 5553 5554 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5555 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5556 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5557 return false; 5558 else 5559 msleep(1); 5560 } 5561 5562 return true; 5563 } 5564 5565 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5566 { 5567 u32 i; 5568 5569 for (i = 0; i < num_instances; i++) { 5570 if (amdgpu_ras_boot_error_detected(adev, i)) 5571 amdgpu_ras_boot_time_error_reporting(adev, i); 5572 } 5573 } 5574 5575 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5576 { 5577 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5578 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5579 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5580 int ret = 0; 5581 5582 if (amdgpu_ras_check_critical_address(adev, start)) 5583 return 0; 5584 5585 mutex_lock(&con->page_rsv_lock); 5586 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5587 if (ret == -ENOENT) 5588 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5589 mutex_unlock(&con->page_rsv_lock); 5590 5591 return ret; 5592 } 5593 5594 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5595 const char *fmt, ...) 5596 { 5597 struct va_format vaf; 5598 va_list args; 5599 5600 va_start(args, fmt); 5601 vaf.fmt = fmt; 5602 vaf.va = &args; 5603 5604 if (RAS_EVENT_ID_IS_VALID(event_id)) 5605 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5606 else 5607 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5608 5609 va_end(args); 5610 } 5611 5612 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5613 { 5614 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5615 5616 if (amdgpu_uniras_enabled(adev)) 5617 return amdgpu_ras_mgr_is_rma(adev); 5618 5619 if (!con) 5620 return false; 5621 5622 return con->is_rma; 5623 } 5624 5625 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5626 struct amdgpu_bo *bo) 5627 { 5628 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5629 struct amdgpu_vram_mgr_resource *vres; 5630 struct ras_critical_region *region; 5631 struct drm_buddy_block *block; 5632 int ret = 0; 5633 5634 if (!bo || !bo->tbo.resource) 5635 return -EINVAL; 5636 5637 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5638 5639 mutex_lock(&con->critical_region_lock); 5640 5641 /* Check if the bo had been recorded */ 5642 list_for_each_entry(region, &con->critical_region_head, node) 5643 if (region->bo == bo) 5644 goto out; 5645 5646 /* Record new critical amdgpu bo */ 5647 list_for_each_entry(block, &vres->blocks, link) { 5648 region = kzalloc(sizeof(*region), GFP_KERNEL); 5649 if (!region) { 5650 ret = -ENOMEM; 5651 goto out; 5652 } 5653 region->bo = bo; 5654 region->start = amdgpu_vram_mgr_block_start(block); 5655 region->size = amdgpu_vram_mgr_block_size(block); 5656 list_add_tail(®ion->node, &con->critical_region_head); 5657 } 5658 5659 out: 5660 mutex_unlock(&con->critical_region_lock); 5661 5662 return ret; 5663 } 5664 5665 static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5666 { 5667 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5668 } 5669 5670 static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5671 { 5672 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5673 struct ras_critical_region *region, *tmp; 5674 5675 mutex_lock(&con->critical_region_lock); 5676 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5677 list_del(®ion->node); 5678 kfree(region); 5679 } 5680 mutex_unlock(&con->critical_region_lock); 5681 } 5682 5683 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5684 { 5685 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5686 struct ras_critical_region *region; 5687 bool ret = false; 5688 5689 mutex_lock(&con->critical_region_lock); 5690 list_for_each_entry(region, &con->critical_region_head, node) { 5691 if ((region->start <= addr) && 5692 (addr < (region->start + region->size))) { 5693 ret = true; 5694 break; 5695 } 5696 } 5697 mutex_unlock(&con->critical_region_lock); 5698 5699 return ret; 5700 } 5701 5702 void amdgpu_ras_pre_reset(struct amdgpu_device *adev, 5703 struct list_head *device_list) 5704 { 5705 struct amdgpu_device *tmp_adev = NULL; 5706 5707 list_for_each_entry(tmp_adev, device_list, reset_list) { 5708 if (amdgpu_uniras_enabled(tmp_adev)) 5709 amdgpu_ras_mgr_pre_reset(tmp_adev); 5710 } 5711 } 5712 5713 void amdgpu_ras_post_reset(struct amdgpu_device *adev, 5714 struct list_head *device_list) 5715 { 5716 struct amdgpu_device *tmp_adev = NULL; 5717 5718 list_for_each_entry(tmp_adev, device_list, reset_list) { 5719 if (amdgpu_uniras_enabled(tmp_adev)) 5720 amdgpu_ras_mgr_post_reset(tmp_adev); 5721 } 5722 } 5723