1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 45 #ifdef CONFIG_X86_MCE_AMD 46 #include <asm/mce.h> 47 48 static bool notifier_registered; 49 #endif 50 static const char *RAS_FS_NAME = "ras"; 51 52 const char *ras_error_string[] = { 53 "none", 54 "parity", 55 "single_correctable", 56 "multi_uncorrectable", 57 "poison", 58 }; 59 60 const char *ras_block_string[] = { 61 "umc", 62 "sdma", 63 "gfx", 64 "mmhub", 65 "athub", 66 "pcie_bif", 67 "hdp", 68 "xgmi_wafl", 69 "df", 70 "smn", 71 "sem", 72 "mp0", 73 "mp1", 74 "fuse", 75 "mca", 76 "vcn", 77 "jpeg", 78 "ih", 79 "mpio", 80 "mmsch", 81 }; 82 83 const char *ras_mca_block_string[] = { 84 "mca_mp0", 85 "mca_mp1", 86 "mca_mpio", 87 "mca_iohc", 88 }; 89 90 struct amdgpu_ras_block_list { 91 /* ras block link */ 92 struct list_head node; 93 94 struct amdgpu_ras_block_object *ras_obj; 95 }; 96 97 const char *get_ras_block_str(struct ras_common_if *ras_block) 98 { 99 if (!ras_block) 100 return "NULL"; 101 102 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 103 ras_block->block >= ARRAY_SIZE(ras_block_string)) 104 return "OUT OF RANGE"; 105 106 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 107 return ras_mca_block_string[ras_block->sub_block_index]; 108 109 return ras_block_string[ras_block->block]; 110 } 111 112 #define ras_block_str(_BLOCK_) \ 113 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 114 115 #define ras_err_str(i) (ras_error_string[ffs(i)]) 116 117 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 118 119 /* inject address is 52 bits */ 120 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 121 122 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 123 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 124 125 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms 126 127 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 128 129 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 130 131 enum amdgpu_ras_retire_page_reservation { 132 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 133 AMDGPU_RAS_RETIRE_PAGE_PENDING, 134 AMDGPU_RAS_RETIRE_PAGE_FAULT, 135 }; 136 137 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 138 139 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 140 uint64_t addr); 141 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 142 uint64_t addr); 143 #ifdef CONFIG_X86_MCE_AMD 144 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 145 struct mce_notifier_adev_list { 146 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 147 int num_gpu; 148 }; 149 static struct mce_notifier_adev_list mce_adev_list; 150 #endif 151 152 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 153 { 154 if (adev && amdgpu_ras_get_context(adev)) 155 amdgpu_ras_get_context(adev)->error_query_ready = ready; 156 } 157 158 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 159 { 160 if (adev && amdgpu_ras_get_context(adev)) 161 return amdgpu_ras_get_context(adev)->error_query_ready; 162 163 return false; 164 } 165 166 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 167 { 168 struct ras_err_data err_data; 169 struct eeprom_table_record err_rec; 170 int ret; 171 172 if ((address >= adev->gmc.mc_vram_size) || 173 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 174 dev_warn(adev->dev, 175 "RAS WARN: input address 0x%llx is invalid.\n", 176 address); 177 return -EINVAL; 178 } 179 180 if (amdgpu_ras_check_bad_page(adev, address)) { 181 dev_warn(adev->dev, 182 "RAS WARN: 0x%llx has already been marked as bad page!\n", 183 address); 184 return 0; 185 } 186 187 ret = amdgpu_ras_error_data_init(&err_data); 188 if (ret) 189 return ret; 190 191 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 192 err_data.err_addr = &err_rec; 193 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 194 195 if (amdgpu_bad_page_threshold != 0) { 196 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 197 err_data.err_addr_cnt, false); 198 amdgpu_ras_save_bad_pages(adev, NULL); 199 } 200 201 amdgpu_ras_error_data_fini(&err_data); 202 203 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 204 dev_warn(adev->dev, "Clear EEPROM:\n"); 205 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 206 207 return 0; 208 } 209 210 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 211 size_t size, loff_t *pos) 212 { 213 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 214 struct ras_query_if info = { 215 .head = obj->head, 216 }; 217 ssize_t s; 218 char val[128]; 219 220 if (amdgpu_ras_query_error_status(obj->adev, &info)) 221 return -EINVAL; 222 223 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 224 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 225 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 226 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 227 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 228 } 229 230 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 231 "ue", info.ue_count, 232 "ce", info.ce_count); 233 if (*pos >= s) 234 return 0; 235 236 s -= *pos; 237 s = min_t(u64, s, size); 238 239 240 if (copy_to_user(buf, &val[*pos], s)) 241 return -EINVAL; 242 243 *pos += s; 244 245 return s; 246 } 247 248 static const struct file_operations amdgpu_ras_debugfs_ops = { 249 .owner = THIS_MODULE, 250 .read = amdgpu_ras_debugfs_read, 251 .write = NULL, 252 .llseek = default_llseek 253 }; 254 255 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 256 { 257 int i; 258 259 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 260 *block_id = i; 261 if (strcmp(name, ras_block_string[i]) == 0) 262 return 0; 263 } 264 return -EINVAL; 265 } 266 267 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 268 const char __user *buf, size_t size, 269 loff_t *pos, struct ras_debug_if *data) 270 { 271 ssize_t s = min_t(u64, 64, size); 272 char str[65]; 273 char block_name[33]; 274 char err[9] = "ue"; 275 int op = -1; 276 int block_id; 277 uint32_t sub_block; 278 u64 address, value; 279 /* default value is 0 if the mask is not set by user */ 280 u32 instance_mask = 0; 281 282 if (*pos) 283 return -EINVAL; 284 *pos = size; 285 286 memset(str, 0, sizeof(str)); 287 memset(data, 0, sizeof(*data)); 288 289 if (copy_from_user(str, buf, s)) 290 return -EINVAL; 291 292 if (sscanf(str, "disable %32s", block_name) == 1) 293 op = 0; 294 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 295 op = 1; 296 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 297 op = 2; 298 else if (strstr(str, "retire_page") != NULL) 299 op = 3; 300 else if (str[0] && str[1] && str[2] && str[3]) 301 /* ascii string, but commands are not matched. */ 302 return -EINVAL; 303 304 if (op != -1) { 305 if (op == 3) { 306 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 307 sscanf(str, "%*s %llu", &address) != 1) 308 return -EINVAL; 309 310 data->op = op; 311 data->inject.address = address; 312 313 return 0; 314 } 315 316 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 317 return -EINVAL; 318 319 data->head.block = block_id; 320 /* only ue, ce and poison errors are supported */ 321 if (!memcmp("ue", err, 2)) 322 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 323 else if (!memcmp("ce", err, 2)) 324 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 325 else if (!memcmp("poison", err, 6)) 326 data->head.type = AMDGPU_RAS_ERROR__POISON; 327 else 328 return -EINVAL; 329 330 data->op = op; 331 332 if (op == 2) { 333 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 334 &sub_block, &address, &value, &instance_mask) != 4 && 335 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 336 &sub_block, &address, &value, &instance_mask) != 4 && 337 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 338 &sub_block, &address, &value) != 3 && 339 sscanf(str, "%*s %*s %*s %u %llu %llu", 340 &sub_block, &address, &value) != 3) 341 return -EINVAL; 342 data->head.sub_block_index = sub_block; 343 data->inject.address = address; 344 data->inject.value = value; 345 data->inject.instance_mask = instance_mask; 346 } 347 } else { 348 if (size < sizeof(*data)) 349 return -EINVAL; 350 351 if (copy_from_user(data, buf, sizeof(*data))) 352 return -EINVAL; 353 } 354 355 return 0; 356 } 357 358 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 359 struct ras_debug_if *data) 360 { 361 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 362 uint32_t mask, inst_mask = data->inject.instance_mask; 363 364 /* no need to set instance mask if there is only one instance */ 365 if (num_xcc <= 1 && inst_mask) { 366 data->inject.instance_mask = 0; 367 dev_dbg(adev->dev, 368 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 369 inst_mask); 370 371 return; 372 } 373 374 switch (data->head.block) { 375 case AMDGPU_RAS_BLOCK__GFX: 376 mask = GENMASK(num_xcc - 1, 0); 377 break; 378 case AMDGPU_RAS_BLOCK__SDMA: 379 mask = GENMASK(adev->sdma.num_instances - 1, 0); 380 break; 381 case AMDGPU_RAS_BLOCK__VCN: 382 case AMDGPU_RAS_BLOCK__JPEG: 383 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 384 break; 385 default: 386 mask = inst_mask; 387 break; 388 } 389 390 /* remove invalid bits in instance mask */ 391 data->inject.instance_mask &= mask; 392 if (inst_mask != data->inject.instance_mask) 393 dev_dbg(adev->dev, 394 "Adjust RAS inject mask 0x%x to 0x%x\n", 395 inst_mask, data->inject.instance_mask); 396 } 397 398 /** 399 * DOC: AMDGPU RAS debugfs control interface 400 * 401 * The control interface accepts struct ras_debug_if which has two members. 402 * 403 * First member: ras_debug_if::head or ras_debug_if::inject. 404 * 405 * head is used to indicate which IP block will be under control. 406 * 407 * head has four members, they are block, type, sub_block_index, name. 408 * block: which IP will be under control. 409 * type: what kind of error will be enabled/disabled/injected. 410 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 411 * name: the name of IP. 412 * 413 * inject has three more members than head, they are address, value and mask. 414 * As their names indicate, inject operation will write the 415 * value to the address. 416 * 417 * The second member: struct ras_debug_if::op. 418 * It has three kinds of operations. 419 * 420 * - 0: disable RAS on the block. Take ::head as its data. 421 * - 1: enable RAS on the block. Take ::head as its data. 422 * - 2: inject errors on the block. Take ::inject as its data. 423 * 424 * How to use the interface? 425 * 426 * In a program 427 * 428 * Copy the struct ras_debug_if in your code and initialize it. 429 * Write the struct to the control interface. 430 * 431 * From shell 432 * 433 * .. code-block:: bash 434 * 435 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 436 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 437 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 438 * 439 * Where N, is the card which you want to affect. 440 * 441 * "disable" requires only the block. 442 * "enable" requires the block and error type. 443 * "inject" requires the block, error type, address, and value. 444 * 445 * The block is one of: umc, sdma, gfx, etc. 446 * see ras_block_string[] for details 447 * 448 * The error type is one of: ue, ce and poison where, 449 * ue is multi-uncorrectable 450 * ce is single-correctable 451 * poison is poison 452 * 453 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 454 * The address and value are hexadecimal numbers, leading 0x is optional. 455 * The mask means instance mask, is optional, default value is 0x1. 456 * 457 * For instance, 458 * 459 * .. code-block:: bash 460 * 461 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 462 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 463 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 464 * 465 * How to check the result of the operation? 466 * 467 * To check disable/enable, see "ras" features at, 468 * /sys/class/drm/card[0/1/2...]/device/ras/features 469 * 470 * To check inject, see the corresponding error count at, 471 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 472 * 473 * .. note:: 474 * Operations are only allowed on blocks which are supported. 475 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 476 * to see which blocks support RAS on a particular asic. 477 * 478 */ 479 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 480 const char __user *buf, 481 size_t size, loff_t *pos) 482 { 483 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 484 struct ras_debug_if data; 485 int ret = 0; 486 487 if (!amdgpu_ras_get_error_query_ready(adev)) { 488 dev_warn(adev->dev, "RAS WARN: error injection " 489 "currently inaccessible\n"); 490 return size; 491 } 492 493 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 494 if (ret) 495 return ret; 496 497 if (data.op == 3) { 498 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 499 if (!ret) 500 return size; 501 else 502 return ret; 503 } 504 505 if (!amdgpu_ras_is_supported(adev, data.head.block)) 506 return -EINVAL; 507 508 switch (data.op) { 509 case 0: 510 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 511 break; 512 case 1: 513 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 514 break; 515 case 2: 516 if ((data.inject.address >= adev->gmc.mc_vram_size && 517 adev->gmc.mc_vram_size) || 518 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 519 dev_warn(adev->dev, "RAS WARN: input address " 520 "0x%llx is invalid.", 521 data.inject.address); 522 ret = -EINVAL; 523 break; 524 } 525 526 /* umc ce/ue error injection for a bad page is not allowed */ 527 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) && 528 amdgpu_ras_check_bad_page(adev, data.inject.address)) { 529 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has " 530 "already been marked as bad!\n", 531 data.inject.address); 532 break; 533 } 534 535 amdgpu_ras_instance_mask_check(adev, &data); 536 537 /* data.inject.address is offset instead of absolute gpu address */ 538 ret = amdgpu_ras_error_inject(adev, &data.inject); 539 break; 540 default: 541 ret = -EINVAL; 542 break; 543 } 544 545 if (ret) 546 return ret; 547 548 return size; 549 } 550 551 /** 552 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 553 * 554 * Some boards contain an EEPROM which is used to persistently store a list of 555 * bad pages which experiences ECC errors in vram. This interface provides 556 * a way to reset the EEPROM, e.g., after testing error injection. 557 * 558 * Usage: 559 * 560 * .. code-block:: bash 561 * 562 * echo 1 > ../ras/ras_eeprom_reset 563 * 564 * will reset EEPROM table to 0 entries. 565 * 566 */ 567 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 568 const char __user *buf, 569 size_t size, loff_t *pos) 570 { 571 struct amdgpu_device *adev = 572 (struct amdgpu_device *)file_inode(f)->i_private; 573 int ret; 574 575 ret = amdgpu_ras_eeprom_reset_table( 576 &(amdgpu_ras_get_context(adev)->eeprom_control)); 577 578 if (!ret) { 579 /* Something was written to EEPROM. 580 */ 581 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 582 return size; 583 } else { 584 return ret; 585 } 586 } 587 588 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 589 .owner = THIS_MODULE, 590 .read = NULL, 591 .write = amdgpu_ras_debugfs_ctrl_write, 592 .llseek = default_llseek 593 }; 594 595 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 596 .owner = THIS_MODULE, 597 .read = NULL, 598 .write = amdgpu_ras_debugfs_eeprom_write, 599 .llseek = default_llseek 600 }; 601 602 /** 603 * DOC: AMDGPU RAS sysfs Error Count Interface 604 * 605 * It allows the user to read the error count for each IP block on the gpu through 606 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 607 * 608 * It outputs the multiple lines which report the uncorrected (ue) and corrected 609 * (ce) error counts. 610 * 611 * The format of one line is below, 612 * 613 * [ce|ue]: count 614 * 615 * Example: 616 * 617 * .. code-block:: bash 618 * 619 * ue: 0 620 * ce: 1 621 * 622 */ 623 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 624 struct device_attribute *attr, char *buf) 625 { 626 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 627 struct ras_query_if info = { 628 .head = obj->head, 629 }; 630 631 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 632 return sysfs_emit(buf, "Query currently inaccessible\n"); 633 634 if (amdgpu_ras_query_error_status(obj->adev, &info)) 635 return -EINVAL; 636 637 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 638 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 639 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 640 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 641 } 642 643 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 644 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 645 "ce", info.ce_count, "de", info.de_count); 646 else 647 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 648 "ce", info.ce_count); 649 } 650 651 /* obj begin */ 652 653 #define get_obj(obj) do { (obj)->use++; } while (0) 654 #define alive_obj(obj) ((obj)->use) 655 656 static inline void put_obj(struct ras_manager *obj) 657 { 658 if (obj && (--obj->use == 0)) { 659 list_del(&obj->node); 660 amdgpu_ras_error_data_fini(&obj->err_data); 661 } 662 663 if (obj && (obj->use < 0)) 664 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 665 } 666 667 /* make one obj and return it. */ 668 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 669 struct ras_common_if *head) 670 { 671 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 672 struct ras_manager *obj; 673 674 if (!adev->ras_enabled || !con) 675 return NULL; 676 677 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 678 return NULL; 679 680 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 681 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 682 return NULL; 683 684 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 685 } else 686 obj = &con->objs[head->block]; 687 688 /* already exist. return obj? */ 689 if (alive_obj(obj)) 690 return NULL; 691 692 if (amdgpu_ras_error_data_init(&obj->err_data)) 693 return NULL; 694 695 obj->head = *head; 696 obj->adev = adev; 697 list_add(&obj->node, &con->head); 698 get_obj(obj); 699 700 return obj; 701 } 702 703 /* return an obj equal to head, or the first when head is NULL */ 704 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 705 struct ras_common_if *head) 706 { 707 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 708 struct ras_manager *obj; 709 int i; 710 711 if (!adev->ras_enabled || !con) 712 return NULL; 713 714 if (head) { 715 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 716 return NULL; 717 718 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 719 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 720 return NULL; 721 722 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 723 } else 724 obj = &con->objs[head->block]; 725 726 if (alive_obj(obj)) 727 return obj; 728 } else { 729 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 730 obj = &con->objs[i]; 731 if (alive_obj(obj)) 732 return obj; 733 } 734 } 735 736 return NULL; 737 } 738 /* obj end */ 739 740 /* feature ctl begin */ 741 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 742 struct ras_common_if *head) 743 { 744 return adev->ras_hw_enabled & BIT(head->block); 745 } 746 747 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 748 struct ras_common_if *head) 749 { 750 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 751 752 return con->features & BIT(head->block); 753 } 754 755 /* 756 * if obj is not created, then create one. 757 * set feature enable flag. 758 */ 759 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 760 struct ras_common_if *head, int enable) 761 { 762 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 763 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 764 765 /* If hardware does not support ras, then do not create obj. 766 * But if hardware support ras, we can create the obj. 767 * Ras framework checks con->hw_supported to see if it need do 768 * corresponding initialization. 769 * IP checks con->support to see if it need disable ras. 770 */ 771 if (!amdgpu_ras_is_feature_allowed(adev, head)) 772 return 0; 773 774 if (enable) { 775 if (!obj) { 776 obj = amdgpu_ras_create_obj(adev, head); 777 if (!obj) 778 return -EINVAL; 779 } else { 780 /* In case we create obj somewhere else */ 781 get_obj(obj); 782 } 783 con->features |= BIT(head->block); 784 } else { 785 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 786 con->features &= ~BIT(head->block); 787 put_obj(obj); 788 } 789 } 790 791 return 0; 792 } 793 794 /* wrapper of psp_ras_enable_features */ 795 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 796 struct ras_common_if *head, bool enable) 797 { 798 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 799 union ta_ras_cmd_input *info; 800 int ret; 801 802 if (!con) 803 return -EINVAL; 804 805 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 806 /* For gfx ip, regardless of feature support status, */ 807 /* Force issue enable or disable ras feature commands */ 808 if (head->block != AMDGPU_RAS_BLOCK__GFX && 809 !amdgpu_ras_is_feature_allowed(adev, head)) 810 return 0; 811 812 /* Only enable gfx ras feature from host side */ 813 if (head->block == AMDGPU_RAS_BLOCK__GFX && 814 !amdgpu_sriov_vf(adev) && 815 !amdgpu_ras_intr_triggered()) { 816 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 817 if (!info) 818 return -ENOMEM; 819 820 if (!enable) { 821 info->disable_features = (struct ta_ras_disable_features_input) { 822 .block_id = amdgpu_ras_block_to_ta(head->block), 823 .error_type = amdgpu_ras_error_to_ta(head->type), 824 }; 825 } else { 826 info->enable_features = (struct ta_ras_enable_features_input) { 827 .block_id = amdgpu_ras_block_to_ta(head->block), 828 .error_type = amdgpu_ras_error_to_ta(head->type), 829 }; 830 } 831 832 ret = psp_ras_enable_features(&adev->psp, info, enable); 833 if (ret) { 834 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 835 enable ? "enable":"disable", 836 get_ras_block_str(head), 837 amdgpu_ras_is_poison_mode_supported(adev), ret); 838 kfree(info); 839 return ret; 840 } 841 842 kfree(info); 843 } 844 845 /* setup the obj */ 846 __amdgpu_ras_feature_enable(adev, head, enable); 847 848 return 0; 849 } 850 851 /* Only used in device probe stage and called only once. */ 852 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 853 struct ras_common_if *head, bool enable) 854 { 855 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 856 int ret; 857 858 if (!con) 859 return -EINVAL; 860 861 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 862 if (enable) { 863 /* There is no harm to issue a ras TA cmd regardless of 864 * the currecnt ras state. 865 * If current state == target state, it will do nothing 866 * But sometimes it requests driver to reset and repost 867 * with error code -EAGAIN. 868 */ 869 ret = amdgpu_ras_feature_enable(adev, head, 1); 870 /* With old ras TA, we might fail to enable ras. 871 * Log it and just setup the object. 872 * TODO need remove this WA in the future. 873 */ 874 if (ret == -EINVAL) { 875 ret = __amdgpu_ras_feature_enable(adev, head, 1); 876 if (!ret) 877 dev_info(adev->dev, 878 "RAS INFO: %s setup object\n", 879 get_ras_block_str(head)); 880 } 881 } else { 882 /* setup the object then issue a ras TA disable cmd.*/ 883 ret = __amdgpu_ras_feature_enable(adev, head, 1); 884 if (ret) 885 return ret; 886 887 /* gfx block ras disable cmd must send to ras-ta */ 888 if (head->block == AMDGPU_RAS_BLOCK__GFX) 889 con->features |= BIT(head->block); 890 891 ret = amdgpu_ras_feature_enable(adev, head, 0); 892 893 /* clean gfx block ras features flag */ 894 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 895 con->features &= ~BIT(head->block); 896 } 897 } else 898 ret = amdgpu_ras_feature_enable(adev, head, enable); 899 900 return ret; 901 } 902 903 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 904 bool bypass) 905 { 906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 907 struct ras_manager *obj, *tmp; 908 909 list_for_each_entry_safe(obj, tmp, &con->head, node) { 910 /* bypass psp. 911 * aka just release the obj and corresponding flags 912 */ 913 if (bypass) { 914 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 915 break; 916 } else { 917 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 918 break; 919 } 920 } 921 922 return con->features; 923 } 924 925 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 926 bool bypass) 927 { 928 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 929 int i; 930 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 931 932 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 933 struct ras_common_if head = { 934 .block = i, 935 .type = default_ras_type, 936 .sub_block_index = 0, 937 }; 938 939 if (i == AMDGPU_RAS_BLOCK__MCA) 940 continue; 941 942 if (bypass) { 943 /* 944 * bypass psp. vbios enable ras for us. 945 * so just create the obj 946 */ 947 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 948 break; 949 } else { 950 if (amdgpu_ras_feature_enable(adev, &head, 1)) 951 break; 952 } 953 } 954 955 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 956 struct ras_common_if head = { 957 .block = AMDGPU_RAS_BLOCK__MCA, 958 .type = default_ras_type, 959 .sub_block_index = i, 960 }; 961 962 if (bypass) { 963 /* 964 * bypass psp. vbios enable ras for us. 965 * so just create the obj 966 */ 967 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 968 break; 969 } else { 970 if (amdgpu_ras_feature_enable(adev, &head, 1)) 971 break; 972 } 973 } 974 975 return con->features; 976 } 977 /* feature ctl end */ 978 979 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 980 enum amdgpu_ras_block block) 981 { 982 if (!block_obj) 983 return -EINVAL; 984 985 if (block_obj->ras_comm.block == block) 986 return 0; 987 988 return -EINVAL; 989 } 990 991 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 992 enum amdgpu_ras_block block, uint32_t sub_block_index) 993 { 994 struct amdgpu_ras_block_list *node, *tmp; 995 struct amdgpu_ras_block_object *obj; 996 997 if (block >= AMDGPU_RAS_BLOCK__LAST) 998 return NULL; 999 1000 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1001 if (!node->ras_obj) { 1002 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1003 continue; 1004 } 1005 1006 obj = node->ras_obj; 1007 if (obj->ras_block_match) { 1008 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1009 return obj; 1010 } else { 1011 if (amdgpu_ras_block_match_default(obj, block) == 0) 1012 return obj; 1013 } 1014 } 1015 1016 return NULL; 1017 } 1018 1019 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1020 { 1021 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1022 int ret = 0; 1023 1024 /* 1025 * choosing right query method according to 1026 * whether smu support query error information 1027 */ 1028 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1029 if (ret == -EOPNOTSUPP) { 1030 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1032 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1033 1034 /* umc query_ras_error_address is also responsible for clearing 1035 * error status 1036 */ 1037 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1039 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1040 } else if (!ret) { 1041 if (adev->umc.ras && 1042 adev->umc.ras->ecc_info_query_ras_error_count) 1043 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1044 1045 if (adev->umc.ras && 1046 adev->umc.ras->ecc_info_query_ras_error_address) 1047 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1048 } 1049 } 1050 1051 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1052 struct ras_manager *ras_mgr, 1053 struct ras_err_data *err_data, 1054 struct ras_query_context *qctx, 1055 const char *blk_name, 1056 bool is_ue, 1057 bool is_de) 1058 { 1059 struct amdgpu_smuio_mcm_config_info *mcm_info; 1060 struct ras_err_node *err_node; 1061 struct ras_err_info *err_info; 1062 u64 event_id = qctx->evid.event_id; 1063 1064 if (is_ue) { 1065 for_each_ras_error(err_node, err_data) { 1066 err_info = &err_node->err_info; 1067 mcm_info = &err_info->mcm_info; 1068 if (err_info->ue_count) { 1069 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1070 "%lld new uncorrectable hardware errors detected in %s block\n", 1071 mcm_info->socket_id, 1072 mcm_info->die_id, 1073 err_info->ue_count, 1074 blk_name); 1075 } 1076 } 1077 1078 for_each_ras_error(err_node, &ras_mgr->err_data) { 1079 err_info = &err_node->err_info; 1080 mcm_info = &err_info->mcm_info; 1081 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1082 "%lld uncorrectable hardware errors detected in total in %s block\n", 1083 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1084 } 1085 1086 } else { 1087 if (is_de) { 1088 for_each_ras_error(err_node, err_data) { 1089 err_info = &err_node->err_info; 1090 mcm_info = &err_info->mcm_info; 1091 if (err_info->de_count) { 1092 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1093 "%lld new deferred hardware errors detected in %s block\n", 1094 mcm_info->socket_id, 1095 mcm_info->die_id, 1096 err_info->de_count, 1097 blk_name); 1098 } 1099 } 1100 1101 for_each_ras_error(err_node, &ras_mgr->err_data) { 1102 err_info = &err_node->err_info; 1103 mcm_info = &err_info->mcm_info; 1104 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1105 "%lld deferred hardware errors detected in total in %s block\n", 1106 mcm_info->socket_id, mcm_info->die_id, 1107 err_info->de_count, blk_name); 1108 } 1109 } else { 1110 for_each_ras_error(err_node, err_data) { 1111 err_info = &err_node->err_info; 1112 mcm_info = &err_info->mcm_info; 1113 if (err_info->ce_count) { 1114 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1115 "%lld new correctable hardware errors detected in %s block\n", 1116 mcm_info->socket_id, 1117 mcm_info->die_id, 1118 err_info->ce_count, 1119 blk_name); 1120 } 1121 } 1122 1123 for_each_ras_error(err_node, &ras_mgr->err_data) { 1124 err_info = &err_node->err_info; 1125 mcm_info = &err_info->mcm_info; 1126 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1127 "%lld correctable hardware errors detected in total in %s block\n", 1128 mcm_info->socket_id, mcm_info->die_id, 1129 err_info->ce_count, blk_name); 1130 } 1131 } 1132 } 1133 } 1134 1135 static inline bool err_data_has_source_info(struct ras_err_data *data) 1136 { 1137 return !list_empty(&data->err_node_list); 1138 } 1139 1140 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1141 struct ras_query_if *query_if, 1142 struct ras_err_data *err_data, 1143 struct ras_query_context *qctx) 1144 { 1145 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1146 const char *blk_name = get_ras_block_str(&query_if->head); 1147 u64 event_id = qctx->evid.event_id; 1148 1149 if (err_data->ce_count) { 1150 if (err_data_has_source_info(err_data)) { 1151 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1152 blk_name, false, false); 1153 } else if (!adev->aid_mask && 1154 adev->smuio.funcs && 1155 adev->smuio.funcs->get_socket_id && 1156 adev->smuio.funcs->get_die_id) { 1157 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1158 "%ld correctable hardware errors " 1159 "detected in %s block\n", 1160 adev->smuio.funcs->get_socket_id(adev), 1161 adev->smuio.funcs->get_die_id(adev), 1162 ras_mgr->err_data.ce_count, 1163 blk_name); 1164 } else { 1165 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1166 "detected in %s block\n", 1167 ras_mgr->err_data.ce_count, 1168 blk_name); 1169 } 1170 } 1171 1172 if (err_data->ue_count) { 1173 if (err_data_has_source_info(err_data)) { 1174 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1175 blk_name, true, false); 1176 } else if (!adev->aid_mask && 1177 adev->smuio.funcs && 1178 adev->smuio.funcs->get_socket_id && 1179 adev->smuio.funcs->get_die_id) { 1180 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1181 "%ld uncorrectable hardware errors " 1182 "detected in %s block\n", 1183 adev->smuio.funcs->get_socket_id(adev), 1184 adev->smuio.funcs->get_die_id(adev), 1185 ras_mgr->err_data.ue_count, 1186 blk_name); 1187 } else { 1188 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1189 "detected in %s block\n", 1190 ras_mgr->err_data.ue_count, 1191 blk_name); 1192 } 1193 } 1194 1195 if (err_data->de_count) { 1196 if (err_data_has_source_info(err_data)) { 1197 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1198 blk_name, false, true); 1199 } else if (!adev->aid_mask && 1200 adev->smuio.funcs && 1201 adev->smuio.funcs->get_socket_id && 1202 adev->smuio.funcs->get_die_id) { 1203 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1204 "%ld deferred hardware errors " 1205 "detected in %s block\n", 1206 adev->smuio.funcs->get_socket_id(adev), 1207 adev->smuio.funcs->get_die_id(adev), 1208 ras_mgr->err_data.de_count, 1209 blk_name); 1210 } else { 1211 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1212 "detected in %s block\n", 1213 ras_mgr->err_data.de_count, 1214 blk_name); 1215 } 1216 } 1217 } 1218 1219 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1220 struct ras_query_if *query_if, 1221 struct ras_err_data *err_data, 1222 struct ras_query_context *qctx) 1223 { 1224 unsigned long new_ue, new_ce, new_de; 1225 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1226 const char *blk_name = get_ras_block_str(&query_if->head); 1227 u64 event_id = qctx->evid.event_id; 1228 1229 new_ce = err_data->ce_count - obj->err_data.ce_count; 1230 new_ue = err_data->ue_count - obj->err_data.ue_count; 1231 new_de = err_data->de_count - obj->err_data.de_count; 1232 1233 if (new_ce) { 1234 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1235 "detected in %s block\n", 1236 new_ce, 1237 blk_name); 1238 } 1239 1240 if (new_ue) { 1241 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1242 "detected in %s block\n", 1243 new_ue, 1244 blk_name); 1245 } 1246 1247 if (new_de) { 1248 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1249 "detected in %s block\n", 1250 new_de, 1251 blk_name); 1252 } 1253 } 1254 1255 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1256 { 1257 struct ras_err_node *err_node; 1258 struct ras_err_info *err_info; 1259 1260 if (err_data_has_source_info(err_data)) { 1261 for_each_ras_error(err_node, err_data) { 1262 err_info = &err_node->err_info; 1263 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1264 &err_info->mcm_info, err_info->de_count); 1265 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1266 &err_info->mcm_info, err_info->ce_count); 1267 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1268 &err_info->mcm_info, err_info->ue_count); 1269 } 1270 } else { 1271 /* for legacy asic path which doesn't has error source info */ 1272 obj->err_data.ue_count += err_data->ue_count; 1273 obj->err_data.ce_count += err_data->ce_count; 1274 obj->err_data.de_count += err_data->de_count; 1275 } 1276 } 1277 1278 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1279 struct ras_err_data *err_data) 1280 { 1281 /* Host reports absolute counts */ 1282 obj->err_data.ue_count = err_data->ue_count; 1283 obj->err_data.ce_count = err_data->ce_count; 1284 obj->err_data.de_count = err_data->de_count; 1285 } 1286 1287 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1288 { 1289 struct ras_common_if head; 1290 1291 memset(&head, 0, sizeof(head)); 1292 head.block = blk; 1293 1294 return amdgpu_ras_find_obj(adev, &head); 1295 } 1296 1297 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1298 const struct aca_info *aca_info, void *data) 1299 { 1300 struct ras_manager *obj; 1301 1302 /* in resume phase, no need to create aca fs node */ 1303 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1304 return 0; 1305 1306 obj = get_ras_manager(adev, blk); 1307 if (!obj) 1308 return -EINVAL; 1309 1310 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1311 } 1312 1313 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1314 { 1315 struct ras_manager *obj; 1316 1317 obj = get_ras_manager(adev, blk); 1318 if (!obj) 1319 return -EINVAL; 1320 1321 amdgpu_aca_remove_handle(&obj->aca_handle); 1322 1323 return 0; 1324 } 1325 1326 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1327 enum aca_error_type type, struct ras_err_data *err_data, 1328 struct ras_query_context *qctx) 1329 { 1330 struct ras_manager *obj; 1331 1332 obj = get_ras_manager(adev, blk); 1333 if (!obj) 1334 return -EINVAL; 1335 1336 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1337 } 1338 1339 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1340 struct aca_handle *handle, char *buf, void *data) 1341 { 1342 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1343 struct ras_query_if info = { 1344 .head = obj->head, 1345 }; 1346 1347 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1348 return sysfs_emit(buf, "Query currently inaccessible\n"); 1349 1350 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1351 return -EINVAL; 1352 1353 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1354 "ce", info.ce_count, "de", info.de_count); 1355 } 1356 1357 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1358 struct ras_query_if *info, 1359 struct ras_err_data *err_data, 1360 struct ras_query_context *qctx, 1361 unsigned int error_query_mode) 1362 { 1363 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1364 struct amdgpu_ras_block_object *block_obj = NULL; 1365 int ret; 1366 1367 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1368 return -EINVAL; 1369 1370 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1371 return -EINVAL; 1372 1373 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1374 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1375 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1376 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1377 amdgpu_ras_get_ecc_info(adev, err_data); 1378 } else { 1379 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1380 if (!block_obj || !block_obj->hw_ops) { 1381 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1382 get_ras_block_str(&info->head)); 1383 return -EINVAL; 1384 } 1385 1386 if (block_obj->hw_ops->query_ras_error_count) 1387 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1388 1389 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1390 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1391 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1392 if (block_obj->hw_ops->query_ras_error_status) 1393 block_obj->hw_ops->query_ras_error_status(adev); 1394 } 1395 } 1396 } else { 1397 if (amdgpu_aca_is_enabled(adev)) { 1398 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1399 if (ret) 1400 return ret; 1401 1402 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1403 if (ret) 1404 return ret; 1405 1406 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1407 if (ret) 1408 return ret; 1409 } else { 1410 /* FIXME: add code to check return value later */ 1411 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1412 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1413 } 1414 } 1415 1416 return 0; 1417 } 1418 1419 /* query/inject/cure begin */ 1420 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1421 struct ras_query_if *info, 1422 enum ras_event_type type) 1423 { 1424 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1425 struct ras_err_data err_data; 1426 struct ras_query_context qctx; 1427 unsigned int error_query_mode; 1428 int ret; 1429 1430 if (!obj) 1431 return -EINVAL; 1432 1433 ret = amdgpu_ras_error_data_init(&err_data); 1434 if (ret) 1435 return ret; 1436 1437 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1438 return -EINVAL; 1439 1440 memset(&qctx, 0, sizeof(qctx)); 1441 qctx.evid.type = type; 1442 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1443 1444 if (!down_read_trylock(&adev->reset_domain->sem)) { 1445 ret = -EIO; 1446 goto out_fini_err_data; 1447 } 1448 1449 ret = amdgpu_ras_query_error_status_helper(adev, info, 1450 &err_data, 1451 &qctx, 1452 error_query_mode); 1453 up_read(&adev->reset_domain->sem); 1454 if (ret) 1455 goto out_fini_err_data; 1456 1457 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1458 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1459 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1460 } else { 1461 /* Host provides absolute error counts. First generate the report 1462 * using the previous VF internal count against new host count. 1463 * Then Update VF internal count. 1464 */ 1465 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1466 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1467 } 1468 1469 info->ue_count = obj->err_data.ue_count; 1470 info->ce_count = obj->err_data.ce_count; 1471 info->de_count = obj->err_data.de_count; 1472 1473 out_fini_err_data: 1474 amdgpu_ras_error_data_fini(&err_data); 1475 1476 return ret; 1477 } 1478 1479 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1480 { 1481 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1482 } 1483 1484 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1485 enum amdgpu_ras_block block) 1486 { 1487 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1488 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1489 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1490 1491 if (!block_obj || !block_obj->hw_ops) { 1492 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1493 ras_block_str(block)); 1494 return -EOPNOTSUPP; 1495 } 1496 1497 if (!amdgpu_ras_is_supported(adev, block) || 1498 !amdgpu_ras_get_aca_debug_mode(adev)) 1499 return -EOPNOTSUPP; 1500 1501 /* skip ras error reset in gpu reset */ 1502 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1503 ((smu_funcs && smu_funcs->set_debug_mode) || 1504 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1505 return -EOPNOTSUPP; 1506 1507 if (block_obj->hw_ops->reset_ras_error_count) 1508 block_obj->hw_ops->reset_ras_error_count(adev); 1509 1510 return 0; 1511 } 1512 1513 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1514 enum amdgpu_ras_block block) 1515 { 1516 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1517 1518 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1519 return 0; 1520 1521 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1522 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1523 if (block_obj->hw_ops->reset_ras_error_status) 1524 block_obj->hw_ops->reset_ras_error_status(adev); 1525 } 1526 1527 return 0; 1528 } 1529 1530 /* wrapper of psp_ras_trigger_error */ 1531 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1532 struct ras_inject_if *info) 1533 { 1534 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1535 struct ta_ras_trigger_error_input block_info = { 1536 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1537 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1538 .sub_block_index = info->head.sub_block_index, 1539 .address = info->address, 1540 .value = info->value, 1541 }; 1542 int ret = -EINVAL; 1543 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1544 info->head.block, 1545 info->head.sub_block_index); 1546 1547 /* inject on guest isn't allowed, return success directly */ 1548 if (amdgpu_sriov_vf(adev)) 1549 return 0; 1550 1551 if (!obj) 1552 return -EINVAL; 1553 1554 if (!block_obj || !block_obj->hw_ops) { 1555 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1556 get_ras_block_str(&info->head)); 1557 return -EINVAL; 1558 } 1559 1560 /* Calculate XGMI relative offset */ 1561 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1562 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1563 block_info.address = 1564 amdgpu_xgmi_get_relative_phy_addr(adev, 1565 block_info.address); 1566 } 1567 1568 if (block_obj->hw_ops->ras_error_inject) { 1569 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1570 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1571 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1572 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1573 info->instance_mask); 1574 } else { 1575 /* default path */ 1576 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1577 } 1578 1579 if (ret) 1580 dev_err(adev->dev, "ras inject %s failed %d\n", 1581 get_ras_block_str(&info->head), ret); 1582 1583 return ret; 1584 } 1585 1586 /** 1587 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1588 * @adev: pointer to AMD GPU device 1589 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1590 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1591 * @query_info: pointer to ras_query_if 1592 * 1593 * Return 0 for query success or do nothing, otherwise return an error 1594 * on failures 1595 */ 1596 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1597 unsigned long *ce_count, 1598 unsigned long *ue_count, 1599 struct ras_query_if *query_info) 1600 { 1601 int ret; 1602 1603 if (!query_info) 1604 /* do nothing if query_info is not specified */ 1605 return 0; 1606 1607 ret = amdgpu_ras_query_error_status(adev, query_info); 1608 if (ret) 1609 return ret; 1610 1611 *ce_count += query_info->ce_count; 1612 *ue_count += query_info->ue_count; 1613 1614 /* some hardware/IP supports read to clear 1615 * no need to explictly reset the err status after the query call */ 1616 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1617 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1618 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1619 dev_warn(adev->dev, 1620 "Failed to reset error counter and error status\n"); 1621 } 1622 1623 return 0; 1624 } 1625 1626 /** 1627 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1628 * @adev: pointer to AMD GPU device 1629 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1630 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1631 * errors. 1632 * @query_info: pointer to ras_query_if if the query request is only for 1633 * specific ip block; if info is NULL, then the qurey request is for 1634 * all the ip blocks that support query ras error counters/status 1635 * 1636 * If set, @ce_count or @ue_count, count and return the corresponding 1637 * error counts in those integer pointers. Return 0 if the device 1638 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1639 */ 1640 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1641 unsigned long *ce_count, 1642 unsigned long *ue_count, 1643 struct ras_query_if *query_info) 1644 { 1645 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1646 struct ras_manager *obj; 1647 unsigned long ce, ue; 1648 int ret; 1649 1650 if (!adev->ras_enabled || !con) 1651 return -EOPNOTSUPP; 1652 1653 /* Don't count since no reporting. 1654 */ 1655 if (!ce_count && !ue_count) 1656 return 0; 1657 1658 ce = 0; 1659 ue = 0; 1660 if (!query_info) { 1661 /* query all the ip blocks that support ras query interface */ 1662 list_for_each_entry(obj, &con->head, node) { 1663 struct ras_query_if info = { 1664 .head = obj->head, 1665 }; 1666 1667 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1668 } 1669 } else { 1670 /* query specific ip block */ 1671 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1672 } 1673 1674 if (ret) 1675 return ret; 1676 1677 if (ce_count) 1678 *ce_count = ce; 1679 1680 if (ue_count) 1681 *ue_count = ue; 1682 1683 return 0; 1684 } 1685 /* query/inject/cure end */ 1686 1687 1688 /* sysfs begin */ 1689 1690 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1691 struct ras_badpage **bps, unsigned int *count); 1692 1693 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1694 { 1695 switch (flags) { 1696 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1697 return "R"; 1698 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1699 return "P"; 1700 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1701 default: 1702 return "F"; 1703 } 1704 } 1705 1706 /** 1707 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1708 * 1709 * It allows user to read the bad pages of vram on the gpu through 1710 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1711 * 1712 * It outputs multiple lines, and each line stands for one gpu page. 1713 * 1714 * The format of one line is below, 1715 * gpu pfn : gpu page size : flags 1716 * 1717 * gpu pfn and gpu page size are printed in hex format. 1718 * flags can be one of below character, 1719 * 1720 * R: reserved, this gpu page is reserved and not able to use. 1721 * 1722 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1723 * in next window of page_reserve. 1724 * 1725 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1726 * 1727 * Examples: 1728 * 1729 * .. code-block:: bash 1730 * 1731 * 0x00000001 : 0x00001000 : R 1732 * 0x00000002 : 0x00001000 : P 1733 * 1734 */ 1735 1736 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1737 struct kobject *kobj, struct bin_attribute *attr, 1738 char *buf, loff_t ppos, size_t count) 1739 { 1740 struct amdgpu_ras *con = 1741 container_of(attr, struct amdgpu_ras, badpages_attr); 1742 struct amdgpu_device *adev = con->adev; 1743 const unsigned int element_size = 1744 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1745 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1746 unsigned int end = div64_ul(ppos + count - 1, element_size); 1747 ssize_t s = 0; 1748 struct ras_badpage *bps = NULL; 1749 unsigned int bps_count = 0; 1750 1751 memset(buf, 0, count); 1752 1753 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1754 return 0; 1755 1756 for (; start < end && start < bps_count; start++) 1757 s += scnprintf(&buf[s], element_size + 1, 1758 "0x%08x : 0x%08x : %1s\n", 1759 bps[start].bp, 1760 bps[start].size, 1761 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1762 1763 kfree(bps); 1764 1765 return s; 1766 } 1767 1768 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1769 struct device_attribute *attr, char *buf) 1770 { 1771 struct amdgpu_ras *con = 1772 container_of(attr, struct amdgpu_ras, features_attr); 1773 1774 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1775 } 1776 1777 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1778 struct device_attribute *attr, char *buf) 1779 { 1780 struct amdgpu_ras *con = 1781 container_of(attr, struct amdgpu_ras, version_attr); 1782 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version); 1783 } 1784 1785 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1786 struct device_attribute *attr, char *buf) 1787 { 1788 struct amdgpu_ras *con = 1789 container_of(attr, struct amdgpu_ras, schema_attr); 1790 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1791 } 1792 1793 static struct { 1794 enum ras_event_type type; 1795 const char *name; 1796 } dump_event[] = { 1797 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 1798 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 1799 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 1800 }; 1801 1802 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 1803 struct device_attribute *attr, char *buf) 1804 { 1805 struct amdgpu_ras *con = 1806 container_of(attr, struct amdgpu_ras, event_state_attr); 1807 struct ras_event_manager *event_mgr = con->event_mgr; 1808 struct ras_event_state *event_state; 1809 int i, size = 0; 1810 1811 if (!event_mgr) 1812 return -EINVAL; 1813 1814 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 1815 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 1816 event_state = &event_mgr->event_state[dump_event[i].type]; 1817 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 1818 dump_event[i].name, 1819 atomic64_read(&event_state->count), 1820 event_state->last_seqno); 1821 } 1822 1823 return (ssize_t)size; 1824 } 1825 1826 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1827 { 1828 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1829 1830 if (adev->dev->kobj.sd) 1831 sysfs_remove_file_from_group(&adev->dev->kobj, 1832 &con->badpages_attr.attr, 1833 RAS_FS_NAME); 1834 } 1835 1836 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 1837 { 1838 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1839 struct attribute *attrs[] = { 1840 &con->features_attr.attr, 1841 &con->version_attr.attr, 1842 &con->schema_attr.attr, 1843 &con->event_state_attr.attr, 1844 NULL 1845 }; 1846 struct attribute_group group = { 1847 .name = RAS_FS_NAME, 1848 .attrs = attrs, 1849 }; 1850 1851 if (adev->dev->kobj.sd) 1852 sysfs_remove_group(&adev->dev->kobj, &group); 1853 1854 return 0; 1855 } 1856 1857 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1858 struct ras_common_if *head) 1859 { 1860 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1861 1862 if (amdgpu_aca_is_enabled(adev)) 1863 return 0; 1864 1865 if (!obj || obj->attr_inuse) 1866 return -EINVAL; 1867 1868 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 1869 return 0; 1870 1871 get_obj(obj); 1872 1873 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1874 "%s_err_count", head->name); 1875 1876 obj->sysfs_attr = (struct device_attribute){ 1877 .attr = { 1878 .name = obj->fs_data.sysfs_name, 1879 .mode = S_IRUGO, 1880 }, 1881 .show = amdgpu_ras_sysfs_read, 1882 }; 1883 sysfs_attr_init(&obj->sysfs_attr.attr); 1884 1885 if (sysfs_add_file_to_group(&adev->dev->kobj, 1886 &obj->sysfs_attr.attr, 1887 RAS_FS_NAME)) { 1888 put_obj(obj); 1889 return -EINVAL; 1890 } 1891 1892 obj->attr_inuse = 1; 1893 1894 return 0; 1895 } 1896 1897 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1898 struct ras_common_if *head) 1899 { 1900 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1901 1902 if (amdgpu_aca_is_enabled(adev)) 1903 return 0; 1904 1905 if (!obj || !obj->attr_inuse) 1906 return -EINVAL; 1907 1908 if (adev->dev->kobj.sd) 1909 sysfs_remove_file_from_group(&adev->dev->kobj, 1910 &obj->sysfs_attr.attr, 1911 RAS_FS_NAME); 1912 obj->attr_inuse = 0; 1913 put_obj(obj); 1914 1915 return 0; 1916 } 1917 1918 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1919 { 1920 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1921 struct ras_manager *obj, *tmp; 1922 1923 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1924 amdgpu_ras_sysfs_remove(adev, &obj->head); 1925 } 1926 1927 if (amdgpu_bad_page_threshold != 0) 1928 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1929 1930 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 1931 1932 return 0; 1933 } 1934 /* sysfs end */ 1935 1936 /** 1937 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 1938 * 1939 * Normally when there is an uncorrectable error, the driver will reset 1940 * the GPU to recover. However, in the event of an unrecoverable error, 1941 * the driver provides an interface to reboot the system automatically 1942 * in that event. 1943 * 1944 * The following file in debugfs provides that interface: 1945 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 1946 * 1947 * Usage: 1948 * 1949 * .. code-block:: bash 1950 * 1951 * echo true > .../ras/auto_reboot 1952 * 1953 */ 1954 /* debugfs begin */ 1955 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 1956 { 1957 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1958 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 1959 struct drm_minor *minor = adev_to_drm(adev)->primary; 1960 struct dentry *dir; 1961 1962 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 1963 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 1964 &amdgpu_ras_debugfs_ctrl_ops); 1965 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 1966 &amdgpu_ras_debugfs_eeprom_ops); 1967 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 1968 &con->bad_page_cnt_threshold); 1969 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 1970 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 1971 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 1972 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 1973 &amdgpu_ras_debugfs_eeprom_size_ops); 1974 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 1975 S_IRUGO, dir, adev, 1976 &amdgpu_ras_debugfs_eeprom_table_ops); 1977 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 1978 1979 /* 1980 * After one uncorrectable error happens, usually GPU recovery will 1981 * be scheduled. But due to the known problem in GPU recovery failing 1982 * to bring GPU back, below interface provides one direct way to 1983 * user to reboot system automatically in such case within 1984 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 1985 * will never be called. 1986 */ 1987 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 1988 1989 /* 1990 * User could set this not to clean up hardware's error count register 1991 * of RAS IPs during ras recovery. 1992 */ 1993 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 1994 &con->disable_ras_err_cnt_harvest); 1995 return dir; 1996 } 1997 1998 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 1999 struct ras_fs_if *head, 2000 struct dentry *dir) 2001 { 2002 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2003 2004 if (!obj || !dir) 2005 return; 2006 2007 get_obj(obj); 2008 2009 memcpy(obj->fs_data.debugfs_name, 2010 head->debugfs_name, 2011 sizeof(obj->fs_data.debugfs_name)); 2012 2013 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2014 obj, &amdgpu_ras_debugfs_ops); 2015 } 2016 2017 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2018 { 2019 bool ret; 2020 2021 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2022 case IP_VERSION(13, 0, 6): 2023 case IP_VERSION(13, 0, 12): 2024 case IP_VERSION(13, 0, 14): 2025 ret = true; 2026 break; 2027 default: 2028 ret = false; 2029 break; 2030 } 2031 2032 return ret; 2033 } 2034 2035 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2036 { 2037 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2038 struct dentry *dir; 2039 struct ras_manager *obj; 2040 struct ras_fs_if fs_info; 2041 2042 /* 2043 * it won't be called in resume path, no need to check 2044 * suspend and gpu reset status 2045 */ 2046 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2047 return; 2048 2049 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2050 2051 list_for_each_entry(obj, &con->head, node) { 2052 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2053 (obj->attr_inuse == 1)) { 2054 sprintf(fs_info.debugfs_name, "%s_err_inject", 2055 get_ras_block_str(&obj->head)); 2056 fs_info.head = obj->head; 2057 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2058 } 2059 } 2060 2061 if (amdgpu_ras_aca_is_supported(adev)) { 2062 if (amdgpu_aca_is_enabled(adev)) 2063 amdgpu_aca_smu_debugfs_init(adev, dir); 2064 else 2065 amdgpu_mca_smu_debugfs_init(adev, dir); 2066 } 2067 } 2068 2069 /* debugfs end */ 2070 2071 /* ras fs */ 2072 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2073 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2074 static DEVICE_ATTR(features, S_IRUGO, 2075 amdgpu_ras_sysfs_features_read, NULL); 2076 static DEVICE_ATTR(version, 0444, 2077 amdgpu_ras_sysfs_version_show, NULL); 2078 static DEVICE_ATTR(schema, 0444, 2079 amdgpu_ras_sysfs_schema_show, NULL); 2080 static DEVICE_ATTR(event_state, 0444, 2081 amdgpu_ras_sysfs_event_state_show, NULL); 2082 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2083 { 2084 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2085 struct attribute_group group = { 2086 .name = RAS_FS_NAME, 2087 }; 2088 struct attribute *attrs[] = { 2089 &con->features_attr.attr, 2090 &con->version_attr.attr, 2091 &con->schema_attr.attr, 2092 &con->event_state_attr.attr, 2093 NULL 2094 }; 2095 struct bin_attribute *bin_attrs[] = { 2096 NULL, 2097 NULL, 2098 }; 2099 int r; 2100 2101 group.attrs = attrs; 2102 2103 /* add features entry */ 2104 con->features_attr = dev_attr_features; 2105 sysfs_attr_init(attrs[0]); 2106 2107 /* add version entry */ 2108 con->version_attr = dev_attr_version; 2109 sysfs_attr_init(attrs[1]); 2110 2111 /* add schema entry */ 2112 con->schema_attr = dev_attr_schema; 2113 sysfs_attr_init(attrs[2]); 2114 2115 /* add event_state entry */ 2116 con->event_state_attr = dev_attr_event_state; 2117 sysfs_attr_init(attrs[3]); 2118 2119 if (amdgpu_bad_page_threshold != 0) { 2120 /* add bad_page_features entry */ 2121 bin_attr_gpu_vram_bad_pages.private = NULL; 2122 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2123 bin_attrs[0] = &con->badpages_attr; 2124 group.bin_attrs = bin_attrs; 2125 sysfs_bin_attr_init(bin_attrs[0]); 2126 } 2127 2128 r = sysfs_create_group(&adev->dev->kobj, &group); 2129 if (r) 2130 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2131 2132 return 0; 2133 } 2134 2135 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2136 { 2137 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2138 struct ras_manager *con_obj, *ip_obj, *tmp; 2139 2140 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2141 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2142 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2143 if (ip_obj) 2144 put_obj(ip_obj); 2145 } 2146 } 2147 2148 amdgpu_ras_sysfs_remove_all(adev); 2149 return 0; 2150 } 2151 /* ras fs end */ 2152 2153 /* ih begin */ 2154 2155 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2156 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2157 * register to check whether the interrupt is triggered or not, and properly 2158 * ack the interrupt if it is there 2159 */ 2160 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2161 { 2162 /* Fatal error events are handled on host side */ 2163 if (amdgpu_sriov_vf(adev)) 2164 return; 2165 /** 2166 * If the current interrupt is caused by a non-fatal RAS error, skip 2167 * check for fatal error. For fatal errors, FED status of all devices 2168 * in XGMI hive gets set when the first device gets fatal error 2169 * interrupt. The error gets propagated to other devices as well, so 2170 * make sure to ack the interrupt regardless of FED status. 2171 */ 2172 if (!amdgpu_ras_get_fed_status(adev) && 2173 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2174 return; 2175 2176 if (adev->nbio.ras && 2177 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2178 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2179 2180 if (adev->nbio.ras && 2181 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2182 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2183 } 2184 2185 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2186 struct amdgpu_iv_entry *entry) 2187 { 2188 bool poison_stat = false; 2189 struct amdgpu_device *adev = obj->adev; 2190 struct amdgpu_ras_block_object *block_obj = 2191 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2192 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2193 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2194 u64 event_id; 2195 int ret; 2196 2197 if (!block_obj || !con) 2198 return; 2199 2200 ret = amdgpu_ras_mark_ras_event(adev, type); 2201 if (ret) 2202 return; 2203 2204 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2205 /* both query_poison_status and handle_poison_consumption are optional, 2206 * but at least one of them should be implemented if we need poison 2207 * consumption handler 2208 */ 2209 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2210 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2211 if (!poison_stat) { 2212 /* Not poison consumption interrupt, no need to handle it */ 2213 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2214 block_obj->ras_comm.name); 2215 2216 return; 2217 } 2218 } 2219 2220 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2221 2222 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2223 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2224 2225 /* gpu reset is fallback for failed and default cases. 2226 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2227 */ 2228 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2229 event_id = amdgpu_ras_acquire_event_id(adev, type); 2230 RAS_EVENT_LOG(adev, event_id, 2231 "GPU reset for %s RAS poison consumption is issued!\n", 2232 block_obj->ras_comm.name); 2233 amdgpu_ras_reset_gpu(adev); 2234 } 2235 2236 if (!poison_stat) 2237 amdgpu_gfx_poison_consumption_handler(adev, entry); 2238 } 2239 2240 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2241 struct amdgpu_iv_entry *entry) 2242 { 2243 struct amdgpu_device *adev = obj->adev; 2244 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2245 u64 event_id; 2246 int ret; 2247 2248 ret = amdgpu_ras_mark_ras_event(adev, type); 2249 if (ret) 2250 return; 2251 2252 event_id = amdgpu_ras_acquire_event_id(adev, type); 2253 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2254 2255 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2256 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2257 2258 atomic_inc(&con->page_retirement_req_cnt); 2259 atomic_inc(&con->poison_creation_count); 2260 2261 wake_up(&con->page_retirement_wq); 2262 } 2263 } 2264 2265 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2266 struct amdgpu_iv_entry *entry) 2267 { 2268 struct ras_ih_data *data = &obj->ih_data; 2269 struct ras_err_data err_data; 2270 int ret; 2271 2272 if (!data->cb) 2273 return; 2274 2275 ret = amdgpu_ras_error_data_init(&err_data); 2276 if (ret) 2277 return; 2278 2279 /* Let IP handle its data, maybe we need get the output 2280 * from the callback to update the error type/count, etc 2281 */ 2282 amdgpu_ras_set_fed(obj->adev, true); 2283 ret = data->cb(obj->adev, &err_data, entry); 2284 /* ue will trigger an interrupt, and in that case 2285 * we need do a reset to recovery the whole system. 2286 * But leave IP do that recovery, here we just dispatch 2287 * the error. 2288 */ 2289 if (ret == AMDGPU_RAS_SUCCESS) { 2290 /* these counts could be left as 0 if 2291 * some blocks do not count error number 2292 */ 2293 obj->err_data.ue_count += err_data.ue_count; 2294 obj->err_data.ce_count += err_data.ce_count; 2295 obj->err_data.de_count += err_data.de_count; 2296 } 2297 2298 amdgpu_ras_error_data_fini(&err_data); 2299 } 2300 2301 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2302 { 2303 struct ras_ih_data *data = &obj->ih_data; 2304 struct amdgpu_iv_entry entry; 2305 2306 while (data->rptr != data->wptr) { 2307 rmb(); 2308 memcpy(&entry, &data->ring[data->rptr], 2309 data->element_size); 2310 2311 wmb(); 2312 data->rptr = (data->aligned_element_size + 2313 data->rptr) % data->ring_size; 2314 2315 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2316 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2317 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2318 else 2319 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2320 } else { 2321 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2322 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2323 else 2324 dev_warn(obj->adev->dev, 2325 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2326 } 2327 } 2328 } 2329 2330 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2331 { 2332 struct ras_ih_data *data = 2333 container_of(work, struct ras_ih_data, ih_work); 2334 struct ras_manager *obj = 2335 container_of(data, struct ras_manager, ih_data); 2336 2337 amdgpu_ras_interrupt_handler(obj); 2338 } 2339 2340 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2341 struct ras_dispatch_if *info) 2342 { 2343 struct ras_manager *obj; 2344 struct ras_ih_data *data; 2345 2346 obj = amdgpu_ras_find_obj(adev, &info->head); 2347 if (!obj) 2348 return -EINVAL; 2349 2350 data = &obj->ih_data; 2351 2352 if (data->inuse == 0) 2353 return 0; 2354 2355 /* Might be overflow... */ 2356 memcpy(&data->ring[data->wptr], info->entry, 2357 data->element_size); 2358 2359 wmb(); 2360 data->wptr = (data->aligned_element_size + 2361 data->wptr) % data->ring_size; 2362 2363 schedule_work(&data->ih_work); 2364 2365 return 0; 2366 } 2367 2368 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2369 struct ras_common_if *head) 2370 { 2371 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2372 struct ras_ih_data *data; 2373 2374 if (!obj) 2375 return -EINVAL; 2376 2377 data = &obj->ih_data; 2378 if (data->inuse == 0) 2379 return 0; 2380 2381 cancel_work_sync(&data->ih_work); 2382 2383 kfree(data->ring); 2384 memset(data, 0, sizeof(*data)); 2385 put_obj(obj); 2386 2387 return 0; 2388 } 2389 2390 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2391 struct ras_common_if *head) 2392 { 2393 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2394 struct ras_ih_data *data; 2395 struct amdgpu_ras_block_object *ras_obj; 2396 2397 if (!obj) { 2398 /* in case we registe the IH before enable ras feature */ 2399 obj = amdgpu_ras_create_obj(adev, head); 2400 if (!obj) 2401 return -EINVAL; 2402 } else 2403 get_obj(obj); 2404 2405 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2406 2407 data = &obj->ih_data; 2408 /* add the callback.etc */ 2409 *data = (struct ras_ih_data) { 2410 .inuse = 0, 2411 .cb = ras_obj->ras_cb, 2412 .element_size = sizeof(struct amdgpu_iv_entry), 2413 .rptr = 0, 2414 .wptr = 0, 2415 }; 2416 2417 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2418 2419 data->aligned_element_size = ALIGN(data->element_size, 8); 2420 /* the ring can store 64 iv entries. */ 2421 data->ring_size = 64 * data->aligned_element_size; 2422 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2423 if (!data->ring) { 2424 put_obj(obj); 2425 return -ENOMEM; 2426 } 2427 2428 /* IH is ready */ 2429 data->inuse = 1; 2430 2431 return 0; 2432 } 2433 2434 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2435 { 2436 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2437 struct ras_manager *obj, *tmp; 2438 2439 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2440 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2441 } 2442 2443 return 0; 2444 } 2445 /* ih end */ 2446 2447 /* traversal all IPs except NBIO to query error counter */ 2448 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2449 { 2450 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2451 struct ras_manager *obj; 2452 2453 if (!adev->ras_enabled || !con) 2454 return; 2455 2456 list_for_each_entry(obj, &con->head, node) { 2457 struct ras_query_if info = { 2458 .head = obj->head, 2459 }; 2460 2461 /* 2462 * PCIE_BIF IP has one different isr by ras controller 2463 * interrupt, the specific ras counter query will be 2464 * done in that isr. So skip such block from common 2465 * sync flood interrupt isr calling. 2466 */ 2467 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2468 continue; 2469 2470 /* 2471 * this is a workaround for aldebaran, skip send msg to 2472 * smu to get ecc_info table due to smu handle get ecc 2473 * info table failed temporarily. 2474 * should be removed until smu fix handle ecc_info table. 2475 */ 2476 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2477 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2478 IP_VERSION(13, 0, 2))) 2479 continue; 2480 2481 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2482 2483 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2484 IP_VERSION(11, 0, 2) && 2485 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2486 IP_VERSION(11, 0, 4) && 2487 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2488 IP_VERSION(13, 0, 0)) { 2489 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2490 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2491 } 2492 } 2493 } 2494 2495 /* Parse RdRspStatus and WrRspStatus */ 2496 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2497 struct ras_query_if *info) 2498 { 2499 struct amdgpu_ras_block_object *block_obj; 2500 /* 2501 * Only two block need to query read/write 2502 * RspStatus at current state 2503 */ 2504 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2505 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2506 return; 2507 2508 block_obj = amdgpu_ras_get_ras_block(adev, 2509 info->head.block, 2510 info->head.sub_block_index); 2511 2512 if (!block_obj || !block_obj->hw_ops) { 2513 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2514 get_ras_block_str(&info->head)); 2515 return; 2516 } 2517 2518 if (block_obj->hw_ops->query_ras_error_status) 2519 block_obj->hw_ops->query_ras_error_status(adev); 2520 2521 } 2522 2523 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2524 { 2525 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2526 struct ras_manager *obj; 2527 2528 if (!adev->ras_enabled || !con) 2529 return; 2530 2531 list_for_each_entry(obj, &con->head, node) { 2532 struct ras_query_if info = { 2533 .head = obj->head, 2534 }; 2535 2536 amdgpu_ras_error_status_query(adev, &info); 2537 } 2538 } 2539 2540 /* recovery begin */ 2541 2542 /* return 0 on success. 2543 * caller need free bps. 2544 */ 2545 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2546 struct ras_badpage **bps, unsigned int *count) 2547 { 2548 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2549 struct ras_err_handler_data *data; 2550 int i = 0; 2551 int ret = 0, status; 2552 2553 if (!con || !con->eh_data || !bps || !count) 2554 return -EINVAL; 2555 2556 mutex_lock(&con->recovery_lock); 2557 data = con->eh_data; 2558 if (!data || data->count == 0) { 2559 *bps = NULL; 2560 ret = -EINVAL; 2561 goto out; 2562 } 2563 2564 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL); 2565 if (!*bps) { 2566 ret = -ENOMEM; 2567 goto out; 2568 } 2569 2570 for (; i < data->count; i++) { 2571 (*bps)[i] = (struct ras_badpage){ 2572 .bp = data->bps[i].retired_page, 2573 .size = AMDGPU_GPU_PAGE_SIZE, 2574 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 2575 }; 2576 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 2577 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT); 2578 if (status == -EBUSY) 2579 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 2580 else if (status == -ENOENT) 2581 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 2582 } 2583 2584 *count = data->count; 2585 out: 2586 mutex_unlock(&con->recovery_lock); 2587 return ret; 2588 } 2589 2590 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2591 struct amdgpu_hive_info *hive, bool status) 2592 { 2593 struct amdgpu_device *tmp_adev; 2594 2595 if (hive) { 2596 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2597 amdgpu_ras_set_fed(tmp_adev, status); 2598 } else { 2599 amdgpu_ras_set_fed(adev, status); 2600 } 2601 } 2602 2603 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2604 { 2605 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2606 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2607 int hive_ras_recovery = 0; 2608 2609 if (hive) { 2610 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2611 amdgpu_put_xgmi_hive(hive); 2612 } 2613 2614 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2615 return true; 2616 2617 return false; 2618 } 2619 2620 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2621 { 2622 if (amdgpu_ras_intr_triggered()) 2623 return RAS_EVENT_TYPE_FATAL; 2624 else 2625 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2626 } 2627 2628 static void amdgpu_ras_do_recovery(struct work_struct *work) 2629 { 2630 struct amdgpu_ras *ras = 2631 container_of(work, struct amdgpu_ras, recovery_work); 2632 struct amdgpu_device *remote_adev = NULL; 2633 struct amdgpu_device *adev = ras->adev; 2634 struct list_head device_list, *device_list_handle = NULL; 2635 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2636 enum ras_event_type type; 2637 2638 if (hive) { 2639 atomic_set(&hive->ras_recovery, 1); 2640 2641 /* If any device which is part of the hive received RAS fatal 2642 * error interrupt, set fatal error status on all. This 2643 * condition will need a recovery, and flag will be cleared 2644 * as part of recovery. 2645 */ 2646 list_for_each_entry(remote_adev, &hive->device_list, 2647 gmc.xgmi.head) 2648 if (amdgpu_ras_get_fed_status(remote_adev)) { 2649 amdgpu_ras_set_fed_all(adev, hive, true); 2650 break; 2651 } 2652 } 2653 if (!ras->disable_ras_err_cnt_harvest) { 2654 2655 /* Build list of devices to query RAS related errors */ 2656 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2657 device_list_handle = &hive->device_list; 2658 } else { 2659 INIT_LIST_HEAD(&device_list); 2660 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2661 device_list_handle = &device_list; 2662 } 2663 2664 type = amdgpu_ras_get_fatal_error_event(adev); 2665 list_for_each_entry(remote_adev, 2666 device_list_handle, gmc.xgmi.head) { 2667 amdgpu_ras_query_err_status(remote_adev); 2668 amdgpu_ras_log_on_err_counter(remote_adev, type); 2669 } 2670 2671 } 2672 2673 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2674 struct amdgpu_reset_context reset_context; 2675 memset(&reset_context, 0, sizeof(reset_context)); 2676 2677 reset_context.method = AMD_RESET_METHOD_NONE; 2678 reset_context.reset_req_dev = adev; 2679 reset_context.src = AMDGPU_RESET_SRC_RAS; 2680 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2681 2682 /* Perform full reset in fatal error mode */ 2683 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2684 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2685 else { 2686 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2687 2688 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2689 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2690 reset_context.method = AMD_RESET_METHOD_MODE2; 2691 } 2692 2693 /* Fatal error occurs in poison mode, mode1 reset is used to 2694 * recover gpu. 2695 */ 2696 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2697 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2698 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2699 2700 psp_fatal_error_recovery_quirk(&adev->psp); 2701 } 2702 } 2703 2704 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2705 } 2706 atomic_set(&ras->in_recovery, 0); 2707 if (hive) { 2708 atomic_set(&hive->ras_recovery, 0); 2709 amdgpu_put_xgmi_hive(hive); 2710 } 2711 } 2712 2713 /* alloc/realloc bps array */ 2714 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2715 struct ras_err_handler_data *data, int pages) 2716 { 2717 unsigned int old_space = data->count + data->space_left; 2718 unsigned int new_space = old_space + pages; 2719 unsigned int align_space = ALIGN(new_space, 512); 2720 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL); 2721 2722 if (!bps) { 2723 return -ENOMEM; 2724 } 2725 2726 if (data->bps) { 2727 memcpy(bps, data->bps, 2728 data->count * sizeof(*data->bps)); 2729 kfree(data->bps); 2730 } 2731 2732 data->bps = bps; 2733 data->space_left += align_space - old_space; 2734 return 0; 2735 } 2736 2737 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2738 struct eeprom_table_record *bps, 2739 struct ras_err_data *err_data) 2740 { 2741 struct ta_ras_query_address_input addr_in; 2742 uint32_t socket = 0; 2743 int ret = 0; 2744 2745 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2746 socket = adev->smuio.funcs->get_socket_id(adev); 2747 2748 /* reinit err_data */ 2749 err_data->err_addr_cnt = 0; 2750 err_data->err_addr_len = adev->umc.retire_unit; 2751 2752 memset(&addr_in, 0, sizeof(addr_in)); 2753 addr_in.ma.err_addr = bps->address; 2754 addr_in.ma.socket_id = socket; 2755 addr_in.ma.ch_inst = bps->mem_channel; 2756 /* tell RAS TA the node instance is not used */ 2757 addr_in.ma.node_inst = TA_RAS_INV_NODE; 2758 2759 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2760 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 2761 &addr_in, NULL, false); 2762 2763 return ret; 2764 } 2765 2766 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 2767 struct eeprom_table_record *bps, 2768 struct ras_err_data *err_data) 2769 { 2770 struct ta_ras_query_address_input addr_in; 2771 uint32_t die_id, socket = 0; 2772 2773 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2774 socket = adev->smuio.funcs->get_socket_id(adev); 2775 2776 /* although die id is gotten from PA in nps1 mode, the id is 2777 * fitable for any nps mode 2778 */ 2779 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 2780 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 2781 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 2782 else 2783 return -EINVAL; 2784 2785 /* reinit err_data */ 2786 err_data->err_addr_cnt = 0; 2787 err_data->err_addr_len = adev->umc.retire_unit; 2788 2789 memset(&addr_in, 0, sizeof(addr_in)); 2790 addr_in.ma.err_addr = bps->address; 2791 addr_in.ma.ch_inst = bps->mem_channel; 2792 addr_in.ma.umc_inst = bps->mcumc_id; 2793 addr_in.ma.node_inst = die_id; 2794 addr_in.ma.socket_id = socket; 2795 2796 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2797 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 2798 &addr_in, NULL, false); 2799 else 2800 return -EINVAL; 2801 } 2802 2803 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 2804 struct eeprom_table_record *bps, int count) 2805 { 2806 int j; 2807 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2808 struct ras_err_handler_data *data = con->eh_data; 2809 2810 for (j = 0; j < count; j++) { 2811 if (amdgpu_ras_check_bad_page_unlock(con, 2812 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2813 continue; 2814 2815 if (!data->space_left && 2816 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2817 return -ENOMEM; 2818 } 2819 2820 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 2821 2822 memcpy(&data->bps[data->count], &(bps[j]), 2823 sizeof(struct eeprom_table_record)); 2824 data->count++; 2825 data->space_left--; 2826 } 2827 2828 return 0; 2829 } 2830 2831 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 2832 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2833 enum amdgpu_memory_partition nps) 2834 { 2835 int i = 0; 2836 enum amdgpu_memory_partition save_nps; 2837 2838 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2839 2840 /*old asics just have pa in eeprom*/ 2841 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2842 memcpy(err_data->err_addr, bps, 2843 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2844 goto out; 2845 } 2846 2847 for (i = 0; i < adev->umc.retire_unit; i++) 2848 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2849 2850 if (save_nps) { 2851 if (save_nps == nps) { 2852 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2853 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2854 return -EINVAL; 2855 } else { 2856 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 2857 return -EINVAL; 2858 } 2859 } else { 2860 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 2861 if (nps == AMDGPU_NPS1_PARTITION_MODE) 2862 memcpy(err_data->err_addr, bps, 2863 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2864 else 2865 return -EOPNOTSUPP; 2866 } 2867 } 2868 2869 out: 2870 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 2871 } 2872 2873 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 2874 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2875 enum amdgpu_memory_partition nps) 2876 { 2877 enum amdgpu_memory_partition save_nps; 2878 2879 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2880 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2881 2882 if (save_nps == nps) { 2883 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2884 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2885 return -EINVAL; 2886 } else { 2887 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 2888 return -EINVAL; 2889 } 2890 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 2891 adev->umc.retire_unit); 2892 } 2893 2894 /* it deal with vram only. */ 2895 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 2896 struct eeprom_table_record *bps, int pages, bool from_rom) 2897 { 2898 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2899 struct ras_err_data err_data; 2900 struct amdgpu_ras_eeprom_control *control = 2901 &adev->psp.ras_context.ras->eeprom_control; 2902 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 2903 int ret = 0; 2904 uint32_t i; 2905 2906 if (!con || !con->eh_data || !bps || pages <= 0) 2907 return 0; 2908 2909 if (from_rom) { 2910 err_data.err_addr = 2911 kcalloc(adev->umc.retire_unit, 2912 sizeof(struct eeprom_table_record), GFP_KERNEL); 2913 if (!err_data.err_addr) { 2914 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 2915 return -ENOMEM; 2916 } 2917 2918 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 2919 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 2920 } 2921 2922 mutex_lock(&con->recovery_lock); 2923 2924 if (from_rom) { 2925 for (i = 0; i < pages; i++) { 2926 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 2927 if ((bps[i].address == bps[i + 1].address) && 2928 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 2929 //deal with retire_unit records a time 2930 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 2931 &bps[i], &err_data, nps); 2932 if (ret) 2933 goto free; 2934 i += (adev->umc.retire_unit - 1); 2935 } else { 2936 break; 2937 } 2938 } else { 2939 break; 2940 } 2941 } 2942 for (; i < pages; i++) { 2943 ret = __amdgpu_ras_convert_rec_from_rom(adev, 2944 &bps[i], &err_data, nps); 2945 if (ret) 2946 goto free; 2947 } 2948 } else { 2949 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 2950 } 2951 2952 free: 2953 if (from_rom) 2954 kfree(err_data.err_addr); 2955 mutex_unlock(&con->recovery_lock); 2956 2957 return ret; 2958 } 2959 2960 /* 2961 * write error record array to eeprom, the function should be 2962 * protected by recovery_lock 2963 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 2964 */ 2965 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 2966 unsigned long *new_cnt) 2967 { 2968 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2969 struct ras_err_handler_data *data; 2970 struct amdgpu_ras_eeprom_control *control; 2971 int save_count, unit_num, bad_page_num, i; 2972 2973 if (!con || !con->eh_data) { 2974 if (new_cnt) 2975 *new_cnt = 0; 2976 2977 return 0; 2978 } 2979 2980 mutex_lock(&con->recovery_lock); 2981 control = &con->eeprom_control; 2982 data = con->eh_data; 2983 bad_page_num = control->ras_num_bad_pages; 2984 save_count = data->count - bad_page_num; 2985 mutex_unlock(&con->recovery_lock); 2986 2987 unit_num = save_count / adev->umc.retire_unit; 2988 if (new_cnt) 2989 *new_cnt = unit_num; 2990 2991 /* only new entries are saved */ 2992 if (save_count > 0) { 2993 /*old asics only save pa to eeprom like before*/ 2994 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2995 if (amdgpu_ras_eeprom_append(control, 2996 &data->bps[bad_page_num], save_count)) { 2997 dev_err(adev->dev, "Failed to save EEPROM table data!"); 2998 return -EIO; 2999 } 3000 } else { 3001 for (i = 0; i < unit_num; i++) { 3002 if (amdgpu_ras_eeprom_append(control, 3003 &data->bps[bad_page_num + 3004 i * adev->umc.retire_unit], 1)) { 3005 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3006 return -EIO; 3007 } 3008 } 3009 } 3010 3011 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3012 } 3013 3014 return 0; 3015 } 3016 3017 /* 3018 * read error record array in eeprom and reserve enough space for 3019 * storing new bad pages 3020 */ 3021 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3022 { 3023 struct amdgpu_ras_eeprom_control *control = 3024 &adev->psp.ras_context.ras->eeprom_control; 3025 struct eeprom_table_record *bps; 3026 int ret, i = 0; 3027 3028 /* no bad page record, skip eeprom access */ 3029 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3030 return 0; 3031 3032 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3033 if (!bps) 3034 return -ENOMEM; 3035 3036 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3037 if (ret) { 3038 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3039 } else { 3040 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3041 for (i = 0; i < control->ras_num_recs; i++) { 3042 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3043 if ((bps[i].address == bps[i + 1].address) && 3044 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3045 control->ras_num_pa_recs += adev->umc.retire_unit; 3046 i += (adev->umc.retire_unit - 1); 3047 } else { 3048 control->ras_num_mca_recs += 3049 (control->ras_num_recs - i); 3050 break; 3051 } 3052 } else { 3053 control->ras_num_mca_recs += (control->ras_num_recs - i); 3054 break; 3055 } 3056 } 3057 } 3058 3059 ret = amdgpu_ras_eeprom_check(control); 3060 if (ret) 3061 goto out; 3062 3063 /* HW not usable */ 3064 if (amdgpu_ras_is_rma(adev)) { 3065 ret = -EHWPOISON; 3066 goto out; 3067 } 3068 3069 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3070 } 3071 3072 out: 3073 kfree(bps); 3074 return ret; 3075 } 3076 3077 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3078 uint64_t addr) 3079 { 3080 struct ras_err_handler_data *data = con->eh_data; 3081 int i; 3082 3083 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3084 for (i = 0; i < data->count; i++) 3085 if (addr == data->bps[i].retired_page) 3086 return true; 3087 3088 return false; 3089 } 3090 3091 /* 3092 * check if an address belongs to bad page 3093 * 3094 * Note: this check is only for umc block 3095 */ 3096 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3097 uint64_t addr) 3098 { 3099 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3100 bool ret = false; 3101 3102 if (!con || !con->eh_data) 3103 return ret; 3104 3105 mutex_lock(&con->recovery_lock); 3106 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3107 mutex_unlock(&con->recovery_lock); 3108 return ret; 3109 } 3110 3111 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3112 uint32_t max_count) 3113 { 3114 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3115 3116 /* 3117 * amdgpu_bad_page_threshold is used to config 3118 * the threshold for the number of bad pages. 3119 * -1: Threshold is set to default value 3120 * Driver will issue a warning message when threshold is reached 3121 * and continue runtime services. 3122 * 0: Disable bad page retirement 3123 * Driver will not retire bad pages 3124 * which is intended for debugging purpose. 3125 * -2: Threshold is determined by a formula 3126 * that assumes 1 bad page per 100M of local memory. 3127 * Driver will continue runtime services when threhold is reached. 3128 * 0 < threshold < max number of bad page records in EEPROM, 3129 * A user-defined threshold is set 3130 * Driver will halt runtime services when this custom threshold is reached. 3131 */ 3132 if (amdgpu_bad_page_threshold == -2) { 3133 u64 val = adev->gmc.mc_vram_size; 3134 3135 do_div(val, RAS_BAD_PAGE_COVER); 3136 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3137 max_count); 3138 } else if (amdgpu_bad_page_threshold == -1) { 3139 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3140 } else { 3141 con->bad_page_cnt_threshold = min_t(int, max_count, 3142 amdgpu_bad_page_threshold); 3143 } 3144 } 3145 3146 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3147 enum amdgpu_ras_block block, uint16_t pasid, 3148 pasid_notify pasid_fn, void *data, uint32_t reset) 3149 { 3150 int ret = 0; 3151 struct ras_poison_msg poison_msg; 3152 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3153 3154 memset(&poison_msg, 0, sizeof(poison_msg)); 3155 poison_msg.block = block; 3156 poison_msg.pasid = pasid; 3157 poison_msg.reset = reset; 3158 poison_msg.pasid_fn = pasid_fn; 3159 poison_msg.data = data; 3160 3161 ret = kfifo_put(&con->poison_fifo, poison_msg); 3162 if (!ret) { 3163 dev_err(adev->dev, "Poison message fifo is full!\n"); 3164 return -ENOSPC; 3165 } 3166 3167 return 0; 3168 } 3169 3170 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3171 struct ras_poison_msg *poison_msg) 3172 { 3173 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3174 3175 return kfifo_get(&con->poison_fifo, poison_msg); 3176 } 3177 3178 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3179 { 3180 mutex_init(&ecc_log->lock); 3181 3182 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3183 ecc_log->de_queried_count = 0; 3184 ecc_log->prev_de_queried_count = 0; 3185 } 3186 3187 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3188 { 3189 struct radix_tree_iter iter; 3190 void __rcu **slot; 3191 struct ras_ecc_err *ecc_err; 3192 3193 mutex_lock(&ecc_log->lock); 3194 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3195 ecc_err = radix_tree_deref_slot(slot); 3196 kfree(ecc_err->err_pages.pfn); 3197 kfree(ecc_err); 3198 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3199 } 3200 mutex_unlock(&ecc_log->lock); 3201 3202 mutex_destroy(&ecc_log->lock); 3203 ecc_log->de_queried_count = 0; 3204 ecc_log->prev_de_queried_count = 0; 3205 } 3206 3207 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3208 uint32_t delayed_ms) 3209 { 3210 int ret; 3211 3212 mutex_lock(&con->umc_ecc_log.lock); 3213 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3214 UMC_ECC_NEW_DETECTED_TAG); 3215 mutex_unlock(&con->umc_ecc_log.lock); 3216 3217 if (ret) 3218 schedule_delayed_work(&con->page_retirement_dwork, 3219 msecs_to_jiffies(delayed_ms)); 3220 3221 return ret ? true : false; 3222 } 3223 3224 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3225 { 3226 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3227 page_retirement_dwork.work); 3228 struct amdgpu_device *adev = con->adev; 3229 struct ras_err_data err_data; 3230 unsigned long err_cnt; 3231 3232 /* If gpu reset is ongoing, delay retiring the bad pages */ 3233 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3234 amdgpu_ras_schedule_retirement_dwork(con, 3235 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3236 return; 3237 } 3238 3239 amdgpu_ras_error_data_init(&err_data); 3240 3241 amdgpu_umc_handle_bad_pages(adev, &err_data); 3242 err_cnt = err_data.err_addr_cnt; 3243 3244 amdgpu_ras_error_data_fini(&err_data); 3245 3246 if (err_cnt && amdgpu_ras_is_rma(adev)) 3247 amdgpu_ras_reset_gpu(adev); 3248 3249 amdgpu_ras_schedule_retirement_dwork(con, 3250 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3251 } 3252 3253 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3254 uint32_t poison_creation_count) 3255 { 3256 int ret = 0; 3257 struct ras_ecc_log_info *ecc_log; 3258 struct ras_query_if info; 3259 uint32_t timeout = 0; 3260 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3261 uint64_t de_queried_count; 3262 uint32_t new_detect_count, total_detect_count; 3263 uint32_t need_query_count = poison_creation_count; 3264 bool query_data_timeout = false; 3265 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3266 3267 memset(&info, 0, sizeof(info)); 3268 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3269 3270 ecc_log = &ras->umc_ecc_log; 3271 total_detect_count = 0; 3272 do { 3273 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3274 if (ret) 3275 return ret; 3276 3277 de_queried_count = ecc_log->de_queried_count; 3278 if (de_queried_count > ecc_log->prev_de_queried_count) { 3279 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count; 3280 ecc_log->prev_de_queried_count = de_queried_count; 3281 timeout = 0; 3282 } else { 3283 new_detect_count = 0; 3284 } 3285 3286 if (new_detect_count) { 3287 total_detect_count += new_detect_count; 3288 } else { 3289 if (!timeout && need_query_count) 3290 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3291 3292 if (timeout) { 3293 if (!--timeout) { 3294 query_data_timeout = true; 3295 break; 3296 } 3297 msleep(1); 3298 } 3299 } 3300 } while (total_detect_count < need_query_count); 3301 3302 if (query_data_timeout) { 3303 dev_warn(adev->dev, "Can't find deferred error! count: %u\n", 3304 (need_query_count - total_detect_count)); 3305 return -ENOENT; 3306 } 3307 3308 if (total_detect_count) 3309 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3310 3311 return 0; 3312 } 3313 3314 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3315 { 3316 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3317 struct ras_poison_msg msg; 3318 int ret; 3319 3320 do { 3321 ret = kfifo_get(&con->poison_fifo, &msg); 3322 } while (ret); 3323 } 3324 3325 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3326 uint32_t msg_count, uint32_t *gpu_reset) 3327 { 3328 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3329 uint32_t reset_flags = 0, reset = 0; 3330 struct ras_poison_msg msg; 3331 int ret, i; 3332 3333 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3334 3335 for (i = 0; i < msg_count; i++) { 3336 ret = amdgpu_ras_get_poison_req(adev, &msg); 3337 if (!ret) 3338 continue; 3339 3340 if (msg.pasid_fn) 3341 msg.pasid_fn(adev, msg.pasid, msg.data); 3342 3343 reset_flags |= msg.reset; 3344 } 3345 3346 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3347 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3348 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3349 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3350 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3351 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3352 else 3353 reset = reset_flags; 3354 3355 flush_delayed_work(&con->page_retirement_dwork); 3356 3357 con->gpu_reset_flags |= reset; 3358 amdgpu_ras_reset_gpu(adev); 3359 3360 *gpu_reset = reset; 3361 3362 /* Wait for gpu recovery to complete */ 3363 flush_work(&con->recovery_work); 3364 } 3365 3366 return 0; 3367 } 3368 3369 static int amdgpu_ras_page_retirement_thread(void *param) 3370 { 3371 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3372 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3373 uint32_t poison_creation_count, msg_count; 3374 uint32_t gpu_reset; 3375 int ret; 3376 3377 while (!kthread_should_stop()) { 3378 3379 wait_event_interruptible(con->page_retirement_wq, 3380 kthread_should_stop() || 3381 atomic_read(&con->page_retirement_req_cnt)); 3382 3383 if (kthread_should_stop()) 3384 break; 3385 3386 gpu_reset = 0; 3387 3388 do { 3389 poison_creation_count = atomic_read(&con->poison_creation_count); 3390 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3391 if (ret == -EIO) 3392 break; 3393 3394 if (poison_creation_count) { 3395 atomic_sub(poison_creation_count, &con->poison_creation_count); 3396 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3397 } 3398 } while (atomic_read(&con->poison_creation_count)); 3399 3400 if (ret != -EIO) { 3401 msg_count = kfifo_len(&con->poison_fifo); 3402 if (msg_count) { 3403 ret = amdgpu_ras_poison_consumption_handler(adev, 3404 msg_count, &gpu_reset); 3405 if ((ret != -EIO) && 3406 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3407 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3408 } 3409 } 3410 3411 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3412 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3413 /* Clear poison creation request */ 3414 atomic_set(&con->poison_creation_count, 0); 3415 3416 /* Clear poison fifo */ 3417 amdgpu_ras_clear_poison_fifo(adev); 3418 3419 /* Clear all poison requests */ 3420 atomic_set(&con->page_retirement_req_cnt, 0); 3421 3422 if (ret == -EIO) { 3423 /* Wait for mode-1 reset to complete */ 3424 down_read(&adev->reset_domain->sem); 3425 up_read(&adev->reset_domain->sem); 3426 } 3427 3428 /* Wake up work to save bad pages to eeprom */ 3429 schedule_delayed_work(&con->page_retirement_dwork, 0); 3430 } else if (gpu_reset) { 3431 /* gpu just completed mode-2 reset or other reset */ 3432 /* Clear poison consumption messages cached in fifo */ 3433 msg_count = kfifo_len(&con->poison_fifo); 3434 if (msg_count) { 3435 amdgpu_ras_clear_poison_fifo(adev); 3436 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3437 } 3438 3439 /* Wake up work to save bad pages to eeprom */ 3440 schedule_delayed_work(&con->page_retirement_dwork, 0); 3441 } 3442 } 3443 3444 return 0; 3445 } 3446 3447 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3448 { 3449 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3450 struct amdgpu_ras_eeprom_control *control; 3451 int ret; 3452 3453 if (!con || amdgpu_sriov_vf(adev)) 3454 return 0; 3455 3456 control = &con->eeprom_control; 3457 ret = amdgpu_ras_eeprom_init(control); 3458 if (ret) 3459 return ret; 3460 3461 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3462 control->ras_num_pa_recs = control->ras_num_recs; 3463 3464 if (control->ras_num_recs) { 3465 ret = amdgpu_ras_load_bad_pages(adev); 3466 if (ret) 3467 return ret; 3468 3469 amdgpu_dpm_send_hbm_bad_pages_num( 3470 adev, control->ras_num_bad_pages); 3471 3472 if (con->update_channel_flag == true) { 3473 amdgpu_dpm_send_hbm_bad_channel_flag( 3474 adev, control->bad_channel_bitmap); 3475 con->update_channel_flag = false; 3476 } 3477 3478 /* The format action is only applied to new ASICs */ 3479 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3480 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3481 if (!amdgpu_ras_eeprom_reset_table(control)) 3482 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3483 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3484 } 3485 3486 return ret; 3487 } 3488 3489 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3490 { 3491 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3492 struct ras_err_handler_data **data; 3493 u32 max_eeprom_records_count = 0; 3494 int ret; 3495 3496 if (!con || amdgpu_sriov_vf(adev)) 3497 return 0; 3498 3499 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3500 * supports RAS and debugfs is enabled, but when 3501 * adev->ras_enabled is unset, i.e. when "ras_enable" 3502 * module parameter is set to 0. 3503 */ 3504 con->adev = adev; 3505 3506 if (!adev->ras_enabled) 3507 return 0; 3508 3509 data = &con->eh_data; 3510 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3511 if (!*data) { 3512 ret = -ENOMEM; 3513 goto out; 3514 } 3515 3516 mutex_init(&con->recovery_lock); 3517 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3518 atomic_set(&con->in_recovery, 0); 3519 con->eeprom_control.bad_channel_bitmap = 0; 3520 3521 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3522 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3523 3524 if (init_bp_info) { 3525 ret = amdgpu_ras_init_badpage_info(adev); 3526 if (ret) 3527 goto free; 3528 } 3529 3530 mutex_init(&con->page_rsv_lock); 3531 INIT_KFIFO(con->poison_fifo); 3532 mutex_init(&con->page_retirement_lock); 3533 init_waitqueue_head(&con->page_retirement_wq); 3534 atomic_set(&con->page_retirement_req_cnt, 0); 3535 atomic_set(&con->poison_creation_count, 0); 3536 con->page_retirement_thread = 3537 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3538 if (IS_ERR(con->page_retirement_thread)) { 3539 con->page_retirement_thread = NULL; 3540 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3541 } 3542 3543 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3544 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3545 #ifdef CONFIG_X86_MCE_AMD 3546 if ((adev->asic_type == CHIP_ALDEBARAN) && 3547 (adev->gmc.xgmi.connected_to_cpu)) 3548 amdgpu_register_bad_pages_mca_notifier(adev); 3549 #endif 3550 return 0; 3551 3552 free: 3553 kfree((*data)->bps); 3554 kfree(*data); 3555 con->eh_data = NULL; 3556 out: 3557 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3558 3559 /* 3560 * Except error threshold exceeding case, other failure cases in this 3561 * function would not fail amdgpu driver init. 3562 */ 3563 if (!amdgpu_ras_is_rma(adev)) 3564 ret = 0; 3565 else 3566 ret = -EINVAL; 3567 3568 return ret; 3569 } 3570 3571 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3572 { 3573 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3574 struct ras_err_handler_data *data = con->eh_data; 3575 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3576 bool ret; 3577 3578 /* recovery_init failed to init it, fini is useless */ 3579 if (!data) 3580 return 0; 3581 3582 /* Save all cached bad pages to eeprom */ 3583 do { 3584 flush_delayed_work(&con->page_retirement_dwork); 3585 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3586 } while (ret && max_flush_timeout--); 3587 3588 if (con->page_retirement_thread) 3589 kthread_stop(con->page_retirement_thread); 3590 3591 atomic_set(&con->page_retirement_req_cnt, 0); 3592 atomic_set(&con->poison_creation_count, 0); 3593 3594 mutex_destroy(&con->page_rsv_lock); 3595 3596 cancel_work_sync(&con->recovery_work); 3597 3598 cancel_delayed_work_sync(&con->page_retirement_dwork); 3599 3600 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3601 3602 mutex_lock(&con->recovery_lock); 3603 con->eh_data = NULL; 3604 kfree(data->bps); 3605 kfree(data); 3606 mutex_unlock(&con->recovery_lock); 3607 3608 return 0; 3609 } 3610 /* recovery end */ 3611 3612 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3613 { 3614 if (amdgpu_sriov_vf(adev)) { 3615 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3616 case IP_VERSION(13, 0, 2): 3617 case IP_VERSION(13, 0, 6): 3618 case IP_VERSION(13, 0, 12): 3619 case IP_VERSION(13, 0, 14): 3620 return true; 3621 default: 3622 return false; 3623 } 3624 } 3625 3626 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3627 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3628 case IP_VERSION(13, 0, 0): 3629 case IP_VERSION(13, 0, 6): 3630 case IP_VERSION(13, 0, 10): 3631 case IP_VERSION(13, 0, 12): 3632 case IP_VERSION(13, 0, 14): 3633 case IP_VERSION(14, 0, 3): 3634 return true; 3635 default: 3636 return false; 3637 } 3638 } 3639 3640 return adev->asic_type == CHIP_VEGA10 || 3641 adev->asic_type == CHIP_VEGA20 || 3642 adev->asic_type == CHIP_ARCTURUS || 3643 adev->asic_type == CHIP_ALDEBARAN || 3644 adev->asic_type == CHIP_SIENNA_CICHLID; 3645 } 3646 3647 /* 3648 * this is workaround for vega20 workstation sku, 3649 * force enable gfx ras, ignore vbios gfx ras flag 3650 * due to GC EDC can not write 3651 */ 3652 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3653 { 3654 struct atom_context *ctx = adev->mode_info.atom_context; 3655 3656 if (!ctx) 3657 return; 3658 3659 if (strnstr(ctx->vbios_pn, "D16406", 3660 sizeof(ctx->vbios_pn)) || 3661 strnstr(ctx->vbios_pn, "D36002", 3662 sizeof(ctx->vbios_pn))) 3663 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3664 } 3665 3666 /* Query ras capablity via atomfirmware interface */ 3667 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3668 { 3669 /* mem_ecc cap */ 3670 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3671 dev_info(adev->dev, "MEM ECC is active.\n"); 3672 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3673 1 << AMDGPU_RAS_BLOCK__DF); 3674 } else { 3675 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3676 } 3677 3678 /* sram_ecc cap */ 3679 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 3680 dev_info(adev->dev, "SRAM ECC is active.\n"); 3681 if (!amdgpu_sriov_vf(adev)) 3682 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 3683 1 << AMDGPU_RAS_BLOCK__DF); 3684 else 3685 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 3686 1 << AMDGPU_RAS_BLOCK__SDMA | 3687 1 << AMDGPU_RAS_BLOCK__GFX); 3688 3689 /* 3690 * VCN/JPEG RAS can be supported on both bare metal and 3691 * SRIOV environment 3692 */ 3693 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 3694 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 3695 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3)) 3696 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 3697 1 << AMDGPU_RAS_BLOCK__JPEG); 3698 else 3699 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 3700 1 << AMDGPU_RAS_BLOCK__JPEG); 3701 3702 /* 3703 * XGMI RAS is not supported if xgmi num physical nodes 3704 * is zero 3705 */ 3706 if (!adev->gmc.xgmi.num_physical_nodes) 3707 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 3708 } else { 3709 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 3710 } 3711 } 3712 3713 /* Query poison mode from umc/df IP callbacks */ 3714 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 3715 { 3716 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3717 bool df_poison, umc_poison; 3718 3719 /* poison setting is useless on SRIOV guest */ 3720 if (amdgpu_sriov_vf(adev) || !con) 3721 return; 3722 3723 /* Init poison supported flag, the default value is false */ 3724 if (adev->gmc.xgmi.connected_to_cpu || 3725 adev->gmc.is_app_apu) { 3726 /* enabled by default when GPU is connected to CPU */ 3727 con->poison_supported = true; 3728 } else if (adev->df.funcs && 3729 adev->df.funcs->query_ras_poison_mode && 3730 adev->umc.ras && 3731 adev->umc.ras->query_ras_poison_mode) { 3732 df_poison = 3733 adev->df.funcs->query_ras_poison_mode(adev); 3734 umc_poison = 3735 adev->umc.ras->query_ras_poison_mode(adev); 3736 3737 /* Only poison is set in both DF and UMC, we can support it */ 3738 if (df_poison && umc_poison) 3739 con->poison_supported = true; 3740 else if (df_poison != umc_poison) 3741 dev_warn(adev->dev, 3742 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 3743 df_poison, umc_poison); 3744 } 3745 } 3746 3747 /* 3748 * check hardware's ras ability which will be saved in hw_supported. 3749 * if hardware does not support ras, we can skip some ras initializtion and 3750 * forbid some ras operations from IP. 3751 * if software itself, say boot parameter, limit the ras ability. We still 3752 * need allow IP do some limited operations, like disable. In such case, 3753 * we have to initialize ras as normal. but need check if operation is 3754 * allowed or not in each function. 3755 */ 3756 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 3757 { 3758 adev->ras_hw_enabled = adev->ras_enabled = 0; 3759 3760 if (!amdgpu_ras_asic_supported(adev)) 3761 return; 3762 3763 if (amdgpu_sriov_vf(adev)) { 3764 if (amdgpu_virt_get_ras_capability(adev)) 3765 goto init_ras_enabled_flag; 3766 } 3767 3768 /* query ras capability from psp */ 3769 if (amdgpu_psp_get_ras_capability(&adev->psp)) 3770 goto init_ras_enabled_flag; 3771 3772 /* query ras capablity from bios */ 3773 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 3774 amdgpu_ras_query_ras_capablity_from_vbios(adev); 3775 } else { 3776 /* driver only manages a few IP blocks RAS feature 3777 * when GPU is connected cpu through XGMI */ 3778 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 3779 1 << AMDGPU_RAS_BLOCK__SDMA | 3780 1 << AMDGPU_RAS_BLOCK__MMHUB); 3781 } 3782 3783 /* apply asic specific settings (vega20 only for now) */ 3784 amdgpu_ras_get_quirks(adev); 3785 3786 /* query poison mode from umc/df ip callback */ 3787 amdgpu_ras_query_poison_mode(adev); 3788 3789 init_ras_enabled_flag: 3790 /* hw_supported needs to be aligned with RAS block mask. */ 3791 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 3792 3793 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 3794 adev->ras_hw_enabled & amdgpu_ras_mask; 3795 3796 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 3797 adev->aca.is_enabled = 3798 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 3799 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 3800 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 3801 3802 /* bad page feature is not applicable to specific app platform */ 3803 if (adev->gmc.is_app_apu && 3804 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 3805 amdgpu_bad_page_threshold = 0; 3806 } 3807 3808 static void amdgpu_ras_counte_dw(struct work_struct *work) 3809 { 3810 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3811 ras_counte_delay_work.work); 3812 struct amdgpu_device *adev = con->adev; 3813 struct drm_device *dev = adev_to_drm(adev); 3814 unsigned long ce_count, ue_count; 3815 int res; 3816 3817 res = pm_runtime_get_sync(dev->dev); 3818 if (res < 0) 3819 goto Out; 3820 3821 /* Cache new values. 3822 */ 3823 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 3824 atomic_set(&con->ras_ce_count, ce_count); 3825 atomic_set(&con->ras_ue_count, ue_count); 3826 } 3827 3828 pm_runtime_mark_last_busy(dev->dev); 3829 Out: 3830 pm_runtime_put_autosuspend(dev->dev); 3831 } 3832 3833 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 3834 { 3835 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 3836 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 3837 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 3838 AMDGPU_RAS_ERROR__PARITY; 3839 } 3840 3841 static void ras_event_mgr_init(struct ras_event_manager *mgr) 3842 { 3843 struct ras_event_state *event_state; 3844 int i; 3845 3846 memset(mgr, 0, sizeof(*mgr)); 3847 atomic64_set(&mgr->seqno, 0); 3848 3849 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 3850 event_state = &mgr->event_state[i]; 3851 event_state->last_seqno = RAS_EVENT_INVALID_ID; 3852 atomic64_set(&event_state->count, 0); 3853 } 3854 } 3855 3856 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 3857 { 3858 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3859 struct amdgpu_hive_info *hive; 3860 3861 if (!ras) 3862 return; 3863 3864 hive = amdgpu_get_xgmi_hive(adev); 3865 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 3866 3867 /* init event manager with node 0 on xgmi system */ 3868 if (!amdgpu_reset_in_recovery(adev)) { 3869 if (!hive || adev->gmc.xgmi.node_id == 0) 3870 ras_event_mgr_init(ras->event_mgr); 3871 } 3872 3873 if (hive) 3874 amdgpu_put_xgmi_hive(hive); 3875 } 3876 3877 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 3878 { 3879 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3880 3881 if (!con || (adev->flags & AMD_IS_APU)) 3882 return; 3883 3884 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3885 case IP_VERSION(13, 0, 2): 3886 case IP_VERSION(13, 0, 6): 3887 case IP_VERSION(13, 0, 12): 3888 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 3889 break; 3890 case IP_VERSION(13, 0, 14): 3891 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 3892 break; 3893 default: 3894 break; 3895 } 3896 } 3897 3898 int amdgpu_ras_init(struct amdgpu_device *adev) 3899 { 3900 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3901 int r; 3902 3903 if (con) 3904 return 0; 3905 3906 con = kzalloc(sizeof(*con) + 3907 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 3908 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 3909 GFP_KERNEL); 3910 if (!con) 3911 return -ENOMEM; 3912 3913 con->adev = adev; 3914 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 3915 atomic_set(&con->ras_ce_count, 0); 3916 atomic_set(&con->ras_ue_count, 0); 3917 3918 con->objs = (struct ras_manager *)(con + 1); 3919 3920 amdgpu_ras_set_context(adev, con); 3921 3922 amdgpu_ras_check_supported(adev); 3923 3924 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 3925 /* set gfx block ras context feature for VEGA20 Gaming 3926 * send ras disable cmd to ras ta during ras late init. 3927 */ 3928 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 3929 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 3930 3931 return 0; 3932 } 3933 3934 r = 0; 3935 goto release_con; 3936 } 3937 3938 con->update_channel_flag = false; 3939 con->features = 0; 3940 con->schema = 0; 3941 INIT_LIST_HEAD(&con->head); 3942 /* Might need get this flag from vbios. */ 3943 con->flags = RAS_DEFAULT_FLAGS; 3944 3945 /* initialize nbio ras function ahead of any other 3946 * ras functions so hardware fatal error interrupt 3947 * can be enabled as early as possible */ 3948 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 3949 case IP_VERSION(7, 4, 0): 3950 case IP_VERSION(7, 4, 1): 3951 case IP_VERSION(7, 4, 4): 3952 if (!adev->gmc.xgmi.connected_to_cpu) 3953 adev->nbio.ras = &nbio_v7_4_ras; 3954 break; 3955 case IP_VERSION(4, 3, 0): 3956 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 3957 /* unlike other generation of nbio ras, 3958 * nbio v4_3 only support fatal error interrupt 3959 * to inform software that DF is freezed due to 3960 * system fatal error event. driver should not 3961 * enable nbio ras in such case. Instead, 3962 * check DF RAS */ 3963 adev->nbio.ras = &nbio_v4_3_ras; 3964 break; 3965 case IP_VERSION(6, 3, 1): 3966 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 3967 /* unlike other generation of nbio ras, 3968 * nbif v6_3_1 only support fatal error interrupt 3969 * to inform software that DF is freezed due to 3970 * system fatal error event. driver should not 3971 * enable nbio ras in such case. Instead, 3972 * check DF RAS 3973 */ 3974 adev->nbio.ras = &nbif_v6_3_1_ras; 3975 break; 3976 case IP_VERSION(7, 9, 0): 3977 case IP_VERSION(7, 9, 1): 3978 if (!adev->gmc.is_app_apu) 3979 adev->nbio.ras = &nbio_v7_9_ras; 3980 break; 3981 default: 3982 /* nbio ras is not available */ 3983 break; 3984 } 3985 3986 /* nbio ras block needs to be enabled ahead of other ras blocks 3987 * to handle fatal error */ 3988 r = amdgpu_nbio_ras_sw_init(adev); 3989 if (r) 3990 return r; 3991 3992 if (adev->nbio.ras && 3993 adev->nbio.ras->init_ras_controller_interrupt) { 3994 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 3995 if (r) 3996 goto release_con; 3997 } 3998 3999 if (adev->nbio.ras && 4000 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4001 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4002 if (r) 4003 goto release_con; 4004 } 4005 4006 /* Packed socket_id to ras feature mask bits[31:29] */ 4007 if (adev->smuio.funcs && 4008 adev->smuio.funcs->get_socket_id) 4009 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4010 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4011 4012 /* Get RAS schema for particular SOC */ 4013 con->schema = amdgpu_get_ras_schema(adev); 4014 4015 amdgpu_ras_init_reserved_vram_size(adev); 4016 4017 if (amdgpu_ras_fs_init(adev)) { 4018 r = -EINVAL; 4019 goto release_con; 4020 } 4021 4022 if (amdgpu_ras_aca_is_supported(adev)) { 4023 if (amdgpu_aca_is_enabled(adev)) 4024 r = amdgpu_aca_init(adev); 4025 else 4026 r = amdgpu_mca_init(adev); 4027 if (r) 4028 goto release_con; 4029 } 4030 4031 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4032 "hardware ability[%x] ras_mask[%x]\n", 4033 adev->ras_hw_enabled, adev->ras_enabled); 4034 4035 return 0; 4036 release_con: 4037 amdgpu_ras_set_context(adev, NULL); 4038 kfree(con); 4039 4040 return r; 4041 } 4042 4043 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4044 { 4045 if (adev->gmc.xgmi.connected_to_cpu || 4046 adev->gmc.is_app_apu) 4047 return 1; 4048 return 0; 4049 } 4050 4051 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4052 struct ras_common_if *ras_block) 4053 { 4054 struct ras_query_if info = { 4055 .head = *ras_block, 4056 }; 4057 4058 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4059 return 0; 4060 4061 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4062 DRM_WARN("RAS init harvest failure"); 4063 4064 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4065 DRM_WARN("RAS init harvest reset failure"); 4066 4067 return 0; 4068 } 4069 4070 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4071 { 4072 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4073 4074 if (!con) 4075 return false; 4076 4077 return con->poison_supported; 4078 } 4079 4080 /* helper function to handle common stuff in ip late init phase */ 4081 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4082 struct ras_common_if *ras_block) 4083 { 4084 struct amdgpu_ras_block_object *ras_obj = NULL; 4085 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4086 struct ras_query_if *query_info; 4087 unsigned long ue_count, ce_count; 4088 int r; 4089 4090 /* disable RAS feature per IP block if it is not supported */ 4091 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4092 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4093 return 0; 4094 } 4095 4096 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4097 if (r) { 4098 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4099 /* in resume phase, if fail to enable ras, 4100 * clean up all ras fs nodes, and disable ras */ 4101 goto cleanup; 4102 } else 4103 return r; 4104 } 4105 4106 /* check for errors on warm reset edc persisant supported ASIC */ 4107 amdgpu_persistent_edc_harvesting(adev, ras_block); 4108 4109 /* in resume phase, no need to create ras fs node */ 4110 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4111 return 0; 4112 4113 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4114 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4115 (ras_obj->hw_ops->query_poison_status || 4116 ras_obj->hw_ops->handle_poison_consumption))) { 4117 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4118 if (r) 4119 goto cleanup; 4120 } 4121 4122 if (ras_obj->hw_ops && 4123 (ras_obj->hw_ops->query_ras_error_count || 4124 ras_obj->hw_ops->query_ras_error_status)) { 4125 r = amdgpu_ras_sysfs_create(adev, ras_block); 4126 if (r) 4127 goto interrupt; 4128 4129 /* Those are the cached values at init. 4130 */ 4131 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4132 if (!query_info) 4133 return -ENOMEM; 4134 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4135 4136 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4137 atomic_set(&con->ras_ce_count, ce_count); 4138 atomic_set(&con->ras_ue_count, ue_count); 4139 } 4140 4141 kfree(query_info); 4142 } 4143 4144 return 0; 4145 4146 interrupt: 4147 if (ras_obj->ras_cb) 4148 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4149 cleanup: 4150 amdgpu_ras_feature_enable(adev, ras_block, 0); 4151 return r; 4152 } 4153 4154 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4155 struct ras_common_if *ras_block) 4156 { 4157 return amdgpu_ras_block_late_init(adev, ras_block); 4158 } 4159 4160 /* helper function to remove ras fs node and interrupt handler */ 4161 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4162 struct ras_common_if *ras_block) 4163 { 4164 struct amdgpu_ras_block_object *ras_obj; 4165 if (!ras_block) 4166 return; 4167 4168 amdgpu_ras_sysfs_remove(adev, ras_block); 4169 4170 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4171 if (ras_obj->ras_cb) 4172 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4173 } 4174 4175 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4176 struct ras_common_if *ras_block) 4177 { 4178 return amdgpu_ras_block_late_fini(adev, ras_block); 4179 } 4180 4181 /* do some init work after IP late init as dependence. 4182 * and it runs in resume/gpu reset/booting up cases. 4183 */ 4184 void amdgpu_ras_resume(struct amdgpu_device *adev) 4185 { 4186 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4187 struct ras_manager *obj, *tmp; 4188 4189 if (!adev->ras_enabled || !con) { 4190 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4191 amdgpu_release_ras_context(adev); 4192 4193 return; 4194 } 4195 4196 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4197 /* Set up all other IPs which are not implemented. There is a 4198 * tricky thing that IP's actual ras error type should be 4199 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4200 * ERROR_NONE make sense anyway. 4201 */ 4202 amdgpu_ras_enable_all_features(adev, 1); 4203 4204 /* We enable ras on all hw_supported block, but as boot 4205 * parameter might disable some of them and one or more IP has 4206 * not implemented yet. So we disable them on behalf. 4207 */ 4208 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4209 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4210 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4211 /* there should be no any reference. */ 4212 WARN_ON(alive_obj(obj)); 4213 } 4214 } 4215 } 4216 } 4217 4218 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4219 { 4220 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4221 4222 if (!adev->ras_enabled || !con) 4223 return; 4224 4225 amdgpu_ras_disable_all_features(adev, 0); 4226 /* Make sure all ras objects are disabled. */ 4227 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4228 amdgpu_ras_disable_all_features(adev, 1); 4229 } 4230 4231 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4232 { 4233 struct amdgpu_ras_block_list *node, *tmp; 4234 struct amdgpu_ras_block_object *obj; 4235 int r; 4236 4237 amdgpu_ras_event_mgr_init(adev); 4238 4239 if (amdgpu_ras_aca_is_supported(adev)) { 4240 if (amdgpu_reset_in_recovery(adev)) { 4241 if (amdgpu_aca_is_enabled(adev)) 4242 r = amdgpu_aca_reset(adev); 4243 else 4244 r = amdgpu_mca_reset(adev); 4245 if (r) 4246 return r; 4247 } 4248 4249 if (!amdgpu_sriov_vf(adev)) { 4250 if (amdgpu_aca_is_enabled(adev)) 4251 amdgpu_ras_set_aca_debug_mode(adev, false); 4252 else 4253 amdgpu_ras_set_mca_debug_mode(adev, false); 4254 } 4255 } 4256 4257 /* Guest side doesn't need init ras feature */ 4258 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4259 return 0; 4260 4261 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4262 obj = node->ras_obj; 4263 if (!obj) { 4264 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4265 continue; 4266 } 4267 4268 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4269 continue; 4270 4271 if (obj->ras_late_init) { 4272 r = obj->ras_late_init(adev, &obj->ras_comm); 4273 if (r) { 4274 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4275 obj->ras_comm.name, r); 4276 return r; 4277 } 4278 } else 4279 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4280 } 4281 4282 return 0; 4283 } 4284 4285 /* do some fini work before IP fini as dependence */ 4286 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4287 { 4288 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4289 4290 if (!adev->ras_enabled || !con) 4291 return 0; 4292 4293 4294 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4295 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4296 amdgpu_ras_disable_all_features(adev, 0); 4297 amdgpu_ras_recovery_fini(adev); 4298 return 0; 4299 } 4300 4301 int amdgpu_ras_fini(struct amdgpu_device *adev) 4302 { 4303 struct amdgpu_ras_block_list *ras_node, *tmp; 4304 struct amdgpu_ras_block_object *obj = NULL; 4305 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4306 4307 if (!adev->ras_enabled || !con) 4308 return 0; 4309 4310 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4311 if (ras_node->ras_obj) { 4312 obj = ras_node->ras_obj; 4313 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4314 obj->ras_fini) 4315 obj->ras_fini(adev, &obj->ras_comm); 4316 else 4317 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4318 } 4319 4320 /* Clear ras blocks from ras_list and free ras block list node */ 4321 list_del(&ras_node->node); 4322 kfree(ras_node); 4323 } 4324 4325 amdgpu_ras_fs_fini(adev); 4326 amdgpu_ras_interrupt_remove_all(adev); 4327 4328 if (amdgpu_ras_aca_is_supported(adev)) { 4329 if (amdgpu_aca_is_enabled(adev)) 4330 amdgpu_aca_fini(adev); 4331 else 4332 amdgpu_mca_fini(adev); 4333 } 4334 4335 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4336 4337 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4338 amdgpu_ras_disable_all_features(adev, 0); 4339 4340 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4341 4342 amdgpu_ras_set_context(adev, NULL); 4343 kfree(con); 4344 4345 return 0; 4346 } 4347 4348 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4349 { 4350 struct amdgpu_ras *ras; 4351 4352 ras = amdgpu_ras_get_context(adev); 4353 if (!ras) 4354 return false; 4355 4356 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4357 } 4358 4359 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4360 { 4361 struct amdgpu_ras *ras; 4362 4363 ras = amdgpu_ras_get_context(adev); 4364 if (ras) { 4365 if (status) 4366 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4367 else 4368 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4369 } 4370 } 4371 4372 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4373 { 4374 struct amdgpu_ras *ras; 4375 4376 ras = amdgpu_ras_get_context(adev); 4377 if (ras) 4378 ras->ras_err_state = 0; 4379 } 4380 4381 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4382 enum amdgpu_ras_block block) 4383 { 4384 struct amdgpu_ras *ras; 4385 4386 ras = amdgpu_ras_get_context(adev); 4387 if (ras) 4388 set_bit(block, &ras->ras_err_state); 4389 } 4390 4391 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4392 { 4393 struct amdgpu_ras *ras; 4394 4395 ras = amdgpu_ras_get_context(adev); 4396 if (ras) { 4397 if (block == AMDGPU_RAS_BLOCK__ANY) 4398 return (ras->ras_err_state != 0); 4399 else 4400 return test_bit(block, &ras->ras_err_state) || 4401 test_bit(AMDGPU_RAS_BLOCK__LAST, 4402 &ras->ras_err_state); 4403 } 4404 4405 return false; 4406 } 4407 4408 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4409 { 4410 struct amdgpu_ras *ras; 4411 4412 ras = amdgpu_ras_get_context(adev); 4413 if (!ras) 4414 return NULL; 4415 4416 return ras->event_mgr; 4417 } 4418 4419 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4420 const void *caller) 4421 { 4422 struct ras_event_manager *event_mgr; 4423 struct ras_event_state *event_state; 4424 int ret = 0; 4425 4426 if (type >= RAS_EVENT_TYPE_COUNT) { 4427 ret = -EINVAL; 4428 goto out; 4429 } 4430 4431 event_mgr = __get_ras_event_mgr(adev); 4432 if (!event_mgr) { 4433 ret = -EINVAL; 4434 goto out; 4435 } 4436 4437 event_state = &event_mgr->event_state[type]; 4438 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4439 atomic64_inc(&event_state->count); 4440 4441 out: 4442 if (ret && caller) 4443 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4444 (int)type, caller, ret); 4445 4446 return ret; 4447 } 4448 4449 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4450 { 4451 struct ras_event_manager *event_mgr; 4452 u64 id; 4453 4454 if (type >= RAS_EVENT_TYPE_COUNT) 4455 return RAS_EVENT_INVALID_ID; 4456 4457 switch (type) { 4458 case RAS_EVENT_TYPE_FATAL: 4459 case RAS_EVENT_TYPE_POISON_CREATION: 4460 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4461 event_mgr = __get_ras_event_mgr(adev); 4462 if (!event_mgr) 4463 return RAS_EVENT_INVALID_ID; 4464 4465 id = event_mgr->event_state[type].last_seqno; 4466 break; 4467 case RAS_EVENT_TYPE_INVALID: 4468 default: 4469 id = RAS_EVENT_INVALID_ID; 4470 break; 4471 } 4472 4473 return id; 4474 } 4475 4476 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4477 { 4478 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4479 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4480 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4481 u64 event_id; 4482 4483 if (amdgpu_ras_mark_ras_event(adev, type)) 4484 return; 4485 4486 event_id = amdgpu_ras_acquire_event_id(adev, type); 4487 4488 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4489 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4490 4491 amdgpu_ras_set_fed(adev, true); 4492 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4493 amdgpu_ras_reset_gpu(adev); 4494 } 4495 } 4496 4497 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4498 { 4499 if (adev->asic_type == CHIP_VEGA20 && 4500 adev->pm.fw_version <= 0x283400) { 4501 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4502 amdgpu_ras_intr_triggered(); 4503 } 4504 4505 return false; 4506 } 4507 4508 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4509 { 4510 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4511 4512 if (!con) 4513 return; 4514 4515 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4516 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4517 amdgpu_ras_set_context(adev, NULL); 4518 kfree(con); 4519 } 4520 } 4521 4522 #ifdef CONFIG_X86_MCE_AMD 4523 static struct amdgpu_device *find_adev(uint32_t node_id) 4524 { 4525 int i; 4526 struct amdgpu_device *adev = NULL; 4527 4528 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4529 adev = mce_adev_list.devs[i]; 4530 4531 if (adev && adev->gmc.xgmi.connected_to_cpu && 4532 adev->gmc.xgmi.physical_node_id == node_id) 4533 break; 4534 adev = NULL; 4535 } 4536 4537 return adev; 4538 } 4539 4540 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4541 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4542 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4543 #define GPU_ID_OFFSET 8 4544 4545 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4546 unsigned long val, void *data) 4547 { 4548 struct mce *m = (struct mce *)data; 4549 struct amdgpu_device *adev = NULL; 4550 uint32_t gpu_id = 0; 4551 uint32_t umc_inst = 0, ch_inst = 0; 4552 4553 /* 4554 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4555 * and error occurred in DramECC (Extended error code = 0) then only 4556 * process the error, else bail out. 4557 */ 4558 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4559 (XEC(m->status, 0x3f) == 0x0))) 4560 return NOTIFY_DONE; 4561 4562 /* 4563 * If it is correctable error, return. 4564 */ 4565 if (mce_is_correctable(m)) 4566 return NOTIFY_OK; 4567 4568 /* 4569 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4570 */ 4571 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4572 4573 adev = find_adev(gpu_id); 4574 if (!adev) { 4575 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4576 gpu_id); 4577 return NOTIFY_DONE; 4578 } 4579 4580 /* 4581 * If it is uncorrectable error, then find out UMC instance and 4582 * channel index. 4583 */ 4584 umc_inst = GET_UMC_INST(m->ipid); 4585 ch_inst = GET_CHAN_INDEX(m->ipid); 4586 4587 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4588 umc_inst, ch_inst); 4589 4590 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4591 return NOTIFY_OK; 4592 else 4593 return NOTIFY_DONE; 4594 } 4595 4596 static struct notifier_block amdgpu_bad_page_nb = { 4597 .notifier_call = amdgpu_bad_page_notifier, 4598 .priority = MCE_PRIO_UC, 4599 }; 4600 4601 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4602 { 4603 /* 4604 * Add the adev to the mce_adev_list. 4605 * During mode2 reset, amdgpu device is temporarily 4606 * removed from the mgpu_info list which can cause 4607 * page retirement to fail. 4608 * Use this list instead of mgpu_info to find the amdgpu 4609 * device on which the UMC error was reported. 4610 */ 4611 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4612 4613 /* 4614 * Register the x86 notifier only once 4615 * with MCE subsystem. 4616 */ 4617 if (notifier_registered == false) { 4618 mce_register_decode_chain(&amdgpu_bad_page_nb); 4619 notifier_registered = true; 4620 } 4621 } 4622 #endif 4623 4624 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4625 { 4626 if (!adev) 4627 return NULL; 4628 4629 return adev->psp.ras_context.ras; 4630 } 4631 4632 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4633 { 4634 if (!adev) 4635 return -EINVAL; 4636 4637 adev->psp.ras_context.ras = ras_con; 4638 return 0; 4639 } 4640 4641 /* check if ras is supported on block, say, sdma, gfx */ 4642 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4643 unsigned int block) 4644 { 4645 int ret = 0; 4646 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4647 4648 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4649 return 0; 4650 4651 ret = ras && (adev->ras_enabled & (1 << block)); 4652 4653 /* For the special asic with mem ecc enabled but sram ecc 4654 * not enabled, even if the ras block is not supported on 4655 * .ras_enabled, if the asic supports poison mode and the 4656 * ras block has ras configuration, it can be considered 4657 * that the ras block supports ras function. 4658 */ 4659 if (!ret && 4660 (block == AMDGPU_RAS_BLOCK__GFX || 4661 block == AMDGPU_RAS_BLOCK__SDMA || 4662 block == AMDGPU_RAS_BLOCK__VCN || 4663 block == AMDGPU_RAS_BLOCK__JPEG) && 4664 (amdgpu_ras_mask & (1 << block)) && 4665 amdgpu_ras_is_poison_mode_supported(adev) && 4666 amdgpu_ras_get_ras_block(adev, block, 0)) 4667 ret = 1; 4668 4669 return ret; 4670 } 4671 4672 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 4673 { 4674 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4675 4676 /* mode1 is the only selection for RMA status */ 4677 if (amdgpu_ras_is_rma(adev)) { 4678 ras->gpu_reset_flags = 0; 4679 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4680 } 4681 4682 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 4683 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4684 int hive_ras_recovery = 0; 4685 4686 if (hive) { 4687 hive_ras_recovery = atomic_read(&hive->ras_recovery); 4688 amdgpu_put_xgmi_hive(hive); 4689 } 4690 /* In the case of multiple GPUs, after a GPU has started 4691 * resetting all GPUs on hive, other GPUs do not need to 4692 * trigger GPU reset again. 4693 */ 4694 if (!hive_ras_recovery) 4695 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4696 else 4697 atomic_set(&ras->in_recovery, 0); 4698 } else { 4699 flush_work(&ras->recovery_work); 4700 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4701 } 4702 4703 return 0; 4704 } 4705 4706 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 4707 { 4708 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4709 int ret = 0; 4710 4711 if (con) { 4712 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4713 if (!ret) 4714 con->is_aca_debug_mode = enable; 4715 } 4716 4717 return ret; 4718 } 4719 4720 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 4721 { 4722 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4723 int ret = 0; 4724 4725 if (con) { 4726 if (amdgpu_aca_is_enabled(adev)) 4727 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 4728 else 4729 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4730 if (!ret) 4731 con->is_aca_debug_mode = enable; 4732 } 4733 4734 return ret; 4735 } 4736 4737 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 4738 { 4739 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4740 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4741 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4742 4743 if (!con) 4744 return false; 4745 4746 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 4747 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 4748 return con->is_aca_debug_mode; 4749 else 4750 return true; 4751 } 4752 4753 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 4754 unsigned int *error_query_mode) 4755 { 4756 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4757 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4758 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4759 4760 if (!con) { 4761 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 4762 return false; 4763 } 4764 4765 if (amdgpu_sriov_vf(adev)) { 4766 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 4767 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 4768 *error_query_mode = 4769 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 4770 } else { 4771 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 4772 } 4773 4774 return true; 4775 } 4776 4777 /* Register each ip ras block into amdgpu ras */ 4778 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 4779 struct amdgpu_ras_block_object *ras_block_obj) 4780 { 4781 struct amdgpu_ras_block_list *ras_node; 4782 if (!adev || !ras_block_obj) 4783 return -EINVAL; 4784 4785 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 4786 if (!ras_node) 4787 return -ENOMEM; 4788 4789 INIT_LIST_HEAD(&ras_node->node); 4790 ras_node->ras_obj = ras_block_obj; 4791 list_add_tail(&ras_node->node, &adev->ras_list); 4792 4793 return 0; 4794 } 4795 4796 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 4797 { 4798 if (!err_type_name) 4799 return; 4800 4801 switch (err_type) { 4802 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 4803 sprintf(err_type_name, "correctable"); 4804 break; 4805 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 4806 sprintf(err_type_name, "uncorrectable"); 4807 break; 4808 default: 4809 sprintf(err_type_name, "unknown"); 4810 break; 4811 } 4812 } 4813 4814 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 4815 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4816 uint32_t instance, 4817 uint32_t *memory_id) 4818 { 4819 uint32_t err_status_lo_data, err_status_lo_offset; 4820 4821 if (!reg_entry) 4822 return false; 4823 4824 err_status_lo_offset = 4825 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4826 reg_entry->seg_lo, reg_entry->reg_lo); 4827 err_status_lo_data = RREG32(err_status_lo_offset); 4828 4829 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 4830 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 4831 return false; 4832 4833 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 4834 4835 return true; 4836 } 4837 4838 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 4839 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4840 uint32_t instance, 4841 unsigned long *err_cnt) 4842 { 4843 uint32_t err_status_hi_data, err_status_hi_offset; 4844 4845 if (!reg_entry) 4846 return false; 4847 4848 err_status_hi_offset = 4849 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4850 reg_entry->seg_hi, reg_entry->reg_hi); 4851 err_status_hi_data = RREG32(err_status_hi_offset); 4852 4853 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 4854 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 4855 /* keep the check here in case we need to refer to the result later */ 4856 dev_dbg(adev->dev, "Invalid err_info field\n"); 4857 4858 /* read err count */ 4859 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 4860 4861 return true; 4862 } 4863 4864 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 4865 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4866 uint32_t reg_list_size, 4867 const struct amdgpu_ras_memory_id_entry *mem_list, 4868 uint32_t mem_list_size, 4869 uint32_t instance, 4870 uint32_t err_type, 4871 unsigned long *err_count) 4872 { 4873 uint32_t memory_id; 4874 unsigned long err_cnt; 4875 char err_type_name[16]; 4876 uint32_t i, j; 4877 4878 for (i = 0; i < reg_list_size; i++) { 4879 /* query memory_id from err_status_lo */ 4880 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 4881 instance, &memory_id)) 4882 continue; 4883 4884 /* query err_cnt from err_status_hi */ 4885 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 4886 instance, &err_cnt) || 4887 !err_cnt) 4888 continue; 4889 4890 *err_count += err_cnt; 4891 4892 /* log the errors */ 4893 amdgpu_ras_get_error_type_name(err_type, err_type_name); 4894 if (!mem_list) { 4895 /* memory_list is not supported */ 4896 dev_info(adev->dev, 4897 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 4898 err_cnt, err_type_name, 4899 reg_list[i].block_name, 4900 instance, memory_id); 4901 } else { 4902 for (j = 0; j < mem_list_size; j++) { 4903 if (memory_id == mem_list[j].memory_id) { 4904 dev_info(adev->dev, 4905 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 4906 err_cnt, err_type_name, 4907 reg_list[i].block_name, 4908 instance, mem_list[j].name); 4909 break; 4910 } 4911 } 4912 } 4913 } 4914 } 4915 4916 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 4917 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4918 uint32_t reg_list_size, 4919 uint32_t instance) 4920 { 4921 uint32_t err_status_lo_offset, err_status_hi_offset; 4922 uint32_t i; 4923 4924 for (i = 0; i < reg_list_size; i++) { 4925 err_status_lo_offset = 4926 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4927 reg_list[i].seg_lo, reg_list[i].reg_lo); 4928 err_status_hi_offset = 4929 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4930 reg_list[i].seg_hi, reg_list[i].reg_hi); 4931 WREG32(err_status_lo_offset, 0); 4932 WREG32(err_status_hi_offset, 0); 4933 } 4934 } 4935 4936 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 4937 { 4938 memset(err_data, 0, sizeof(*err_data)); 4939 4940 INIT_LIST_HEAD(&err_data->err_node_list); 4941 4942 return 0; 4943 } 4944 4945 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 4946 { 4947 if (!err_node) 4948 return; 4949 4950 list_del(&err_node->node); 4951 kvfree(err_node); 4952 } 4953 4954 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 4955 { 4956 struct ras_err_node *err_node, *tmp; 4957 4958 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 4959 amdgpu_ras_error_node_release(err_node); 4960 } 4961 4962 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 4963 struct amdgpu_smuio_mcm_config_info *mcm_info) 4964 { 4965 struct ras_err_node *err_node; 4966 struct amdgpu_smuio_mcm_config_info *ref_id; 4967 4968 if (!err_data || !mcm_info) 4969 return NULL; 4970 4971 for_each_ras_error(err_node, err_data) { 4972 ref_id = &err_node->err_info.mcm_info; 4973 4974 if (mcm_info->socket_id == ref_id->socket_id && 4975 mcm_info->die_id == ref_id->die_id) 4976 return err_node; 4977 } 4978 4979 return NULL; 4980 } 4981 4982 static struct ras_err_node *amdgpu_ras_error_node_new(void) 4983 { 4984 struct ras_err_node *err_node; 4985 4986 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 4987 if (!err_node) 4988 return NULL; 4989 4990 INIT_LIST_HEAD(&err_node->node); 4991 4992 return err_node; 4993 } 4994 4995 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 4996 { 4997 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 4998 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 4999 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5000 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5001 5002 if (unlikely(infoa->socket_id != infob->socket_id)) 5003 return infoa->socket_id - infob->socket_id; 5004 else 5005 return infoa->die_id - infob->die_id; 5006 5007 return 0; 5008 } 5009 5010 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5011 struct amdgpu_smuio_mcm_config_info *mcm_info) 5012 { 5013 struct ras_err_node *err_node; 5014 5015 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5016 if (err_node) 5017 return &err_node->err_info; 5018 5019 err_node = amdgpu_ras_error_node_new(); 5020 if (!err_node) 5021 return NULL; 5022 5023 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5024 5025 err_data->err_list_count++; 5026 list_add_tail(&err_node->node, &err_data->err_node_list); 5027 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5028 5029 return &err_node->err_info; 5030 } 5031 5032 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5033 struct amdgpu_smuio_mcm_config_info *mcm_info, 5034 u64 count) 5035 { 5036 struct ras_err_info *err_info; 5037 5038 if (!err_data || !mcm_info) 5039 return -EINVAL; 5040 5041 if (!count) 5042 return 0; 5043 5044 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5045 if (!err_info) 5046 return -EINVAL; 5047 5048 err_info->ue_count += count; 5049 err_data->ue_count += count; 5050 5051 return 0; 5052 } 5053 5054 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5055 struct amdgpu_smuio_mcm_config_info *mcm_info, 5056 u64 count) 5057 { 5058 struct ras_err_info *err_info; 5059 5060 if (!err_data || !mcm_info) 5061 return -EINVAL; 5062 5063 if (!count) 5064 return 0; 5065 5066 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5067 if (!err_info) 5068 return -EINVAL; 5069 5070 err_info->ce_count += count; 5071 err_data->ce_count += count; 5072 5073 return 0; 5074 } 5075 5076 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5077 struct amdgpu_smuio_mcm_config_info *mcm_info, 5078 u64 count) 5079 { 5080 struct ras_err_info *err_info; 5081 5082 if (!err_data || !mcm_info) 5083 return -EINVAL; 5084 5085 if (!count) 5086 return 0; 5087 5088 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5089 if (!err_info) 5090 return -EINVAL; 5091 5092 err_info->de_count += count; 5093 err_data->de_count += count; 5094 5095 return 0; 5096 } 5097 5098 #define mmMP0_SMN_C2PMSG_92 0x1609C 5099 #define mmMP0_SMN_C2PMSG_126 0x160BE 5100 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5101 u32 instance) 5102 { 5103 u32 socket_id, aid_id, hbm_id; 5104 u32 fw_status; 5105 u32 boot_error; 5106 u64 reg_addr; 5107 5108 /* The pattern for smn addressing in other SOC could be different from 5109 * the one for aqua_vanjaram. We should revisit the code if the pattern 5110 * is changed. In such case, replace the aqua_vanjaram implementation 5111 * with more common helper */ 5112 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5113 aqua_vanjaram_encode_ext_smn_addressing(instance); 5114 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5115 5116 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5117 aqua_vanjaram_encode_ext_smn_addressing(instance); 5118 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5119 5120 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5121 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5122 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5123 5124 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5125 dev_info(adev->dev, 5126 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5127 socket_id, aid_id, hbm_id, fw_status); 5128 5129 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5130 dev_info(adev->dev, 5131 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5132 socket_id, aid_id, fw_status); 5133 5134 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5135 dev_info(adev->dev, 5136 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5137 socket_id, aid_id, fw_status); 5138 5139 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5140 dev_info(adev->dev, 5141 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5142 socket_id, aid_id, fw_status); 5143 5144 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5145 dev_info(adev->dev, 5146 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5147 socket_id, aid_id, fw_status); 5148 5149 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5150 dev_info(adev->dev, 5151 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5152 socket_id, aid_id, fw_status); 5153 5154 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5155 dev_info(adev->dev, 5156 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5157 socket_id, aid_id, hbm_id, fw_status); 5158 5159 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5160 dev_info(adev->dev, 5161 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5162 socket_id, aid_id, hbm_id, fw_status); 5163 5164 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5165 dev_info(adev->dev, 5166 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5167 socket_id, aid_id, fw_status); 5168 5169 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5170 dev_info(adev->dev, 5171 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5172 socket_id, aid_id, fw_status); 5173 } 5174 5175 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5176 u32 instance) 5177 { 5178 u64 reg_addr; 5179 u32 reg_data; 5180 int retry_loop; 5181 5182 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5183 aqua_vanjaram_encode_ext_smn_addressing(instance); 5184 5185 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5186 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5187 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5188 return false; 5189 else 5190 msleep(1); 5191 } 5192 5193 return true; 5194 } 5195 5196 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5197 { 5198 u32 i; 5199 5200 for (i = 0; i < num_instances; i++) { 5201 if (amdgpu_ras_boot_error_detected(adev, i)) 5202 amdgpu_ras_boot_time_error_reporting(adev, i); 5203 } 5204 } 5205 5206 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5207 { 5208 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5209 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5210 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5211 int ret = 0; 5212 5213 mutex_lock(&con->page_rsv_lock); 5214 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5215 if (ret == -ENOENT) 5216 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5217 mutex_unlock(&con->page_rsv_lock); 5218 5219 return ret; 5220 } 5221 5222 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5223 const char *fmt, ...) 5224 { 5225 struct va_format vaf; 5226 va_list args; 5227 5228 va_start(args, fmt); 5229 vaf.fmt = fmt; 5230 vaf.va = &args; 5231 5232 if (RAS_EVENT_ID_IS_VALID(event_id)) 5233 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5234 else 5235 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5236 5237 va_end(args); 5238 } 5239 5240 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5241 { 5242 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5243 5244 if (!con) 5245 return false; 5246 5247 return con->is_rma; 5248 } 5249