1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/debugfs.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/uaccess.h> 28 #include <linux/reboot.h> 29 #include <linux/syscalls.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/list_sort.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_atomfirmware.h" 36 #include "amdgpu_xgmi.h" 37 #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38 #include "nbio_v4_3.h" 39 #include "nbif_v6_3_1.h" 40 #include "nbio_v7_9.h" 41 #include "atom.h" 42 #include "amdgpu_reset.h" 43 #include "amdgpu_psp.h" 44 45 #ifdef CONFIG_X86_MCE_AMD 46 #include <asm/mce.h> 47 48 static bool notifier_registered; 49 #endif 50 static const char *RAS_FS_NAME = "ras"; 51 52 const char *ras_error_string[] = { 53 "none", 54 "parity", 55 "single_correctable", 56 "multi_uncorrectable", 57 "poison", 58 }; 59 60 const char *ras_block_string[] = { 61 "umc", 62 "sdma", 63 "gfx", 64 "mmhub", 65 "athub", 66 "pcie_bif", 67 "hdp", 68 "xgmi_wafl", 69 "df", 70 "smn", 71 "sem", 72 "mp0", 73 "mp1", 74 "fuse", 75 "mca", 76 "vcn", 77 "jpeg", 78 "ih", 79 "mpio", 80 }; 81 82 const char *ras_mca_block_string[] = { 83 "mca_mp0", 84 "mca_mp1", 85 "mca_mpio", 86 "mca_iohc", 87 }; 88 89 struct amdgpu_ras_block_list { 90 /* ras block link */ 91 struct list_head node; 92 93 struct amdgpu_ras_block_object *ras_obj; 94 }; 95 96 const char *get_ras_block_str(struct ras_common_if *ras_block) 97 { 98 if (!ras_block) 99 return "NULL"; 100 101 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 102 ras_block->block >= ARRAY_SIZE(ras_block_string)) 103 return "OUT OF RANGE"; 104 105 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 106 return ras_mca_block_string[ras_block->sub_block_index]; 107 108 return ras_block_string[ras_block->block]; 109 } 110 111 #define ras_block_str(_BLOCK_) \ 112 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 113 114 #define ras_err_str(i) (ras_error_string[ffs(i)]) 115 116 #define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 117 118 /* inject address is 52 bits */ 119 #define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 120 121 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 122 #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 123 124 #define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms 125 126 #define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 127 128 #define MAX_FLUSH_RETIRE_DWORK_TIMES 100 129 130 enum amdgpu_ras_retire_page_reservation { 131 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 132 AMDGPU_RAS_RETIRE_PAGE_PENDING, 133 AMDGPU_RAS_RETIRE_PAGE_FAULT, 134 }; 135 136 atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 137 138 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 139 uint64_t addr); 140 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 141 uint64_t addr); 142 #ifdef CONFIG_X86_MCE_AMD 143 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 144 struct mce_notifier_adev_list { 145 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 146 int num_gpu; 147 }; 148 static struct mce_notifier_adev_list mce_adev_list; 149 #endif 150 151 void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 152 { 153 if (adev && amdgpu_ras_get_context(adev)) 154 amdgpu_ras_get_context(adev)->error_query_ready = ready; 155 } 156 157 static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 158 { 159 if (adev && amdgpu_ras_get_context(adev)) 160 return amdgpu_ras_get_context(adev)->error_query_ready; 161 162 return false; 163 } 164 165 static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 166 { 167 struct ras_err_data err_data; 168 struct eeprom_table_record err_rec; 169 int ret; 170 171 if ((address >= adev->gmc.mc_vram_size) || 172 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 173 dev_warn(adev->dev, 174 "RAS WARN: input address 0x%llx is invalid.\n", 175 address); 176 return -EINVAL; 177 } 178 179 if (amdgpu_ras_check_bad_page(adev, address)) { 180 dev_warn(adev->dev, 181 "RAS WARN: 0x%llx has already been marked as bad page!\n", 182 address); 183 return 0; 184 } 185 186 ret = amdgpu_ras_error_data_init(&err_data); 187 if (ret) 188 return ret; 189 190 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 191 err_data.err_addr = &err_rec; 192 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 193 194 if (amdgpu_bad_page_threshold != 0) { 195 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 196 err_data.err_addr_cnt, false); 197 amdgpu_ras_save_bad_pages(adev, NULL); 198 } 199 200 amdgpu_ras_error_data_fini(&err_data); 201 202 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 203 dev_warn(adev->dev, "Clear EEPROM:\n"); 204 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 205 206 return 0; 207 } 208 209 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 210 size_t size, loff_t *pos) 211 { 212 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 213 struct ras_query_if info = { 214 .head = obj->head, 215 }; 216 ssize_t s; 217 char val[128]; 218 219 if (amdgpu_ras_query_error_status(obj->adev, &info)) 220 return -EINVAL; 221 222 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 223 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 224 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 225 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 226 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 227 } 228 229 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 230 "ue", info.ue_count, 231 "ce", info.ce_count); 232 if (*pos >= s) 233 return 0; 234 235 s -= *pos; 236 s = min_t(u64, s, size); 237 238 239 if (copy_to_user(buf, &val[*pos], s)) 240 return -EINVAL; 241 242 *pos += s; 243 244 return s; 245 } 246 247 static const struct file_operations amdgpu_ras_debugfs_ops = { 248 .owner = THIS_MODULE, 249 .read = amdgpu_ras_debugfs_read, 250 .write = NULL, 251 .llseek = default_llseek 252 }; 253 254 static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 255 { 256 int i; 257 258 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 259 *block_id = i; 260 if (strcmp(name, ras_block_string[i]) == 0) 261 return 0; 262 } 263 return -EINVAL; 264 } 265 266 static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 267 const char __user *buf, size_t size, 268 loff_t *pos, struct ras_debug_if *data) 269 { 270 ssize_t s = min_t(u64, 64, size); 271 char str[65]; 272 char block_name[33]; 273 char err[9] = "ue"; 274 int op = -1; 275 int block_id; 276 uint32_t sub_block; 277 u64 address, value; 278 /* default value is 0 if the mask is not set by user */ 279 u32 instance_mask = 0; 280 281 if (*pos) 282 return -EINVAL; 283 *pos = size; 284 285 memset(str, 0, sizeof(str)); 286 memset(data, 0, sizeof(*data)); 287 288 if (copy_from_user(str, buf, s)) 289 return -EINVAL; 290 291 if (sscanf(str, "disable %32s", block_name) == 1) 292 op = 0; 293 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 294 op = 1; 295 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 296 op = 2; 297 else if (strstr(str, "retire_page") != NULL) 298 op = 3; 299 else if (str[0] && str[1] && str[2] && str[3]) 300 /* ascii string, but commands are not matched. */ 301 return -EINVAL; 302 303 if (op != -1) { 304 if (op == 3) { 305 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 306 sscanf(str, "%*s %llu", &address) != 1) 307 return -EINVAL; 308 309 data->op = op; 310 data->inject.address = address; 311 312 return 0; 313 } 314 315 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 316 return -EINVAL; 317 318 data->head.block = block_id; 319 /* only ue, ce and poison errors are supported */ 320 if (!memcmp("ue", err, 2)) 321 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 322 else if (!memcmp("ce", err, 2)) 323 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 324 else if (!memcmp("poison", err, 6)) 325 data->head.type = AMDGPU_RAS_ERROR__POISON; 326 else 327 return -EINVAL; 328 329 data->op = op; 330 331 if (op == 2) { 332 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 333 &sub_block, &address, &value, &instance_mask) != 4 && 334 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 335 &sub_block, &address, &value, &instance_mask) != 4 && 336 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 337 &sub_block, &address, &value) != 3 && 338 sscanf(str, "%*s %*s %*s %u %llu %llu", 339 &sub_block, &address, &value) != 3) 340 return -EINVAL; 341 data->head.sub_block_index = sub_block; 342 data->inject.address = address; 343 data->inject.value = value; 344 data->inject.instance_mask = instance_mask; 345 } 346 } else { 347 if (size < sizeof(*data)) 348 return -EINVAL; 349 350 if (copy_from_user(data, buf, sizeof(*data))) 351 return -EINVAL; 352 } 353 354 return 0; 355 } 356 357 static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 358 struct ras_debug_if *data) 359 { 360 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 361 uint32_t mask, inst_mask = data->inject.instance_mask; 362 363 /* no need to set instance mask if there is only one instance */ 364 if (num_xcc <= 1 && inst_mask) { 365 data->inject.instance_mask = 0; 366 dev_dbg(adev->dev, 367 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 368 inst_mask); 369 370 return; 371 } 372 373 switch (data->head.block) { 374 case AMDGPU_RAS_BLOCK__GFX: 375 mask = GENMASK(num_xcc - 1, 0); 376 break; 377 case AMDGPU_RAS_BLOCK__SDMA: 378 mask = GENMASK(adev->sdma.num_instances - 1, 0); 379 break; 380 case AMDGPU_RAS_BLOCK__VCN: 381 case AMDGPU_RAS_BLOCK__JPEG: 382 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 383 break; 384 default: 385 mask = inst_mask; 386 break; 387 } 388 389 /* remove invalid bits in instance mask */ 390 data->inject.instance_mask &= mask; 391 if (inst_mask != data->inject.instance_mask) 392 dev_dbg(adev->dev, 393 "Adjust RAS inject mask 0x%x to 0x%x\n", 394 inst_mask, data->inject.instance_mask); 395 } 396 397 /** 398 * DOC: AMDGPU RAS debugfs control interface 399 * 400 * The control interface accepts struct ras_debug_if which has two members. 401 * 402 * First member: ras_debug_if::head or ras_debug_if::inject. 403 * 404 * head is used to indicate which IP block will be under control. 405 * 406 * head has four members, they are block, type, sub_block_index, name. 407 * block: which IP will be under control. 408 * type: what kind of error will be enabled/disabled/injected. 409 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 410 * name: the name of IP. 411 * 412 * inject has three more members than head, they are address, value and mask. 413 * As their names indicate, inject operation will write the 414 * value to the address. 415 * 416 * The second member: struct ras_debug_if::op. 417 * It has three kinds of operations. 418 * 419 * - 0: disable RAS on the block. Take ::head as its data. 420 * - 1: enable RAS on the block. Take ::head as its data. 421 * - 2: inject errors on the block. Take ::inject as its data. 422 * 423 * How to use the interface? 424 * 425 * In a program 426 * 427 * Copy the struct ras_debug_if in your code and initialize it. 428 * Write the struct to the control interface. 429 * 430 * From shell 431 * 432 * .. code-block:: bash 433 * 434 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 435 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 436 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 437 * 438 * Where N, is the card which you want to affect. 439 * 440 * "disable" requires only the block. 441 * "enable" requires the block and error type. 442 * "inject" requires the block, error type, address, and value. 443 * 444 * The block is one of: umc, sdma, gfx, etc. 445 * see ras_block_string[] for details 446 * 447 * The error type is one of: ue, ce and poison where, 448 * ue is multi-uncorrectable 449 * ce is single-correctable 450 * poison is poison 451 * 452 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 453 * The address and value are hexadecimal numbers, leading 0x is optional. 454 * The mask means instance mask, is optional, default value is 0x1. 455 * 456 * For instance, 457 * 458 * .. code-block:: bash 459 * 460 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 461 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 462 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 463 * 464 * How to check the result of the operation? 465 * 466 * To check disable/enable, see "ras" features at, 467 * /sys/class/drm/card[0/1/2...]/device/ras/features 468 * 469 * To check inject, see the corresponding error count at, 470 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 471 * 472 * .. note:: 473 * Operations are only allowed on blocks which are supported. 474 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 475 * to see which blocks support RAS on a particular asic. 476 * 477 */ 478 static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 479 const char __user *buf, 480 size_t size, loff_t *pos) 481 { 482 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 483 struct ras_debug_if data; 484 int ret = 0; 485 486 if (!amdgpu_ras_get_error_query_ready(adev)) { 487 dev_warn(adev->dev, "RAS WARN: error injection " 488 "currently inaccessible\n"); 489 return size; 490 } 491 492 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 493 if (ret) 494 return ret; 495 496 if (data.op == 3) { 497 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 498 if (!ret) 499 return size; 500 else 501 return ret; 502 } 503 504 if (!amdgpu_ras_is_supported(adev, data.head.block)) 505 return -EINVAL; 506 507 switch (data.op) { 508 case 0: 509 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 510 break; 511 case 1: 512 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 513 break; 514 case 2: 515 if ((data.inject.address >= adev->gmc.mc_vram_size && 516 adev->gmc.mc_vram_size) || 517 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) { 518 dev_warn(adev->dev, "RAS WARN: input address " 519 "0x%llx is invalid.", 520 data.inject.address); 521 ret = -EINVAL; 522 break; 523 } 524 525 /* umc ce/ue error injection for a bad page is not allowed */ 526 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) && 527 amdgpu_ras_check_bad_page(adev, data.inject.address)) { 528 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has " 529 "already been marked as bad!\n", 530 data.inject.address); 531 break; 532 } 533 534 amdgpu_ras_instance_mask_check(adev, &data); 535 536 /* data.inject.address is offset instead of absolute gpu address */ 537 ret = amdgpu_ras_error_inject(adev, &data.inject); 538 break; 539 default: 540 ret = -EINVAL; 541 break; 542 } 543 544 if (ret) 545 return ret; 546 547 return size; 548 } 549 550 /** 551 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 552 * 553 * Some boards contain an EEPROM which is used to persistently store a list of 554 * bad pages which experiences ECC errors in vram. This interface provides 555 * a way to reset the EEPROM, e.g., after testing error injection. 556 * 557 * Usage: 558 * 559 * .. code-block:: bash 560 * 561 * echo 1 > ../ras/ras_eeprom_reset 562 * 563 * will reset EEPROM table to 0 entries. 564 * 565 */ 566 static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 567 const char __user *buf, 568 size_t size, loff_t *pos) 569 { 570 struct amdgpu_device *adev = 571 (struct amdgpu_device *)file_inode(f)->i_private; 572 int ret; 573 574 ret = amdgpu_ras_eeprom_reset_table( 575 &(amdgpu_ras_get_context(adev)->eeprom_control)); 576 577 if (!ret) { 578 /* Something was written to EEPROM. 579 */ 580 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 581 return size; 582 } else { 583 return ret; 584 } 585 } 586 587 static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 588 .owner = THIS_MODULE, 589 .read = NULL, 590 .write = amdgpu_ras_debugfs_ctrl_write, 591 .llseek = default_llseek 592 }; 593 594 static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 595 .owner = THIS_MODULE, 596 .read = NULL, 597 .write = amdgpu_ras_debugfs_eeprom_write, 598 .llseek = default_llseek 599 }; 600 601 /** 602 * DOC: AMDGPU RAS sysfs Error Count Interface 603 * 604 * It allows the user to read the error count for each IP block on the gpu through 605 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 606 * 607 * It outputs the multiple lines which report the uncorrected (ue) and corrected 608 * (ce) error counts. 609 * 610 * The format of one line is below, 611 * 612 * [ce|ue]: count 613 * 614 * Example: 615 * 616 * .. code-block:: bash 617 * 618 * ue: 0 619 * ce: 1 620 * 621 */ 622 static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 623 struct device_attribute *attr, char *buf) 624 { 625 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 626 struct ras_query_if info = { 627 .head = obj->head, 628 }; 629 630 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 631 return sysfs_emit(buf, "Query currently inaccessible\n"); 632 633 if (amdgpu_ras_query_error_status(obj->adev, &info)) 634 return -EINVAL; 635 636 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 637 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 638 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 639 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 640 } 641 642 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 643 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 644 "ce", info.ce_count, "de", info.de_count); 645 else 646 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 647 "ce", info.ce_count); 648 } 649 650 /* obj begin */ 651 652 #define get_obj(obj) do { (obj)->use++; } while (0) 653 #define alive_obj(obj) ((obj)->use) 654 655 static inline void put_obj(struct ras_manager *obj) 656 { 657 if (obj && (--obj->use == 0)) { 658 list_del(&obj->node); 659 amdgpu_ras_error_data_fini(&obj->err_data); 660 } 661 662 if (obj && (obj->use < 0)) 663 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 664 } 665 666 /* make one obj and return it. */ 667 static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 668 struct ras_common_if *head) 669 { 670 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 671 struct ras_manager *obj; 672 673 if (!adev->ras_enabled || !con) 674 return NULL; 675 676 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 677 return NULL; 678 679 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 680 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 681 return NULL; 682 683 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 684 } else 685 obj = &con->objs[head->block]; 686 687 /* already exist. return obj? */ 688 if (alive_obj(obj)) 689 return NULL; 690 691 if (amdgpu_ras_error_data_init(&obj->err_data)) 692 return NULL; 693 694 obj->head = *head; 695 obj->adev = adev; 696 list_add(&obj->node, &con->head); 697 get_obj(obj); 698 699 return obj; 700 } 701 702 /* return an obj equal to head, or the first when head is NULL */ 703 struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 704 struct ras_common_if *head) 705 { 706 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 707 struct ras_manager *obj; 708 int i; 709 710 if (!adev->ras_enabled || !con) 711 return NULL; 712 713 if (head) { 714 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 715 return NULL; 716 717 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 718 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 719 return NULL; 720 721 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 722 } else 723 obj = &con->objs[head->block]; 724 725 if (alive_obj(obj)) 726 return obj; 727 } else { 728 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 729 obj = &con->objs[i]; 730 if (alive_obj(obj)) 731 return obj; 732 } 733 } 734 735 return NULL; 736 } 737 /* obj end */ 738 739 /* feature ctl begin */ 740 static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 741 struct ras_common_if *head) 742 { 743 return adev->ras_hw_enabled & BIT(head->block); 744 } 745 746 static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 747 struct ras_common_if *head) 748 { 749 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 750 751 return con->features & BIT(head->block); 752 } 753 754 /* 755 * if obj is not created, then create one. 756 * set feature enable flag. 757 */ 758 static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 759 struct ras_common_if *head, int enable) 760 { 761 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 762 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 763 764 /* If hardware does not support ras, then do not create obj. 765 * But if hardware support ras, we can create the obj. 766 * Ras framework checks con->hw_supported to see if it need do 767 * corresponding initialization. 768 * IP checks con->support to see if it need disable ras. 769 */ 770 if (!amdgpu_ras_is_feature_allowed(adev, head)) 771 return 0; 772 773 if (enable) { 774 if (!obj) { 775 obj = amdgpu_ras_create_obj(adev, head); 776 if (!obj) 777 return -EINVAL; 778 } else { 779 /* In case we create obj somewhere else */ 780 get_obj(obj); 781 } 782 con->features |= BIT(head->block); 783 } else { 784 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 785 con->features &= ~BIT(head->block); 786 put_obj(obj); 787 } 788 } 789 790 return 0; 791 } 792 793 /* wrapper of psp_ras_enable_features */ 794 int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 795 struct ras_common_if *head, bool enable) 796 { 797 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 798 union ta_ras_cmd_input *info; 799 int ret; 800 801 if (!con) 802 return -EINVAL; 803 804 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 805 /* For gfx ip, regardless of feature support status, */ 806 /* Force issue enable or disable ras feature commands */ 807 if (head->block != AMDGPU_RAS_BLOCK__GFX && 808 !amdgpu_ras_is_feature_allowed(adev, head)) 809 return 0; 810 811 /* Only enable gfx ras feature from host side */ 812 if (head->block == AMDGPU_RAS_BLOCK__GFX && 813 !amdgpu_sriov_vf(adev) && 814 !amdgpu_ras_intr_triggered()) { 815 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 816 if (!info) 817 return -ENOMEM; 818 819 if (!enable) { 820 info->disable_features = (struct ta_ras_disable_features_input) { 821 .block_id = amdgpu_ras_block_to_ta(head->block), 822 .error_type = amdgpu_ras_error_to_ta(head->type), 823 }; 824 } else { 825 info->enable_features = (struct ta_ras_enable_features_input) { 826 .block_id = amdgpu_ras_block_to_ta(head->block), 827 .error_type = amdgpu_ras_error_to_ta(head->type), 828 }; 829 } 830 831 ret = psp_ras_enable_features(&adev->psp, info, enable); 832 if (ret) { 833 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 834 enable ? "enable":"disable", 835 get_ras_block_str(head), 836 amdgpu_ras_is_poison_mode_supported(adev), ret); 837 kfree(info); 838 return ret; 839 } 840 841 kfree(info); 842 } 843 844 /* setup the obj */ 845 __amdgpu_ras_feature_enable(adev, head, enable); 846 847 return 0; 848 } 849 850 /* Only used in device probe stage and called only once. */ 851 int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 852 struct ras_common_if *head, bool enable) 853 { 854 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 855 int ret; 856 857 if (!con) 858 return -EINVAL; 859 860 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 861 if (enable) { 862 /* There is no harm to issue a ras TA cmd regardless of 863 * the currecnt ras state. 864 * If current state == target state, it will do nothing 865 * But sometimes it requests driver to reset and repost 866 * with error code -EAGAIN. 867 */ 868 ret = amdgpu_ras_feature_enable(adev, head, 1); 869 /* With old ras TA, we might fail to enable ras. 870 * Log it and just setup the object. 871 * TODO need remove this WA in the future. 872 */ 873 if (ret == -EINVAL) { 874 ret = __amdgpu_ras_feature_enable(adev, head, 1); 875 if (!ret) 876 dev_info(adev->dev, 877 "RAS INFO: %s setup object\n", 878 get_ras_block_str(head)); 879 } 880 } else { 881 /* setup the object then issue a ras TA disable cmd.*/ 882 ret = __amdgpu_ras_feature_enable(adev, head, 1); 883 if (ret) 884 return ret; 885 886 /* gfx block ras disable cmd must send to ras-ta */ 887 if (head->block == AMDGPU_RAS_BLOCK__GFX) 888 con->features |= BIT(head->block); 889 890 ret = amdgpu_ras_feature_enable(adev, head, 0); 891 892 /* clean gfx block ras features flag */ 893 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 894 con->features &= ~BIT(head->block); 895 } 896 } else 897 ret = amdgpu_ras_feature_enable(adev, head, enable); 898 899 return ret; 900 } 901 902 static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 903 bool bypass) 904 { 905 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 906 struct ras_manager *obj, *tmp; 907 908 list_for_each_entry_safe(obj, tmp, &con->head, node) { 909 /* bypass psp. 910 * aka just release the obj and corresponding flags 911 */ 912 if (bypass) { 913 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 914 break; 915 } else { 916 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 917 break; 918 } 919 } 920 921 return con->features; 922 } 923 924 static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 925 bool bypass) 926 { 927 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 928 int i; 929 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 930 931 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 932 struct ras_common_if head = { 933 .block = i, 934 .type = default_ras_type, 935 .sub_block_index = 0, 936 }; 937 938 if (i == AMDGPU_RAS_BLOCK__MCA) 939 continue; 940 941 if (bypass) { 942 /* 943 * bypass psp. vbios enable ras for us. 944 * so just create the obj 945 */ 946 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 947 break; 948 } else { 949 if (amdgpu_ras_feature_enable(adev, &head, 1)) 950 break; 951 } 952 } 953 954 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 955 struct ras_common_if head = { 956 .block = AMDGPU_RAS_BLOCK__MCA, 957 .type = default_ras_type, 958 .sub_block_index = i, 959 }; 960 961 if (bypass) { 962 /* 963 * bypass psp. vbios enable ras for us. 964 * so just create the obj 965 */ 966 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 967 break; 968 } else { 969 if (amdgpu_ras_feature_enable(adev, &head, 1)) 970 break; 971 } 972 } 973 974 return con->features; 975 } 976 /* feature ctl end */ 977 978 static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 979 enum amdgpu_ras_block block) 980 { 981 if (!block_obj) 982 return -EINVAL; 983 984 if (block_obj->ras_comm.block == block) 985 return 0; 986 987 return -EINVAL; 988 } 989 990 static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 991 enum amdgpu_ras_block block, uint32_t sub_block_index) 992 { 993 struct amdgpu_ras_block_list *node, *tmp; 994 struct amdgpu_ras_block_object *obj; 995 996 if (block >= AMDGPU_RAS_BLOCK__LAST) 997 return NULL; 998 999 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1000 if (!node->ras_obj) { 1001 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1002 continue; 1003 } 1004 1005 obj = node->ras_obj; 1006 if (obj->ras_block_match) { 1007 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1008 return obj; 1009 } else { 1010 if (amdgpu_ras_block_match_default(obj, block) == 0) 1011 return obj; 1012 } 1013 } 1014 1015 return NULL; 1016 } 1017 1018 static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1019 { 1020 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1021 int ret = 0; 1022 1023 /* 1024 * choosing right query method according to 1025 * whether smu support query error information 1026 */ 1027 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1028 if (ret == -EOPNOTSUPP) { 1029 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1030 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1032 1033 /* umc query_ras_error_address is also responsible for clearing 1034 * error status 1035 */ 1036 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1037 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1039 } else if (!ret) { 1040 if (adev->umc.ras && 1041 adev->umc.ras->ecc_info_query_ras_error_count) 1042 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1043 1044 if (adev->umc.ras && 1045 adev->umc.ras->ecc_info_query_ras_error_address) 1046 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1047 } 1048 } 1049 1050 static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1051 struct ras_manager *ras_mgr, 1052 struct ras_err_data *err_data, 1053 struct ras_query_context *qctx, 1054 const char *blk_name, 1055 bool is_ue, 1056 bool is_de) 1057 { 1058 struct amdgpu_smuio_mcm_config_info *mcm_info; 1059 struct ras_err_node *err_node; 1060 struct ras_err_info *err_info; 1061 u64 event_id = qctx->evid.event_id; 1062 1063 if (is_ue) { 1064 for_each_ras_error(err_node, err_data) { 1065 err_info = &err_node->err_info; 1066 mcm_info = &err_info->mcm_info; 1067 if (err_info->ue_count) { 1068 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1069 "%lld new uncorrectable hardware errors detected in %s block\n", 1070 mcm_info->socket_id, 1071 mcm_info->die_id, 1072 err_info->ue_count, 1073 blk_name); 1074 } 1075 } 1076 1077 for_each_ras_error(err_node, &ras_mgr->err_data) { 1078 err_info = &err_node->err_info; 1079 mcm_info = &err_info->mcm_info; 1080 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1081 "%lld uncorrectable hardware errors detected in total in %s block\n", 1082 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1083 } 1084 1085 } else { 1086 if (is_de) { 1087 for_each_ras_error(err_node, err_data) { 1088 err_info = &err_node->err_info; 1089 mcm_info = &err_info->mcm_info; 1090 if (err_info->de_count) { 1091 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1092 "%lld new deferred hardware errors detected in %s block\n", 1093 mcm_info->socket_id, 1094 mcm_info->die_id, 1095 err_info->de_count, 1096 blk_name); 1097 } 1098 } 1099 1100 for_each_ras_error(err_node, &ras_mgr->err_data) { 1101 err_info = &err_node->err_info; 1102 mcm_info = &err_info->mcm_info; 1103 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1104 "%lld deferred hardware errors detected in total in %s block\n", 1105 mcm_info->socket_id, mcm_info->die_id, 1106 err_info->de_count, blk_name); 1107 } 1108 } else { 1109 for_each_ras_error(err_node, err_data) { 1110 err_info = &err_node->err_info; 1111 mcm_info = &err_info->mcm_info; 1112 if (err_info->ce_count) { 1113 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1114 "%lld new correctable hardware errors detected in %s block\n", 1115 mcm_info->socket_id, 1116 mcm_info->die_id, 1117 err_info->ce_count, 1118 blk_name); 1119 } 1120 } 1121 1122 for_each_ras_error(err_node, &ras_mgr->err_data) { 1123 err_info = &err_node->err_info; 1124 mcm_info = &err_info->mcm_info; 1125 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1126 "%lld correctable hardware errors detected in total in %s block\n", 1127 mcm_info->socket_id, mcm_info->die_id, 1128 err_info->ce_count, blk_name); 1129 } 1130 } 1131 } 1132 } 1133 1134 static inline bool err_data_has_source_info(struct ras_err_data *data) 1135 { 1136 return !list_empty(&data->err_node_list); 1137 } 1138 1139 static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1140 struct ras_query_if *query_if, 1141 struct ras_err_data *err_data, 1142 struct ras_query_context *qctx) 1143 { 1144 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1145 const char *blk_name = get_ras_block_str(&query_if->head); 1146 u64 event_id = qctx->evid.event_id; 1147 1148 if (err_data->ce_count) { 1149 if (err_data_has_source_info(err_data)) { 1150 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1151 blk_name, false, false); 1152 } else if (!adev->aid_mask && 1153 adev->smuio.funcs && 1154 adev->smuio.funcs->get_socket_id && 1155 adev->smuio.funcs->get_die_id) { 1156 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1157 "%ld correctable hardware errors " 1158 "detected in %s block\n", 1159 adev->smuio.funcs->get_socket_id(adev), 1160 adev->smuio.funcs->get_die_id(adev), 1161 ras_mgr->err_data.ce_count, 1162 blk_name); 1163 } else { 1164 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1165 "detected in %s block\n", 1166 ras_mgr->err_data.ce_count, 1167 blk_name); 1168 } 1169 } 1170 1171 if (err_data->ue_count) { 1172 if (err_data_has_source_info(err_data)) { 1173 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1174 blk_name, true, false); 1175 } else if (!adev->aid_mask && 1176 adev->smuio.funcs && 1177 adev->smuio.funcs->get_socket_id && 1178 adev->smuio.funcs->get_die_id) { 1179 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1180 "%ld uncorrectable hardware errors " 1181 "detected in %s block\n", 1182 adev->smuio.funcs->get_socket_id(adev), 1183 adev->smuio.funcs->get_die_id(adev), 1184 ras_mgr->err_data.ue_count, 1185 blk_name); 1186 } else { 1187 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1188 "detected in %s block\n", 1189 ras_mgr->err_data.ue_count, 1190 blk_name); 1191 } 1192 } 1193 1194 if (err_data->de_count) { 1195 if (err_data_has_source_info(err_data)) { 1196 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1197 blk_name, false, true); 1198 } else if (!adev->aid_mask && 1199 adev->smuio.funcs && 1200 adev->smuio.funcs->get_socket_id && 1201 adev->smuio.funcs->get_die_id) { 1202 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1203 "%ld deferred hardware errors " 1204 "detected in %s block\n", 1205 adev->smuio.funcs->get_socket_id(adev), 1206 adev->smuio.funcs->get_die_id(adev), 1207 ras_mgr->err_data.de_count, 1208 blk_name); 1209 } else { 1210 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1211 "detected in %s block\n", 1212 ras_mgr->err_data.de_count, 1213 blk_name); 1214 } 1215 } 1216 } 1217 1218 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1219 struct ras_query_if *query_if, 1220 struct ras_err_data *err_data, 1221 struct ras_query_context *qctx) 1222 { 1223 unsigned long new_ue, new_ce, new_de; 1224 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1225 const char *blk_name = get_ras_block_str(&query_if->head); 1226 u64 event_id = qctx->evid.event_id; 1227 1228 new_ce = err_data->ce_count - obj->err_data.ce_count; 1229 new_ue = err_data->ue_count - obj->err_data.ue_count; 1230 new_de = err_data->de_count - obj->err_data.de_count; 1231 1232 if (new_ce) { 1233 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1234 "detected in %s block\n", 1235 new_ce, 1236 blk_name); 1237 } 1238 1239 if (new_ue) { 1240 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1241 "detected in %s block\n", 1242 new_ue, 1243 blk_name); 1244 } 1245 1246 if (new_de) { 1247 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1248 "detected in %s block\n", 1249 new_de, 1250 blk_name); 1251 } 1252 } 1253 1254 static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1255 { 1256 struct ras_err_node *err_node; 1257 struct ras_err_info *err_info; 1258 1259 if (err_data_has_source_info(err_data)) { 1260 for_each_ras_error(err_node, err_data) { 1261 err_info = &err_node->err_info; 1262 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1263 &err_info->mcm_info, err_info->de_count); 1264 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1265 &err_info->mcm_info, err_info->ce_count); 1266 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1267 &err_info->mcm_info, err_info->ue_count); 1268 } 1269 } else { 1270 /* for legacy asic path which doesn't has error source info */ 1271 obj->err_data.ue_count += err_data->ue_count; 1272 obj->err_data.ce_count += err_data->ce_count; 1273 obj->err_data.de_count += err_data->de_count; 1274 } 1275 } 1276 1277 static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1278 struct ras_err_data *err_data) 1279 { 1280 /* Host reports absolute counts */ 1281 obj->err_data.ue_count = err_data->ue_count; 1282 obj->err_data.ce_count = err_data->ce_count; 1283 obj->err_data.de_count = err_data->de_count; 1284 } 1285 1286 static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1287 { 1288 struct ras_common_if head; 1289 1290 memset(&head, 0, sizeof(head)); 1291 head.block = blk; 1292 1293 return amdgpu_ras_find_obj(adev, &head); 1294 } 1295 1296 int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1297 const struct aca_info *aca_info, void *data) 1298 { 1299 struct ras_manager *obj; 1300 1301 /* in resume phase, no need to create aca fs node */ 1302 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1303 return 0; 1304 1305 obj = get_ras_manager(adev, blk); 1306 if (!obj) 1307 return -EINVAL; 1308 1309 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1310 } 1311 1312 int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1313 { 1314 struct ras_manager *obj; 1315 1316 obj = get_ras_manager(adev, blk); 1317 if (!obj) 1318 return -EINVAL; 1319 1320 amdgpu_aca_remove_handle(&obj->aca_handle); 1321 1322 return 0; 1323 } 1324 1325 static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1326 enum aca_error_type type, struct ras_err_data *err_data, 1327 struct ras_query_context *qctx) 1328 { 1329 struct ras_manager *obj; 1330 1331 obj = get_ras_manager(adev, blk); 1332 if (!obj) 1333 return -EINVAL; 1334 1335 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1336 } 1337 1338 ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1339 struct aca_handle *handle, char *buf, void *data) 1340 { 1341 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1342 struct ras_query_if info = { 1343 .head = obj->head, 1344 }; 1345 1346 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1347 return sysfs_emit(buf, "Query currently inaccessible\n"); 1348 1349 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1350 return -EINVAL; 1351 1352 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1353 "ce", info.ce_count, "de", info.de_count); 1354 } 1355 1356 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1357 struct ras_query_if *info, 1358 struct ras_err_data *err_data, 1359 struct ras_query_context *qctx, 1360 unsigned int error_query_mode) 1361 { 1362 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1363 struct amdgpu_ras_block_object *block_obj = NULL; 1364 int ret; 1365 1366 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1367 return -EINVAL; 1368 1369 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1370 return -EINVAL; 1371 1372 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1373 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1374 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1375 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1376 amdgpu_ras_get_ecc_info(adev, err_data); 1377 } else { 1378 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1379 if (!block_obj || !block_obj->hw_ops) { 1380 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1381 get_ras_block_str(&info->head)); 1382 return -EINVAL; 1383 } 1384 1385 if (block_obj->hw_ops->query_ras_error_count) 1386 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1387 1388 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1389 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1390 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1391 if (block_obj->hw_ops->query_ras_error_status) 1392 block_obj->hw_ops->query_ras_error_status(adev); 1393 } 1394 } 1395 } else { 1396 if (amdgpu_aca_is_enabled(adev)) { 1397 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1398 if (ret) 1399 return ret; 1400 1401 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1402 if (ret) 1403 return ret; 1404 1405 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1406 if (ret) 1407 return ret; 1408 } else { 1409 /* FIXME: add code to check return value later */ 1410 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1411 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1412 } 1413 } 1414 1415 return 0; 1416 } 1417 1418 /* query/inject/cure begin */ 1419 static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1420 struct ras_query_if *info, 1421 enum ras_event_type type) 1422 { 1423 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1424 struct ras_err_data err_data; 1425 struct ras_query_context qctx; 1426 unsigned int error_query_mode; 1427 int ret; 1428 1429 if (!obj) 1430 return -EINVAL; 1431 1432 ret = amdgpu_ras_error_data_init(&err_data); 1433 if (ret) 1434 return ret; 1435 1436 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1437 return -EINVAL; 1438 1439 memset(&qctx, 0, sizeof(qctx)); 1440 qctx.evid.type = type; 1441 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1442 1443 if (!down_read_trylock(&adev->reset_domain->sem)) { 1444 ret = -EIO; 1445 goto out_fini_err_data; 1446 } 1447 1448 ret = amdgpu_ras_query_error_status_helper(adev, info, 1449 &err_data, 1450 &qctx, 1451 error_query_mode); 1452 up_read(&adev->reset_domain->sem); 1453 if (ret) 1454 goto out_fini_err_data; 1455 1456 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1457 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1458 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1459 } else { 1460 /* Host provides absolute error counts. First generate the report 1461 * using the previous VF internal count against new host count. 1462 * Then Update VF internal count. 1463 */ 1464 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1465 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1466 } 1467 1468 info->ue_count = obj->err_data.ue_count; 1469 info->ce_count = obj->err_data.ce_count; 1470 info->de_count = obj->err_data.de_count; 1471 1472 out_fini_err_data: 1473 amdgpu_ras_error_data_fini(&err_data); 1474 1475 return ret; 1476 } 1477 1478 int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1479 { 1480 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1481 } 1482 1483 int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1484 enum amdgpu_ras_block block) 1485 { 1486 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1487 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1488 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1489 1490 if (!block_obj || !block_obj->hw_ops) { 1491 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1492 ras_block_str(block)); 1493 return -EOPNOTSUPP; 1494 } 1495 1496 if (!amdgpu_ras_is_supported(adev, block) || 1497 !amdgpu_ras_get_aca_debug_mode(adev)) 1498 return -EOPNOTSUPP; 1499 1500 /* skip ras error reset in gpu reset */ 1501 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1502 ((smu_funcs && smu_funcs->set_debug_mode) || 1503 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1504 return -EOPNOTSUPP; 1505 1506 if (block_obj->hw_ops->reset_ras_error_count) 1507 block_obj->hw_ops->reset_ras_error_count(adev); 1508 1509 return 0; 1510 } 1511 1512 int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1513 enum amdgpu_ras_block block) 1514 { 1515 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1516 1517 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1518 return 0; 1519 1520 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1521 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1522 if (block_obj->hw_ops->reset_ras_error_status) 1523 block_obj->hw_ops->reset_ras_error_status(adev); 1524 } 1525 1526 return 0; 1527 } 1528 1529 /* wrapper of psp_ras_trigger_error */ 1530 int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1531 struct ras_inject_if *info) 1532 { 1533 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1534 struct ta_ras_trigger_error_input block_info = { 1535 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1536 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1537 .sub_block_index = info->head.sub_block_index, 1538 .address = info->address, 1539 .value = info->value, 1540 }; 1541 int ret = -EINVAL; 1542 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1543 info->head.block, 1544 info->head.sub_block_index); 1545 1546 /* inject on guest isn't allowed, return success directly */ 1547 if (amdgpu_sriov_vf(adev)) 1548 return 0; 1549 1550 if (!obj) 1551 return -EINVAL; 1552 1553 if (!block_obj || !block_obj->hw_ops) { 1554 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1555 get_ras_block_str(&info->head)); 1556 return -EINVAL; 1557 } 1558 1559 /* Calculate XGMI relative offset */ 1560 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1561 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1562 block_info.address = 1563 amdgpu_xgmi_get_relative_phy_addr(adev, 1564 block_info.address); 1565 } 1566 1567 if (block_obj->hw_ops->ras_error_inject) { 1568 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1569 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1570 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1571 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1572 info->instance_mask); 1573 } else { 1574 /* default path */ 1575 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1576 } 1577 1578 if (ret) 1579 dev_err(adev->dev, "ras inject %s failed %d\n", 1580 get_ras_block_str(&info->head), ret); 1581 1582 return ret; 1583 } 1584 1585 /** 1586 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1587 * @adev: pointer to AMD GPU device 1588 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1589 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1590 * @query_info: pointer to ras_query_if 1591 * 1592 * Return 0 for query success or do nothing, otherwise return an error 1593 * on failures 1594 */ 1595 static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1596 unsigned long *ce_count, 1597 unsigned long *ue_count, 1598 struct ras_query_if *query_info) 1599 { 1600 int ret; 1601 1602 if (!query_info) 1603 /* do nothing if query_info is not specified */ 1604 return 0; 1605 1606 ret = amdgpu_ras_query_error_status(adev, query_info); 1607 if (ret) 1608 return ret; 1609 1610 *ce_count += query_info->ce_count; 1611 *ue_count += query_info->ue_count; 1612 1613 /* some hardware/IP supports read to clear 1614 * no need to explictly reset the err status after the query call */ 1615 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1616 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1617 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1618 dev_warn(adev->dev, 1619 "Failed to reset error counter and error status\n"); 1620 } 1621 1622 return 0; 1623 } 1624 1625 /** 1626 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1627 * @adev: pointer to AMD GPU device 1628 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1629 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1630 * errors. 1631 * @query_info: pointer to ras_query_if if the query request is only for 1632 * specific ip block; if info is NULL, then the qurey request is for 1633 * all the ip blocks that support query ras error counters/status 1634 * 1635 * If set, @ce_count or @ue_count, count and return the corresponding 1636 * error counts in those integer pointers. Return 0 if the device 1637 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1638 */ 1639 int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1640 unsigned long *ce_count, 1641 unsigned long *ue_count, 1642 struct ras_query_if *query_info) 1643 { 1644 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1645 struct ras_manager *obj; 1646 unsigned long ce, ue; 1647 int ret; 1648 1649 if (!adev->ras_enabled || !con) 1650 return -EOPNOTSUPP; 1651 1652 /* Don't count since no reporting. 1653 */ 1654 if (!ce_count && !ue_count) 1655 return 0; 1656 1657 ce = 0; 1658 ue = 0; 1659 if (!query_info) { 1660 /* query all the ip blocks that support ras query interface */ 1661 list_for_each_entry(obj, &con->head, node) { 1662 struct ras_query_if info = { 1663 .head = obj->head, 1664 }; 1665 1666 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1667 } 1668 } else { 1669 /* query specific ip block */ 1670 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1671 } 1672 1673 if (ret) 1674 return ret; 1675 1676 if (ce_count) 1677 *ce_count = ce; 1678 1679 if (ue_count) 1680 *ue_count = ue; 1681 1682 return 0; 1683 } 1684 /* query/inject/cure end */ 1685 1686 1687 /* sysfs begin */ 1688 1689 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1690 struct ras_badpage **bps, unsigned int *count); 1691 1692 static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1693 { 1694 switch (flags) { 1695 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1696 return "R"; 1697 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1698 return "P"; 1699 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1700 default: 1701 return "F"; 1702 } 1703 } 1704 1705 /** 1706 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1707 * 1708 * It allows user to read the bad pages of vram on the gpu through 1709 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1710 * 1711 * It outputs multiple lines, and each line stands for one gpu page. 1712 * 1713 * The format of one line is below, 1714 * gpu pfn : gpu page size : flags 1715 * 1716 * gpu pfn and gpu page size are printed in hex format. 1717 * flags can be one of below character, 1718 * 1719 * R: reserved, this gpu page is reserved and not able to use. 1720 * 1721 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1722 * in next window of page_reserve. 1723 * 1724 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1725 * 1726 * Examples: 1727 * 1728 * .. code-block:: bash 1729 * 1730 * 0x00000001 : 0x00001000 : R 1731 * 0x00000002 : 0x00001000 : P 1732 * 1733 */ 1734 1735 static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1736 struct kobject *kobj, struct bin_attribute *attr, 1737 char *buf, loff_t ppos, size_t count) 1738 { 1739 struct amdgpu_ras *con = 1740 container_of(attr, struct amdgpu_ras, badpages_attr); 1741 struct amdgpu_device *adev = con->adev; 1742 const unsigned int element_size = 1743 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1744 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1745 unsigned int end = div64_ul(ppos + count - 1, element_size); 1746 ssize_t s = 0; 1747 struct ras_badpage *bps = NULL; 1748 unsigned int bps_count = 0; 1749 1750 memset(buf, 0, count); 1751 1752 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count)) 1753 return 0; 1754 1755 for (; start < end && start < bps_count; start++) 1756 s += scnprintf(&buf[s], element_size + 1, 1757 "0x%08x : 0x%08x : %1s\n", 1758 bps[start].bp, 1759 bps[start].size, 1760 amdgpu_ras_badpage_flags_str(bps[start].flags)); 1761 1762 kfree(bps); 1763 1764 return s; 1765 } 1766 1767 static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1768 struct device_attribute *attr, char *buf) 1769 { 1770 struct amdgpu_ras *con = 1771 container_of(attr, struct amdgpu_ras, features_attr); 1772 1773 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1774 } 1775 1776 static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1777 struct device_attribute *attr, char *buf) 1778 { 1779 struct amdgpu_ras *con = 1780 container_of(attr, struct amdgpu_ras, version_attr); 1781 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version); 1782 } 1783 1784 static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1785 struct device_attribute *attr, char *buf) 1786 { 1787 struct amdgpu_ras *con = 1788 container_of(attr, struct amdgpu_ras, schema_attr); 1789 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1790 } 1791 1792 static struct { 1793 enum ras_event_type type; 1794 const char *name; 1795 } dump_event[] = { 1796 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 1797 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 1798 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 1799 }; 1800 1801 static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 1802 struct device_attribute *attr, char *buf) 1803 { 1804 struct amdgpu_ras *con = 1805 container_of(attr, struct amdgpu_ras, event_state_attr); 1806 struct ras_event_manager *event_mgr = con->event_mgr; 1807 struct ras_event_state *event_state; 1808 int i, size = 0; 1809 1810 if (!event_mgr) 1811 return -EINVAL; 1812 1813 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 1814 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 1815 event_state = &event_mgr->event_state[dump_event[i].type]; 1816 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 1817 dump_event[i].name, 1818 atomic64_read(&event_state->count), 1819 event_state->last_seqno); 1820 } 1821 1822 return (ssize_t)size; 1823 } 1824 1825 static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 1826 { 1827 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1828 1829 if (adev->dev->kobj.sd) 1830 sysfs_remove_file_from_group(&adev->dev->kobj, 1831 &con->badpages_attr.attr, 1832 RAS_FS_NAME); 1833 } 1834 1835 static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 1836 { 1837 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1838 struct attribute *attrs[] = { 1839 &con->features_attr.attr, 1840 &con->version_attr.attr, 1841 &con->schema_attr.attr, 1842 &con->event_state_attr.attr, 1843 NULL 1844 }; 1845 struct attribute_group group = { 1846 .name = RAS_FS_NAME, 1847 .attrs = attrs, 1848 }; 1849 1850 if (adev->dev->kobj.sd) 1851 sysfs_remove_group(&adev->dev->kobj, &group); 1852 1853 return 0; 1854 } 1855 1856 int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 1857 struct ras_common_if *head) 1858 { 1859 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1860 1861 if (amdgpu_aca_is_enabled(adev)) 1862 return 0; 1863 1864 if (!obj || obj->attr_inuse) 1865 return -EINVAL; 1866 1867 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 1868 return 0; 1869 1870 get_obj(obj); 1871 1872 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 1873 "%s_err_count", head->name); 1874 1875 obj->sysfs_attr = (struct device_attribute){ 1876 .attr = { 1877 .name = obj->fs_data.sysfs_name, 1878 .mode = S_IRUGO, 1879 }, 1880 .show = amdgpu_ras_sysfs_read, 1881 }; 1882 sysfs_attr_init(&obj->sysfs_attr.attr); 1883 1884 if (sysfs_add_file_to_group(&adev->dev->kobj, 1885 &obj->sysfs_attr.attr, 1886 RAS_FS_NAME)) { 1887 put_obj(obj); 1888 return -EINVAL; 1889 } 1890 1891 obj->attr_inuse = 1; 1892 1893 return 0; 1894 } 1895 1896 int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 1897 struct ras_common_if *head) 1898 { 1899 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 1900 1901 if (amdgpu_aca_is_enabled(adev)) 1902 return 0; 1903 1904 if (!obj || !obj->attr_inuse) 1905 return -EINVAL; 1906 1907 if (adev->dev->kobj.sd) 1908 sysfs_remove_file_from_group(&adev->dev->kobj, 1909 &obj->sysfs_attr.attr, 1910 RAS_FS_NAME); 1911 obj->attr_inuse = 0; 1912 put_obj(obj); 1913 1914 return 0; 1915 } 1916 1917 static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 1918 { 1919 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1920 struct ras_manager *obj, *tmp; 1921 1922 list_for_each_entry_safe(obj, tmp, &con->head, node) { 1923 amdgpu_ras_sysfs_remove(adev, &obj->head); 1924 } 1925 1926 if (amdgpu_bad_page_threshold != 0) 1927 amdgpu_ras_sysfs_remove_bad_page_node(adev); 1928 1929 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 1930 1931 return 0; 1932 } 1933 /* sysfs end */ 1934 1935 /** 1936 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 1937 * 1938 * Normally when there is an uncorrectable error, the driver will reset 1939 * the GPU to recover. However, in the event of an unrecoverable error, 1940 * the driver provides an interface to reboot the system automatically 1941 * in that event. 1942 * 1943 * The following file in debugfs provides that interface: 1944 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 1945 * 1946 * Usage: 1947 * 1948 * .. code-block:: bash 1949 * 1950 * echo true > .../ras/auto_reboot 1951 * 1952 */ 1953 /* debugfs begin */ 1954 static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 1955 { 1956 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1957 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 1958 struct drm_minor *minor = adev_to_drm(adev)->primary; 1959 struct dentry *dir; 1960 1961 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 1962 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 1963 &amdgpu_ras_debugfs_ctrl_ops); 1964 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 1965 &amdgpu_ras_debugfs_eeprom_ops); 1966 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 1967 &con->bad_page_cnt_threshold); 1968 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 1969 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 1970 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 1971 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 1972 &amdgpu_ras_debugfs_eeprom_size_ops); 1973 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 1974 S_IRUGO, dir, adev, 1975 &amdgpu_ras_debugfs_eeprom_table_ops); 1976 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 1977 1978 /* 1979 * After one uncorrectable error happens, usually GPU recovery will 1980 * be scheduled. But due to the known problem in GPU recovery failing 1981 * to bring GPU back, below interface provides one direct way to 1982 * user to reboot system automatically in such case within 1983 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 1984 * will never be called. 1985 */ 1986 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 1987 1988 /* 1989 * User could set this not to clean up hardware's error count register 1990 * of RAS IPs during ras recovery. 1991 */ 1992 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 1993 &con->disable_ras_err_cnt_harvest); 1994 return dir; 1995 } 1996 1997 static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 1998 struct ras_fs_if *head, 1999 struct dentry *dir) 2000 { 2001 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2002 2003 if (!obj || !dir) 2004 return; 2005 2006 get_obj(obj); 2007 2008 memcpy(obj->fs_data.debugfs_name, 2009 head->debugfs_name, 2010 sizeof(obj->fs_data.debugfs_name)); 2011 2012 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2013 obj, &amdgpu_ras_debugfs_ops); 2014 } 2015 2016 static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2017 { 2018 bool ret; 2019 2020 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2021 case IP_VERSION(13, 0, 6): 2022 case IP_VERSION(13, 0, 12): 2023 case IP_VERSION(13, 0, 14): 2024 ret = true; 2025 break; 2026 default: 2027 ret = false; 2028 break; 2029 } 2030 2031 return ret; 2032 } 2033 2034 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2035 { 2036 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2037 struct dentry *dir; 2038 struct ras_manager *obj; 2039 struct ras_fs_if fs_info; 2040 2041 /* 2042 * it won't be called in resume path, no need to check 2043 * suspend and gpu reset status 2044 */ 2045 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2046 return; 2047 2048 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2049 2050 list_for_each_entry(obj, &con->head, node) { 2051 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2052 (obj->attr_inuse == 1)) { 2053 sprintf(fs_info.debugfs_name, "%s_err_inject", 2054 get_ras_block_str(&obj->head)); 2055 fs_info.head = obj->head; 2056 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2057 } 2058 } 2059 2060 if (amdgpu_ras_aca_is_supported(adev)) { 2061 if (amdgpu_aca_is_enabled(adev)) 2062 amdgpu_aca_smu_debugfs_init(adev, dir); 2063 else 2064 amdgpu_mca_smu_debugfs_init(adev, dir); 2065 } 2066 } 2067 2068 /* debugfs end */ 2069 2070 /* ras fs */ 2071 static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2072 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2073 static DEVICE_ATTR(features, S_IRUGO, 2074 amdgpu_ras_sysfs_features_read, NULL); 2075 static DEVICE_ATTR(version, 0444, 2076 amdgpu_ras_sysfs_version_show, NULL); 2077 static DEVICE_ATTR(schema, 0444, 2078 amdgpu_ras_sysfs_schema_show, NULL); 2079 static DEVICE_ATTR(event_state, 0444, 2080 amdgpu_ras_sysfs_event_state_show, NULL); 2081 static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2082 { 2083 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2084 struct attribute_group group = { 2085 .name = RAS_FS_NAME, 2086 }; 2087 struct attribute *attrs[] = { 2088 &con->features_attr.attr, 2089 &con->version_attr.attr, 2090 &con->schema_attr.attr, 2091 &con->event_state_attr.attr, 2092 NULL 2093 }; 2094 struct bin_attribute *bin_attrs[] = { 2095 NULL, 2096 NULL, 2097 }; 2098 int r; 2099 2100 group.attrs = attrs; 2101 2102 /* add features entry */ 2103 con->features_attr = dev_attr_features; 2104 sysfs_attr_init(attrs[0]); 2105 2106 /* add version entry */ 2107 con->version_attr = dev_attr_version; 2108 sysfs_attr_init(attrs[1]); 2109 2110 /* add schema entry */ 2111 con->schema_attr = dev_attr_schema; 2112 sysfs_attr_init(attrs[2]); 2113 2114 /* add event_state entry */ 2115 con->event_state_attr = dev_attr_event_state; 2116 sysfs_attr_init(attrs[3]); 2117 2118 if (amdgpu_bad_page_threshold != 0) { 2119 /* add bad_page_features entry */ 2120 bin_attr_gpu_vram_bad_pages.private = NULL; 2121 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2122 bin_attrs[0] = &con->badpages_attr; 2123 group.bin_attrs = bin_attrs; 2124 sysfs_bin_attr_init(bin_attrs[0]); 2125 } 2126 2127 r = sysfs_create_group(&adev->dev->kobj, &group); 2128 if (r) 2129 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2130 2131 return 0; 2132 } 2133 2134 static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2135 { 2136 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2137 struct ras_manager *con_obj, *ip_obj, *tmp; 2138 2139 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2140 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2141 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2142 if (ip_obj) 2143 put_obj(ip_obj); 2144 } 2145 } 2146 2147 amdgpu_ras_sysfs_remove_all(adev); 2148 return 0; 2149 } 2150 /* ras fs end */ 2151 2152 /* ih begin */ 2153 2154 /* For the hardware that cannot enable bif ring for both ras_controller_irq 2155 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2156 * register to check whether the interrupt is triggered or not, and properly 2157 * ack the interrupt if it is there 2158 */ 2159 void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2160 { 2161 /* Fatal error events are handled on host side */ 2162 if (amdgpu_sriov_vf(adev)) 2163 return; 2164 /** 2165 * If the current interrupt is caused by a non-fatal RAS error, skip 2166 * check for fatal error. For fatal errors, FED status of all devices 2167 * in XGMI hive gets set when the first device gets fatal error 2168 * interrupt. The error gets propagated to other devices as well, so 2169 * make sure to ack the interrupt regardless of FED status. 2170 */ 2171 if (!amdgpu_ras_get_fed_status(adev) && 2172 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2173 return; 2174 2175 if (adev->nbio.ras && 2176 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2177 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2178 2179 if (adev->nbio.ras && 2180 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2181 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2182 } 2183 2184 static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2185 struct amdgpu_iv_entry *entry) 2186 { 2187 bool poison_stat = false; 2188 struct amdgpu_device *adev = obj->adev; 2189 struct amdgpu_ras_block_object *block_obj = 2190 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2191 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2192 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2193 u64 event_id; 2194 int ret; 2195 2196 if (!block_obj || !con) 2197 return; 2198 2199 ret = amdgpu_ras_mark_ras_event(adev, type); 2200 if (ret) 2201 return; 2202 2203 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2204 /* both query_poison_status and handle_poison_consumption are optional, 2205 * but at least one of them should be implemented if we need poison 2206 * consumption handler 2207 */ 2208 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2209 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2210 if (!poison_stat) { 2211 /* Not poison consumption interrupt, no need to handle it */ 2212 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2213 block_obj->ras_comm.name); 2214 2215 return; 2216 } 2217 } 2218 2219 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2220 2221 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2222 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2223 2224 /* gpu reset is fallback for failed and default cases. 2225 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2226 */ 2227 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2228 event_id = amdgpu_ras_acquire_event_id(adev, type); 2229 RAS_EVENT_LOG(adev, event_id, 2230 "GPU reset for %s RAS poison consumption is issued!\n", 2231 block_obj->ras_comm.name); 2232 amdgpu_ras_reset_gpu(adev); 2233 } 2234 2235 if (!poison_stat) 2236 amdgpu_gfx_poison_consumption_handler(adev, entry); 2237 } 2238 2239 static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2240 struct amdgpu_iv_entry *entry) 2241 { 2242 struct amdgpu_device *adev = obj->adev; 2243 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2244 u64 event_id; 2245 int ret; 2246 2247 ret = amdgpu_ras_mark_ras_event(adev, type); 2248 if (ret) 2249 return; 2250 2251 event_id = amdgpu_ras_acquire_event_id(adev, type); 2252 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2253 2254 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2255 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2256 2257 atomic_inc(&con->page_retirement_req_cnt); 2258 atomic_inc(&con->poison_creation_count); 2259 2260 wake_up(&con->page_retirement_wq); 2261 } 2262 } 2263 2264 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2265 struct amdgpu_iv_entry *entry) 2266 { 2267 struct ras_ih_data *data = &obj->ih_data; 2268 struct ras_err_data err_data; 2269 int ret; 2270 2271 if (!data->cb) 2272 return; 2273 2274 ret = amdgpu_ras_error_data_init(&err_data); 2275 if (ret) 2276 return; 2277 2278 /* Let IP handle its data, maybe we need get the output 2279 * from the callback to update the error type/count, etc 2280 */ 2281 amdgpu_ras_set_fed(obj->adev, true); 2282 ret = data->cb(obj->adev, &err_data, entry); 2283 /* ue will trigger an interrupt, and in that case 2284 * we need do a reset to recovery the whole system. 2285 * But leave IP do that recovery, here we just dispatch 2286 * the error. 2287 */ 2288 if (ret == AMDGPU_RAS_SUCCESS) { 2289 /* these counts could be left as 0 if 2290 * some blocks do not count error number 2291 */ 2292 obj->err_data.ue_count += err_data.ue_count; 2293 obj->err_data.ce_count += err_data.ce_count; 2294 obj->err_data.de_count += err_data.de_count; 2295 } 2296 2297 amdgpu_ras_error_data_fini(&err_data); 2298 } 2299 2300 static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2301 { 2302 struct ras_ih_data *data = &obj->ih_data; 2303 struct amdgpu_iv_entry entry; 2304 2305 while (data->rptr != data->wptr) { 2306 rmb(); 2307 memcpy(&entry, &data->ring[data->rptr], 2308 data->element_size); 2309 2310 wmb(); 2311 data->rptr = (data->aligned_element_size + 2312 data->rptr) % data->ring_size; 2313 2314 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2315 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2316 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2317 else 2318 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2319 } else { 2320 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2321 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2322 else 2323 dev_warn(obj->adev->dev, 2324 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2325 } 2326 } 2327 } 2328 2329 static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2330 { 2331 struct ras_ih_data *data = 2332 container_of(work, struct ras_ih_data, ih_work); 2333 struct ras_manager *obj = 2334 container_of(data, struct ras_manager, ih_data); 2335 2336 amdgpu_ras_interrupt_handler(obj); 2337 } 2338 2339 int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2340 struct ras_dispatch_if *info) 2341 { 2342 struct ras_manager *obj; 2343 struct ras_ih_data *data; 2344 2345 obj = amdgpu_ras_find_obj(adev, &info->head); 2346 if (!obj) 2347 return -EINVAL; 2348 2349 data = &obj->ih_data; 2350 2351 if (data->inuse == 0) 2352 return 0; 2353 2354 /* Might be overflow... */ 2355 memcpy(&data->ring[data->wptr], info->entry, 2356 data->element_size); 2357 2358 wmb(); 2359 data->wptr = (data->aligned_element_size + 2360 data->wptr) % data->ring_size; 2361 2362 schedule_work(&data->ih_work); 2363 2364 return 0; 2365 } 2366 2367 int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2368 struct ras_common_if *head) 2369 { 2370 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2371 struct ras_ih_data *data; 2372 2373 if (!obj) 2374 return -EINVAL; 2375 2376 data = &obj->ih_data; 2377 if (data->inuse == 0) 2378 return 0; 2379 2380 cancel_work_sync(&data->ih_work); 2381 2382 kfree(data->ring); 2383 memset(data, 0, sizeof(*data)); 2384 put_obj(obj); 2385 2386 return 0; 2387 } 2388 2389 int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2390 struct ras_common_if *head) 2391 { 2392 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2393 struct ras_ih_data *data; 2394 struct amdgpu_ras_block_object *ras_obj; 2395 2396 if (!obj) { 2397 /* in case we registe the IH before enable ras feature */ 2398 obj = amdgpu_ras_create_obj(adev, head); 2399 if (!obj) 2400 return -EINVAL; 2401 } else 2402 get_obj(obj); 2403 2404 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2405 2406 data = &obj->ih_data; 2407 /* add the callback.etc */ 2408 *data = (struct ras_ih_data) { 2409 .inuse = 0, 2410 .cb = ras_obj->ras_cb, 2411 .element_size = sizeof(struct amdgpu_iv_entry), 2412 .rptr = 0, 2413 .wptr = 0, 2414 }; 2415 2416 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2417 2418 data->aligned_element_size = ALIGN(data->element_size, 8); 2419 /* the ring can store 64 iv entries. */ 2420 data->ring_size = 64 * data->aligned_element_size; 2421 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2422 if (!data->ring) { 2423 put_obj(obj); 2424 return -ENOMEM; 2425 } 2426 2427 /* IH is ready */ 2428 data->inuse = 1; 2429 2430 return 0; 2431 } 2432 2433 static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2434 { 2435 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2436 struct ras_manager *obj, *tmp; 2437 2438 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2439 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2440 } 2441 2442 return 0; 2443 } 2444 /* ih end */ 2445 2446 /* traversal all IPs except NBIO to query error counter */ 2447 static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2448 { 2449 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2450 struct ras_manager *obj; 2451 2452 if (!adev->ras_enabled || !con) 2453 return; 2454 2455 list_for_each_entry(obj, &con->head, node) { 2456 struct ras_query_if info = { 2457 .head = obj->head, 2458 }; 2459 2460 /* 2461 * PCIE_BIF IP has one different isr by ras controller 2462 * interrupt, the specific ras counter query will be 2463 * done in that isr. So skip such block from common 2464 * sync flood interrupt isr calling. 2465 */ 2466 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2467 continue; 2468 2469 /* 2470 * this is a workaround for aldebaran, skip send msg to 2471 * smu to get ecc_info table due to smu handle get ecc 2472 * info table failed temporarily. 2473 * should be removed until smu fix handle ecc_info table. 2474 */ 2475 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2476 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2477 IP_VERSION(13, 0, 2))) 2478 continue; 2479 2480 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2481 2482 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2483 IP_VERSION(11, 0, 2) && 2484 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2485 IP_VERSION(11, 0, 4) && 2486 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2487 IP_VERSION(13, 0, 0)) { 2488 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2489 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2490 } 2491 } 2492 } 2493 2494 /* Parse RdRspStatus and WrRspStatus */ 2495 static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2496 struct ras_query_if *info) 2497 { 2498 struct amdgpu_ras_block_object *block_obj; 2499 /* 2500 * Only two block need to query read/write 2501 * RspStatus at current state 2502 */ 2503 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2504 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2505 return; 2506 2507 block_obj = amdgpu_ras_get_ras_block(adev, 2508 info->head.block, 2509 info->head.sub_block_index); 2510 2511 if (!block_obj || !block_obj->hw_ops) { 2512 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2513 get_ras_block_str(&info->head)); 2514 return; 2515 } 2516 2517 if (block_obj->hw_ops->query_ras_error_status) 2518 block_obj->hw_ops->query_ras_error_status(adev); 2519 2520 } 2521 2522 static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2523 { 2524 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2525 struct ras_manager *obj; 2526 2527 if (!adev->ras_enabled || !con) 2528 return; 2529 2530 list_for_each_entry(obj, &con->head, node) { 2531 struct ras_query_if info = { 2532 .head = obj->head, 2533 }; 2534 2535 amdgpu_ras_error_status_query(adev, &info); 2536 } 2537 } 2538 2539 /* recovery begin */ 2540 2541 /* return 0 on success. 2542 * caller need free bps. 2543 */ 2544 static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2545 struct ras_badpage **bps, unsigned int *count) 2546 { 2547 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2548 struct ras_err_handler_data *data; 2549 int i = 0; 2550 int ret = 0, status; 2551 2552 if (!con || !con->eh_data || !bps || !count) 2553 return -EINVAL; 2554 2555 mutex_lock(&con->recovery_lock); 2556 data = con->eh_data; 2557 if (!data || data->count == 0) { 2558 *bps = NULL; 2559 ret = -EINVAL; 2560 goto out; 2561 } 2562 2563 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL); 2564 if (!*bps) { 2565 ret = -ENOMEM; 2566 goto out; 2567 } 2568 2569 for (; i < data->count; i++) { 2570 (*bps)[i] = (struct ras_badpage){ 2571 .bp = data->bps[i].retired_page, 2572 .size = AMDGPU_GPU_PAGE_SIZE, 2573 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED, 2574 }; 2575 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 2576 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT); 2577 if (status == -EBUSY) 2578 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 2579 else if (status == -ENOENT) 2580 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 2581 } 2582 2583 *count = data->count; 2584 out: 2585 mutex_unlock(&con->recovery_lock); 2586 return ret; 2587 } 2588 2589 static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2590 struct amdgpu_hive_info *hive, bool status) 2591 { 2592 struct amdgpu_device *tmp_adev; 2593 2594 if (hive) { 2595 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2596 amdgpu_ras_set_fed(tmp_adev, status); 2597 } else { 2598 amdgpu_ras_set_fed(adev, status); 2599 } 2600 } 2601 2602 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2603 { 2604 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2605 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2606 int hive_ras_recovery = 0; 2607 2608 if (hive) { 2609 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2610 amdgpu_put_xgmi_hive(hive); 2611 } 2612 2613 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2614 return true; 2615 2616 return false; 2617 } 2618 2619 static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2620 { 2621 if (amdgpu_ras_intr_triggered()) 2622 return RAS_EVENT_TYPE_FATAL; 2623 else 2624 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2625 } 2626 2627 static void amdgpu_ras_do_recovery(struct work_struct *work) 2628 { 2629 struct amdgpu_ras *ras = 2630 container_of(work, struct amdgpu_ras, recovery_work); 2631 struct amdgpu_device *remote_adev = NULL; 2632 struct amdgpu_device *adev = ras->adev; 2633 struct list_head device_list, *device_list_handle = NULL; 2634 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2635 enum ras_event_type type; 2636 2637 if (hive) { 2638 atomic_set(&hive->ras_recovery, 1); 2639 2640 /* If any device which is part of the hive received RAS fatal 2641 * error interrupt, set fatal error status on all. This 2642 * condition will need a recovery, and flag will be cleared 2643 * as part of recovery. 2644 */ 2645 list_for_each_entry(remote_adev, &hive->device_list, 2646 gmc.xgmi.head) 2647 if (amdgpu_ras_get_fed_status(remote_adev)) { 2648 amdgpu_ras_set_fed_all(adev, hive, true); 2649 break; 2650 } 2651 } 2652 if (!ras->disable_ras_err_cnt_harvest) { 2653 2654 /* Build list of devices to query RAS related errors */ 2655 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2656 device_list_handle = &hive->device_list; 2657 } else { 2658 INIT_LIST_HEAD(&device_list); 2659 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2660 device_list_handle = &device_list; 2661 } 2662 2663 type = amdgpu_ras_get_fatal_error_event(adev); 2664 list_for_each_entry(remote_adev, 2665 device_list_handle, gmc.xgmi.head) { 2666 amdgpu_ras_query_err_status(remote_adev); 2667 amdgpu_ras_log_on_err_counter(remote_adev, type); 2668 } 2669 2670 } 2671 2672 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2673 struct amdgpu_reset_context reset_context; 2674 memset(&reset_context, 0, sizeof(reset_context)); 2675 2676 reset_context.method = AMD_RESET_METHOD_NONE; 2677 reset_context.reset_req_dev = adev; 2678 reset_context.src = AMDGPU_RESET_SRC_RAS; 2679 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2680 2681 /* Perform full reset in fatal error mode */ 2682 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2683 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2684 else { 2685 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2686 2687 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2688 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2689 reset_context.method = AMD_RESET_METHOD_MODE2; 2690 } 2691 2692 /* Fatal error occurs in poison mode, mode1 reset is used to 2693 * recover gpu. 2694 */ 2695 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2696 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2697 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2698 2699 psp_fatal_error_recovery_quirk(&adev->psp); 2700 } 2701 } 2702 2703 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2704 } 2705 atomic_set(&ras->in_recovery, 0); 2706 if (hive) { 2707 atomic_set(&hive->ras_recovery, 0); 2708 amdgpu_put_xgmi_hive(hive); 2709 } 2710 } 2711 2712 /* alloc/realloc bps array */ 2713 static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2714 struct ras_err_handler_data *data, int pages) 2715 { 2716 unsigned int old_space = data->count + data->space_left; 2717 unsigned int new_space = old_space + pages; 2718 unsigned int align_space = ALIGN(new_space, 512); 2719 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL); 2720 2721 if (!bps) { 2722 return -ENOMEM; 2723 } 2724 2725 if (data->bps) { 2726 memcpy(bps, data->bps, 2727 data->count * sizeof(*data->bps)); 2728 kfree(data->bps); 2729 } 2730 2731 data->bps = bps; 2732 data->space_left += align_space - old_space; 2733 return 0; 2734 } 2735 2736 static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 2737 struct eeprom_table_record *bps, 2738 struct ras_err_data *err_data) 2739 { 2740 struct ta_ras_query_address_input addr_in; 2741 uint32_t socket = 0; 2742 int ret = 0; 2743 2744 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2745 socket = adev->smuio.funcs->get_socket_id(adev); 2746 2747 /* reinit err_data */ 2748 err_data->err_addr_cnt = 0; 2749 err_data->err_addr_len = adev->umc.retire_unit; 2750 2751 memset(&addr_in, 0, sizeof(addr_in)); 2752 addr_in.ma.err_addr = bps->address; 2753 addr_in.ma.socket_id = socket; 2754 addr_in.ma.ch_inst = bps->mem_channel; 2755 /* tell RAS TA the node instance is not used */ 2756 addr_in.ma.node_inst = TA_RAS_INV_NODE; 2757 2758 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2759 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 2760 &addr_in, NULL, false); 2761 2762 return ret; 2763 } 2764 2765 static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 2766 struct eeprom_table_record *bps, 2767 struct ras_err_data *err_data) 2768 { 2769 struct ta_ras_query_address_input addr_in; 2770 uint32_t die_id, socket = 0; 2771 2772 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 2773 socket = adev->smuio.funcs->get_socket_id(adev); 2774 2775 /* although die id is gotten from PA in nps1 mode, the id is 2776 * fitable for any nps mode 2777 */ 2778 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 2779 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 2780 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 2781 else 2782 return -EINVAL; 2783 2784 /* reinit err_data */ 2785 err_data->err_addr_cnt = 0; 2786 err_data->err_addr_len = adev->umc.retire_unit; 2787 2788 memset(&addr_in, 0, sizeof(addr_in)); 2789 addr_in.ma.err_addr = bps->address; 2790 addr_in.ma.ch_inst = bps->mem_channel; 2791 addr_in.ma.umc_inst = bps->mcumc_id; 2792 addr_in.ma.node_inst = die_id; 2793 addr_in.ma.socket_id = socket; 2794 2795 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 2796 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 2797 &addr_in, NULL, false); 2798 else 2799 return -EINVAL; 2800 } 2801 2802 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 2803 struct eeprom_table_record *bps, int count) 2804 { 2805 int j; 2806 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2807 struct ras_err_handler_data *data = con->eh_data; 2808 2809 for (j = 0; j < count; j++) { 2810 if (amdgpu_ras_check_bad_page_unlock(con, 2811 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2812 continue; 2813 2814 if (!data->space_left && 2815 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 2816 return -ENOMEM; 2817 } 2818 2819 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 2820 2821 memcpy(&data->bps[data->count], &(bps[j]), 2822 sizeof(struct eeprom_table_record)); 2823 data->count++; 2824 data->space_left--; 2825 } 2826 2827 return 0; 2828 } 2829 2830 static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 2831 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2832 enum amdgpu_memory_partition nps) 2833 { 2834 int i = 0; 2835 enum amdgpu_memory_partition save_nps; 2836 2837 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2838 2839 /*old asics just have pa in eeprom*/ 2840 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2841 memcpy(err_data->err_addr, bps, 2842 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2843 goto out; 2844 } 2845 2846 for (i = 0; i < adev->umc.retire_unit; i++) 2847 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2848 2849 if (save_nps) { 2850 if (save_nps == nps) { 2851 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2852 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2853 return -EINVAL; 2854 } else { 2855 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 2856 return -EINVAL; 2857 } 2858 } else { 2859 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 2860 if (nps == AMDGPU_NPS1_PARTITION_MODE) 2861 memcpy(err_data->err_addr, bps, 2862 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 2863 else 2864 return -EOPNOTSUPP; 2865 } 2866 } 2867 2868 out: 2869 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 2870 } 2871 2872 static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 2873 struct eeprom_table_record *bps, struct ras_err_data *err_data, 2874 enum amdgpu_memory_partition nps) 2875 { 2876 enum amdgpu_memory_partition save_nps; 2877 2878 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 2879 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 2880 2881 if (save_nps == nps) { 2882 if (amdgpu_umc_pages_in_a_row(adev, err_data, 2883 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 2884 return -EINVAL; 2885 } else { 2886 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 2887 return -EINVAL; 2888 } 2889 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 2890 adev->umc.retire_unit); 2891 } 2892 2893 /* it deal with vram only. */ 2894 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 2895 struct eeprom_table_record *bps, int pages, bool from_rom) 2896 { 2897 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2898 struct ras_err_data err_data; 2899 struct amdgpu_ras_eeprom_control *control = 2900 &adev->psp.ras_context.ras->eeprom_control; 2901 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 2902 int ret = 0; 2903 uint32_t i; 2904 2905 if (!con || !con->eh_data || !bps || pages <= 0) 2906 return 0; 2907 2908 if (from_rom) { 2909 err_data.err_addr = 2910 kcalloc(adev->umc.retire_unit, 2911 sizeof(struct eeprom_table_record), GFP_KERNEL); 2912 if (!err_data.err_addr) { 2913 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 2914 return -ENOMEM; 2915 } 2916 2917 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 2918 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 2919 } 2920 2921 mutex_lock(&con->recovery_lock); 2922 2923 if (from_rom) { 2924 for (i = 0; i < pages; i++) { 2925 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 2926 if ((bps[i].address == bps[i + 1].address) && 2927 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 2928 //deal with retire_unit records a time 2929 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 2930 &bps[i], &err_data, nps); 2931 if (ret) 2932 goto free; 2933 i += (adev->umc.retire_unit - 1); 2934 } else { 2935 break; 2936 } 2937 } else { 2938 break; 2939 } 2940 } 2941 for (; i < pages; i++) { 2942 ret = __amdgpu_ras_convert_rec_from_rom(adev, 2943 &bps[i], &err_data, nps); 2944 if (ret) 2945 goto free; 2946 } 2947 } else { 2948 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 2949 } 2950 2951 free: 2952 if (from_rom) 2953 kfree(err_data.err_addr); 2954 mutex_unlock(&con->recovery_lock); 2955 2956 return ret; 2957 } 2958 2959 /* 2960 * write error record array to eeprom, the function should be 2961 * protected by recovery_lock 2962 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 2963 */ 2964 int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 2965 unsigned long *new_cnt) 2966 { 2967 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2968 struct ras_err_handler_data *data; 2969 struct amdgpu_ras_eeprom_control *control; 2970 int save_count, unit_num, bad_page_num, i; 2971 2972 if (!con || !con->eh_data) { 2973 if (new_cnt) 2974 *new_cnt = 0; 2975 2976 return 0; 2977 } 2978 2979 mutex_lock(&con->recovery_lock); 2980 control = &con->eeprom_control; 2981 data = con->eh_data; 2982 bad_page_num = control->ras_num_bad_pages; 2983 save_count = data->count - bad_page_num; 2984 mutex_unlock(&con->recovery_lock); 2985 2986 unit_num = save_count / adev->umc.retire_unit; 2987 if (new_cnt) 2988 *new_cnt = unit_num; 2989 2990 /* only new entries are saved */ 2991 if (save_count > 0) { 2992 /*old asics only save pa to eeprom like before*/ 2993 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 2994 if (amdgpu_ras_eeprom_append(control, 2995 &data->bps[bad_page_num], save_count)) { 2996 dev_err(adev->dev, "Failed to save EEPROM table data!"); 2997 return -EIO; 2998 } 2999 } else { 3000 for (i = 0; i < unit_num; i++) { 3001 if (amdgpu_ras_eeprom_append(control, 3002 &data->bps[bad_page_num + 3003 i * adev->umc.retire_unit], 1)) { 3004 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3005 return -EIO; 3006 } 3007 } 3008 } 3009 3010 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3011 } 3012 3013 return 0; 3014 } 3015 3016 /* 3017 * read error record array in eeprom and reserve enough space for 3018 * storing new bad pages 3019 */ 3020 static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3021 { 3022 struct amdgpu_ras_eeprom_control *control = 3023 &adev->psp.ras_context.ras->eeprom_control; 3024 struct eeprom_table_record *bps; 3025 int ret, i = 0; 3026 3027 /* no bad page record, skip eeprom access */ 3028 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3029 return 0; 3030 3031 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3032 if (!bps) 3033 return -ENOMEM; 3034 3035 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3036 if (ret) { 3037 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3038 } else { 3039 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3040 for (i = 0; i < control->ras_num_recs; i++) { 3041 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3042 if ((bps[i].address == bps[i + 1].address) && 3043 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3044 control->ras_num_pa_recs += adev->umc.retire_unit; 3045 i += (adev->umc.retire_unit - 1); 3046 } else { 3047 control->ras_num_mca_recs += 3048 (control->ras_num_recs - i); 3049 break; 3050 } 3051 } else { 3052 control->ras_num_mca_recs += (control->ras_num_recs - i); 3053 break; 3054 } 3055 } 3056 } 3057 3058 ret = amdgpu_ras_eeprom_check(control); 3059 if (ret) 3060 goto out; 3061 3062 /* HW not usable */ 3063 if (amdgpu_ras_is_rma(adev)) { 3064 ret = -EHWPOISON; 3065 goto out; 3066 } 3067 3068 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3069 } 3070 3071 out: 3072 kfree(bps); 3073 return ret; 3074 } 3075 3076 static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3077 uint64_t addr) 3078 { 3079 struct ras_err_handler_data *data = con->eh_data; 3080 int i; 3081 3082 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3083 for (i = 0; i < data->count; i++) 3084 if (addr == data->bps[i].retired_page) 3085 return true; 3086 3087 return false; 3088 } 3089 3090 /* 3091 * check if an address belongs to bad page 3092 * 3093 * Note: this check is only for umc block 3094 */ 3095 static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3096 uint64_t addr) 3097 { 3098 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3099 bool ret = false; 3100 3101 if (!con || !con->eh_data) 3102 return ret; 3103 3104 mutex_lock(&con->recovery_lock); 3105 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3106 mutex_unlock(&con->recovery_lock); 3107 return ret; 3108 } 3109 3110 static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3111 uint32_t max_count) 3112 { 3113 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3114 3115 /* 3116 * amdgpu_bad_page_threshold is used to config 3117 * the threshold for the number of bad pages. 3118 * -1: Threshold is set to default value 3119 * Driver will issue a warning message when threshold is reached 3120 * and continue runtime services. 3121 * 0: Disable bad page retirement 3122 * Driver will not retire bad pages 3123 * which is intended for debugging purpose. 3124 * -2: Threshold is determined by a formula 3125 * that assumes 1 bad page per 100M of local memory. 3126 * Driver will continue runtime services when threhold is reached. 3127 * 0 < threshold < max number of bad page records in EEPROM, 3128 * A user-defined threshold is set 3129 * Driver will halt runtime services when this custom threshold is reached. 3130 */ 3131 if (amdgpu_bad_page_threshold == -2) { 3132 u64 val = adev->gmc.mc_vram_size; 3133 3134 do_div(val, RAS_BAD_PAGE_COVER); 3135 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3136 max_count); 3137 } else if (amdgpu_bad_page_threshold == -1) { 3138 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3139 } else { 3140 con->bad_page_cnt_threshold = min_t(int, max_count, 3141 amdgpu_bad_page_threshold); 3142 } 3143 } 3144 3145 int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3146 enum amdgpu_ras_block block, uint16_t pasid, 3147 pasid_notify pasid_fn, void *data, uint32_t reset) 3148 { 3149 int ret = 0; 3150 struct ras_poison_msg poison_msg; 3151 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3152 3153 memset(&poison_msg, 0, sizeof(poison_msg)); 3154 poison_msg.block = block; 3155 poison_msg.pasid = pasid; 3156 poison_msg.reset = reset; 3157 poison_msg.pasid_fn = pasid_fn; 3158 poison_msg.data = data; 3159 3160 ret = kfifo_put(&con->poison_fifo, poison_msg); 3161 if (!ret) { 3162 dev_err(adev->dev, "Poison message fifo is full!\n"); 3163 return -ENOSPC; 3164 } 3165 3166 return 0; 3167 } 3168 3169 static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3170 struct ras_poison_msg *poison_msg) 3171 { 3172 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3173 3174 return kfifo_get(&con->poison_fifo, poison_msg); 3175 } 3176 3177 static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3178 { 3179 mutex_init(&ecc_log->lock); 3180 3181 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3182 ecc_log->de_queried_count = 0; 3183 ecc_log->prev_de_queried_count = 0; 3184 } 3185 3186 static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3187 { 3188 struct radix_tree_iter iter; 3189 void __rcu **slot; 3190 struct ras_ecc_err *ecc_err; 3191 3192 mutex_lock(&ecc_log->lock); 3193 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3194 ecc_err = radix_tree_deref_slot(slot); 3195 kfree(ecc_err->err_pages.pfn); 3196 kfree(ecc_err); 3197 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3198 } 3199 mutex_unlock(&ecc_log->lock); 3200 3201 mutex_destroy(&ecc_log->lock); 3202 ecc_log->de_queried_count = 0; 3203 ecc_log->prev_de_queried_count = 0; 3204 } 3205 3206 static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3207 uint32_t delayed_ms) 3208 { 3209 int ret; 3210 3211 mutex_lock(&con->umc_ecc_log.lock); 3212 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3213 UMC_ECC_NEW_DETECTED_TAG); 3214 mutex_unlock(&con->umc_ecc_log.lock); 3215 3216 if (ret) 3217 schedule_delayed_work(&con->page_retirement_dwork, 3218 msecs_to_jiffies(delayed_ms)); 3219 3220 return ret ? true : false; 3221 } 3222 3223 static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3224 { 3225 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3226 page_retirement_dwork.work); 3227 struct amdgpu_device *adev = con->adev; 3228 struct ras_err_data err_data; 3229 unsigned long err_cnt; 3230 3231 /* If gpu reset is ongoing, delay retiring the bad pages */ 3232 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3233 amdgpu_ras_schedule_retirement_dwork(con, 3234 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3235 return; 3236 } 3237 3238 amdgpu_ras_error_data_init(&err_data); 3239 3240 amdgpu_umc_handle_bad_pages(adev, &err_data); 3241 err_cnt = err_data.err_addr_cnt; 3242 3243 amdgpu_ras_error_data_fini(&err_data); 3244 3245 if (err_cnt && amdgpu_ras_is_rma(adev)) 3246 amdgpu_ras_reset_gpu(adev); 3247 3248 amdgpu_ras_schedule_retirement_dwork(con, 3249 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3250 } 3251 3252 static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3253 uint32_t poison_creation_count) 3254 { 3255 int ret = 0; 3256 struct ras_ecc_log_info *ecc_log; 3257 struct ras_query_if info; 3258 uint32_t timeout = 0; 3259 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3260 uint64_t de_queried_count; 3261 uint32_t new_detect_count, total_detect_count; 3262 uint32_t need_query_count = poison_creation_count; 3263 bool query_data_timeout = false; 3264 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3265 3266 memset(&info, 0, sizeof(info)); 3267 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3268 3269 ecc_log = &ras->umc_ecc_log; 3270 total_detect_count = 0; 3271 do { 3272 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3273 if (ret) 3274 return ret; 3275 3276 de_queried_count = ecc_log->de_queried_count; 3277 if (de_queried_count > ecc_log->prev_de_queried_count) { 3278 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count; 3279 ecc_log->prev_de_queried_count = de_queried_count; 3280 timeout = 0; 3281 } else { 3282 new_detect_count = 0; 3283 } 3284 3285 if (new_detect_count) { 3286 total_detect_count += new_detect_count; 3287 } else { 3288 if (!timeout && need_query_count) 3289 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3290 3291 if (timeout) { 3292 if (!--timeout) { 3293 query_data_timeout = true; 3294 break; 3295 } 3296 msleep(1); 3297 } 3298 } 3299 } while (total_detect_count < need_query_count); 3300 3301 if (query_data_timeout) { 3302 dev_warn(adev->dev, "Can't find deferred error! count: %u\n", 3303 (need_query_count - total_detect_count)); 3304 return -ENOENT; 3305 } 3306 3307 if (total_detect_count) 3308 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3309 3310 return 0; 3311 } 3312 3313 static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3314 { 3315 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3316 struct ras_poison_msg msg; 3317 int ret; 3318 3319 do { 3320 ret = kfifo_get(&con->poison_fifo, &msg); 3321 } while (ret); 3322 } 3323 3324 static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3325 uint32_t msg_count, uint32_t *gpu_reset) 3326 { 3327 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3328 uint32_t reset_flags = 0, reset = 0; 3329 struct ras_poison_msg msg; 3330 int ret, i; 3331 3332 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3333 3334 for (i = 0; i < msg_count; i++) { 3335 ret = amdgpu_ras_get_poison_req(adev, &msg); 3336 if (!ret) 3337 continue; 3338 3339 if (msg.pasid_fn) 3340 msg.pasid_fn(adev, msg.pasid, msg.data); 3341 3342 reset_flags |= msg.reset; 3343 } 3344 3345 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3346 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3347 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3348 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3349 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3350 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3351 else 3352 reset = reset_flags; 3353 3354 flush_delayed_work(&con->page_retirement_dwork); 3355 3356 con->gpu_reset_flags |= reset; 3357 amdgpu_ras_reset_gpu(adev); 3358 3359 *gpu_reset = reset; 3360 3361 /* Wait for gpu recovery to complete */ 3362 flush_work(&con->recovery_work); 3363 } 3364 3365 return 0; 3366 } 3367 3368 static int amdgpu_ras_page_retirement_thread(void *param) 3369 { 3370 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3371 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3372 uint32_t poison_creation_count, msg_count; 3373 uint32_t gpu_reset; 3374 int ret; 3375 3376 while (!kthread_should_stop()) { 3377 3378 wait_event_interruptible(con->page_retirement_wq, 3379 kthread_should_stop() || 3380 atomic_read(&con->page_retirement_req_cnt)); 3381 3382 if (kthread_should_stop()) 3383 break; 3384 3385 gpu_reset = 0; 3386 3387 do { 3388 poison_creation_count = atomic_read(&con->poison_creation_count); 3389 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3390 if (ret == -EIO) 3391 break; 3392 3393 if (poison_creation_count) { 3394 atomic_sub(poison_creation_count, &con->poison_creation_count); 3395 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3396 } 3397 } while (atomic_read(&con->poison_creation_count)); 3398 3399 if (ret != -EIO) { 3400 msg_count = kfifo_len(&con->poison_fifo); 3401 if (msg_count) { 3402 ret = amdgpu_ras_poison_consumption_handler(adev, 3403 msg_count, &gpu_reset); 3404 if ((ret != -EIO) && 3405 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3406 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3407 } 3408 } 3409 3410 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3411 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3412 /* Clear poison creation request */ 3413 atomic_set(&con->poison_creation_count, 0); 3414 3415 /* Clear poison fifo */ 3416 amdgpu_ras_clear_poison_fifo(adev); 3417 3418 /* Clear all poison requests */ 3419 atomic_set(&con->page_retirement_req_cnt, 0); 3420 3421 if (ret == -EIO) { 3422 /* Wait for mode-1 reset to complete */ 3423 down_read(&adev->reset_domain->sem); 3424 up_read(&adev->reset_domain->sem); 3425 } 3426 3427 /* Wake up work to save bad pages to eeprom */ 3428 schedule_delayed_work(&con->page_retirement_dwork, 0); 3429 } else if (gpu_reset) { 3430 /* gpu just completed mode-2 reset or other reset */ 3431 /* Clear poison consumption messages cached in fifo */ 3432 msg_count = kfifo_len(&con->poison_fifo); 3433 if (msg_count) { 3434 amdgpu_ras_clear_poison_fifo(adev); 3435 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3436 } 3437 3438 /* Wake up work to save bad pages to eeprom */ 3439 schedule_delayed_work(&con->page_retirement_dwork, 0); 3440 } 3441 } 3442 3443 return 0; 3444 } 3445 3446 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3447 { 3448 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3449 struct amdgpu_ras_eeprom_control *control; 3450 int ret; 3451 3452 if (!con || amdgpu_sriov_vf(adev)) 3453 return 0; 3454 3455 control = &con->eeprom_control; 3456 ret = amdgpu_ras_eeprom_init(control); 3457 if (ret) 3458 return ret; 3459 3460 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3461 control->ras_num_pa_recs = control->ras_num_recs; 3462 3463 if (control->ras_num_recs) { 3464 ret = amdgpu_ras_load_bad_pages(adev); 3465 if (ret) 3466 return ret; 3467 3468 amdgpu_dpm_send_hbm_bad_pages_num( 3469 adev, control->ras_num_bad_pages); 3470 3471 if (con->update_channel_flag == true) { 3472 amdgpu_dpm_send_hbm_bad_channel_flag( 3473 adev, control->bad_channel_bitmap); 3474 con->update_channel_flag = false; 3475 } 3476 3477 /* The format action is only applied to new ASICs */ 3478 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3479 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3480 if (!amdgpu_ras_eeprom_reset_table(control)) 3481 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3482 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3483 } 3484 3485 return ret; 3486 } 3487 3488 int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3489 { 3490 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3491 struct ras_err_handler_data **data; 3492 u32 max_eeprom_records_count = 0; 3493 int ret; 3494 3495 if (!con || amdgpu_sriov_vf(adev)) 3496 return 0; 3497 3498 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3499 * supports RAS and debugfs is enabled, but when 3500 * adev->ras_enabled is unset, i.e. when "ras_enable" 3501 * module parameter is set to 0. 3502 */ 3503 con->adev = adev; 3504 3505 if (!adev->ras_enabled) 3506 return 0; 3507 3508 data = &con->eh_data; 3509 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3510 if (!*data) { 3511 ret = -ENOMEM; 3512 goto out; 3513 } 3514 3515 mutex_init(&con->recovery_lock); 3516 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3517 atomic_set(&con->in_recovery, 0); 3518 con->eeprom_control.bad_channel_bitmap = 0; 3519 3520 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3521 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3522 3523 if (init_bp_info) { 3524 ret = amdgpu_ras_init_badpage_info(adev); 3525 if (ret) 3526 goto free; 3527 } 3528 3529 mutex_init(&con->page_rsv_lock); 3530 INIT_KFIFO(con->poison_fifo); 3531 mutex_init(&con->page_retirement_lock); 3532 init_waitqueue_head(&con->page_retirement_wq); 3533 atomic_set(&con->page_retirement_req_cnt, 0); 3534 atomic_set(&con->poison_creation_count, 0); 3535 con->page_retirement_thread = 3536 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3537 if (IS_ERR(con->page_retirement_thread)) { 3538 con->page_retirement_thread = NULL; 3539 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3540 } 3541 3542 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3543 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3544 #ifdef CONFIG_X86_MCE_AMD 3545 if ((adev->asic_type == CHIP_ALDEBARAN) && 3546 (adev->gmc.xgmi.connected_to_cpu)) 3547 amdgpu_register_bad_pages_mca_notifier(adev); 3548 #endif 3549 return 0; 3550 3551 free: 3552 kfree((*data)->bps); 3553 kfree(*data); 3554 con->eh_data = NULL; 3555 out: 3556 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3557 3558 /* 3559 * Except error threshold exceeding case, other failure cases in this 3560 * function would not fail amdgpu driver init. 3561 */ 3562 if (!amdgpu_ras_is_rma(adev)) 3563 ret = 0; 3564 else 3565 ret = -EINVAL; 3566 3567 return ret; 3568 } 3569 3570 static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3571 { 3572 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3573 struct ras_err_handler_data *data = con->eh_data; 3574 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3575 bool ret; 3576 3577 /* recovery_init failed to init it, fini is useless */ 3578 if (!data) 3579 return 0; 3580 3581 /* Save all cached bad pages to eeprom */ 3582 do { 3583 flush_delayed_work(&con->page_retirement_dwork); 3584 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3585 } while (ret && max_flush_timeout--); 3586 3587 if (con->page_retirement_thread) 3588 kthread_stop(con->page_retirement_thread); 3589 3590 atomic_set(&con->page_retirement_req_cnt, 0); 3591 atomic_set(&con->poison_creation_count, 0); 3592 3593 mutex_destroy(&con->page_rsv_lock); 3594 3595 cancel_work_sync(&con->recovery_work); 3596 3597 cancel_delayed_work_sync(&con->page_retirement_dwork); 3598 3599 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3600 3601 mutex_lock(&con->recovery_lock); 3602 con->eh_data = NULL; 3603 kfree(data->bps); 3604 kfree(data); 3605 mutex_unlock(&con->recovery_lock); 3606 3607 return 0; 3608 } 3609 /* recovery end */ 3610 3611 static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3612 { 3613 if (amdgpu_sriov_vf(adev)) { 3614 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3615 case IP_VERSION(13, 0, 2): 3616 case IP_VERSION(13, 0, 6): 3617 case IP_VERSION(13, 0, 12): 3618 case IP_VERSION(13, 0, 14): 3619 return true; 3620 default: 3621 return false; 3622 } 3623 } 3624 3625 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3626 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3627 case IP_VERSION(13, 0, 0): 3628 case IP_VERSION(13, 0, 6): 3629 case IP_VERSION(13, 0, 10): 3630 case IP_VERSION(13, 0, 12): 3631 case IP_VERSION(13, 0, 14): 3632 case IP_VERSION(14, 0, 3): 3633 return true; 3634 default: 3635 return false; 3636 } 3637 } 3638 3639 return adev->asic_type == CHIP_VEGA10 || 3640 adev->asic_type == CHIP_VEGA20 || 3641 adev->asic_type == CHIP_ARCTURUS || 3642 adev->asic_type == CHIP_ALDEBARAN || 3643 adev->asic_type == CHIP_SIENNA_CICHLID; 3644 } 3645 3646 /* 3647 * this is workaround for vega20 workstation sku, 3648 * force enable gfx ras, ignore vbios gfx ras flag 3649 * due to GC EDC can not write 3650 */ 3651 static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 3652 { 3653 struct atom_context *ctx = adev->mode_info.atom_context; 3654 3655 if (!ctx) 3656 return; 3657 3658 if (strnstr(ctx->vbios_pn, "D16406", 3659 sizeof(ctx->vbios_pn)) || 3660 strnstr(ctx->vbios_pn, "D36002", 3661 sizeof(ctx->vbios_pn))) 3662 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 3663 } 3664 3665 /* Query ras capablity via atomfirmware interface */ 3666 static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 3667 { 3668 /* mem_ecc cap */ 3669 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 3670 dev_info(adev->dev, "MEM ECC is active.\n"); 3671 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 3672 1 << AMDGPU_RAS_BLOCK__DF); 3673 } else { 3674 dev_info(adev->dev, "MEM ECC is not presented.\n"); 3675 } 3676 3677 /* sram_ecc cap */ 3678 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 3679 dev_info(adev->dev, "SRAM ECC is active.\n"); 3680 if (!amdgpu_sriov_vf(adev)) 3681 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 3682 1 << AMDGPU_RAS_BLOCK__DF); 3683 else 3684 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 3685 1 << AMDGPU_RAS_BLOCK__SDMA | 3686 1 << AMDGPU_RAS_BLOCK__GFX); 3687 3688 /* 3689 * VCN/JPEG RAS can be supported on both bare metal and 3690 * SRIOV environment 3691 */ 3692 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 3693 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 3694 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3)) 3695 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 3696 1 << AMDGPU_RAS_BLOCK__JPEG); 3697 else 3698 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 3699 1 << AMDGPU_RAS_BLOCK__JPEG); 3700 3701 /* 3702 * XGMI RAS is not supported if xgmi num physical nodes 3703 * is zero 3704 */ 3705 if (!adev->gmc.xgmi.num_physical_nodes) 3706 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 3707 } else { 3708 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 3709 } 3710 } 3711 3712 /* Query poison mode from umc/df IP callbacks */ 3713 static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 3714 { 3715 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3716 bool df_poison, umc_poison; 3717 3718 /* poison setting is useless on SRIOV guest */ 3719 if (amdgpu_sriov_vf(adev) || !con) 3720 return; 3721 3722 /* Init poison supported flag, the default value is false */ 3723 if (adev->gmc.xgmi.connected_to_cpu || 3724 adev->gmc.is_app_apu) { 3725 /* enabled by default when GPU is connected to CPU */ 3726 con->poison_supported = true; 3727 } else if (adev->df.funcs && 3728 adev->df.funcs->query_ras_poison_mode && 3729 adev->umc.ras && 3730 adev->umc.ras->query_ras_poison_mode) { 3731 df_poison = 3732 adev->df.funcs->query_ras_poison_mode(adev); 3733 umc_poison = 3734 adev->umc.ras->query_ras_poison_mode(adev); 3735 3736 /* Only poison is set in both DF and UMC, we can support it */ 3737 if (df_poison && umc_poison) 3738 con->poison_supported = true; 3739 else if (df_poison != umc_poison) 3740 dev_warn(adev->dev, 3741 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 3742 df_poison, umc_poison); 3743 } 3744 } 3745 3746 /* 3747 * check hardware's ras ability which will be saved in hw_supported. 3748 * if hardware does not support ras, we can skip some ras initializtion and 3749 * forbid some ras operations from IP. 3750 * if software itself, say boot parameter, limit the ras ability. We still 3751 * need allow IP do some limited operations, like disable. In such case, 3752 * we have to initialize ras as normal. but need check if operation is 3753 * allowed or not in each function. 3754 */ 3755 static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 3756 { 3757 adev->ras_hw_enabled = adev->ras_enabled = 0; 3758 3759 if (!amdgpu_ras_asic_supported(adev)) 3760 return; 3761 3762 if (amdgpu_sriov_vf(adev)) { 3763 if (amdgpu_virt_get_ras_capability(adev)) 3764 goto init_ras_enabled_flag; 3765 } 3766 3767 /* query ras capability from psp */ 3768 if (amdgpu_psp_get_ras_capability(&adev->psp)) 3769 goto init_ras_enabled_flag; 3770 3771 /* query ras capablity from bios */ 3772 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 3773 amdgpu_ras_query_ras_capablity_from_vbios(adev); 3774 } else { 3775 /* driver only manages a few IP blocks RAS feature 3776 * when GPU is connected cpu through XGMI */ 3777 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 3778 1 << AMDGPU_RAS_BLOCK__SDMA | 3779 1 << AMDGPU_RAS_BLOCK__MMHUB); 3780 } 3781 3782 /* apply asic specific settings (vega20 only for now) */ 3783 amdgpu_ras_get_quirks(adev); 3784 3785 /* query poison mode from umc/df ip callback */ 3786 amdgpu_ras_query_poison_mode(adev); 3787 3788 init_ras_enabled_flag: 3789 /* hw_supported needs to be aligned with RAS block mask. */ 3790 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 3791 3792 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 3793 adev->ras_hw_enabled & amdgpu_ras_mask; 3794 3795 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 3796 adev->aca.is_enabled = 3797 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 3798 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 3799 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 3800 3801 /* bad page feature is not applicable to specific app platform */ 3802 if (adev->gmc.is_app_apu && 3803 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 3804 amdgpu_bad_page_threshold = 0; 3805 } 3806 3807 static void amdgpu_ras_counte_dw(struct work_struct *work) 3808 { 3809 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3810 ras_counte_delay_work.work); 3811 struct amdgpu_device *adev = con->adev; 3812 struct drm_device *dev = adev_to_drm(adev); 3813 unsigned long ce_count, ue_count; 3814 int res; 3815 3816 res = pm_runtime_get_sync(dev->dev); 3817 if (res < 0) 3818 goto Out; 3819 3820 /* Cache new values. 3821 */ 3822 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 3823 atomic_set(&con->ras_ce_count, ce_count); 3824 atomic_set(&con->ras_ue_count, ue_count); 3825 } 3826 3827 pm_runtime_mark_last_busy(dev->dev); 3828 Out: 3829 pm_runtime_put_autosuspend(dev->dev); 3830 } 3831 3832 static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 3833 { 3834 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 3835 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 3836 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 3837 AMDGPU_RAS_ERROR__PARITY; 3838 } 3839 3840 static void ras_event_mgr_init(struct ras_event_manager *mgr) 3841 { 3842 struct ras_event_state *event_state; 3843 int i; 3844 3845 memset(mgr, 0, sizeof(*mgr)); 3846 atomic64_set(&mgr->seqno, 0); 3847 3848 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 3849 event_state = &mgr->event_state[i]; 3850 event_state->last_seqno = RAS_EVENT_INVALID_ID; 3851 atomic64_set(&event_state->count, 0); 3852 } 3853 } 3854 3855 static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 3856 { 3857 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3858 struct amdgpu_hive_info *hive; 3859 3860 if (!ras) 3861 return; 3862 3863 hive = amdgpu_get_xgmi_hive(adev); 3864 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 3865 3866 /* init event manager with node 0 on xgmi system */ 3867 if (!amdgpu_reset_in_recovery(adev)) { 3868 if (!hive || adev->gmc.xgmi.node_id == 0) 3869 ras_event_mgr_init(ras->event_mgr); 3870 } 3871 3872 if (hive) 3873 amdgpu_put_xgmi_hive(hive); 3874 } 3875 3876 static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 3877 { 3878 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3879 3880 if (!con || (adev->flags & AMD_IS_APU)) 3881 return; 3882 3883 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3884 case IP_VERSION(13, 0, 2): 3885 case IP_VERSION(13, 0, 6): 3886 case IP_VERSION(13, 0, 12): 3887 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 3888 break; 3889 case IP_VERSION(13, 0, 14): 3890 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 3891 break; 3892 default: 3893 break; 3894 } 3895 } 3896 3897 int amdgpu_ras_init(struct amdgpu_device *adev) 3898 { 3899 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3900 int r; 3901 3902 if (con) 3903 return 0; 3904 3905 con = kzalloc(sizeof(*con) + 3906 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 3907 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 3908 GFP_KERNEL); 3909 if (!con) 3910 return -ENOMEM; 3911 3912 con->adev = adev; 3913 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 3914 atomic_set(&con->ras_ce_count, 0); 3915 atomic_set(&con->ras_ue_count, 0); 3916 3917 con->objs = (struct ras_manager *)(con + 1); 3918 3919 amdgpu_ras_set_context(adev, con); 3920 3921 amdgpu_ras_check_supported(adev); 3922 3923 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 3924 /* set gfx block ras context feature for VEGA20 Gaming 3925 * send ras disable cmd to ras ta during ras late init. 3926 */ 3927 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 3928 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 3929 3930 return 0; 3931 } 3932 3933 r = 0; 3934 goto release_con; 3935 } 3936 3937 con->update_channel_flag = false; 3938 con->features = 0; 3939 con->schema = 0; 3940 INIT_LIST_HEAD(&con->head); 3941 /* Might need get this flag from vbios. */ 3942 con->flags = RAS_DEFAULT_FLAGS; 3943 3944 /* initialize nbio ras function ahead of any other 3945 * ras functions so hardware fatal error interrupt 3946 * can be enabled as early as possible */ 3947 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 3948 case IP_VERSION(7, 4, 0): 3949 case IP_VERSION(7, 4, 1): 3950 case IP_VERSION(7, 4, 4): 3951 if (!adev->gmc.xgmi.connected_to_cpu) 3952 adev->nbio.ras = &nbio_v7_4_ras; 3953 break; 3954 case IP_VERSION(4, 3, 0): 3955 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 3956 /* unlike other generation of nbio ras, 3957 * nbio v4_3 only support fatal error interrupt 3958 * to inform software that DF is freezed due to 3959 * system fatal error event. driver should not 3960 * enable nbio ras in such case. Instead, 3961 * check DF RAS */ 3962 adev->nbio.ras = &nbio_v4_3_ras; 3963 break; 3964 case IP_VERSION(6, 3, 1): 3965 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 3966 /* unlike other generation of nbio ras, 3967 * nbif v6_3_1 only support fatal error interrupt 3968 * to inform software that DF is freezed due to 3969 * system fatal error event. driver should not 3970 * enable nbio ras in such case. Instead, 3971 * check DF RAS 3972 */ 3973 adev->nbio.ras = &nbif_v6_3_1_ras; 3974 break; 3975 case IP_VERSION(7, 9, 0): 3976 case IP_VERSION(7, 9, 1): 3977 if (!adev->gmc.is_app_apu) 3978 adev->nbio.ras = &nbio_v7_9_ras; 3979 break; 3980 default: 3981 /* nbio ras is not available */ 3982 break; 3983 } 3984 3985 /* nbio ras block needs to be enabled ahead of other ras blocks 3986 * to handle fatal error */ 3987 r = amdgpu_nbio_ras_sw_init(adev); 3988 if (r) 3989 return r; 3990 3991 if (adev->nbio.ras && 3992 adev->nbio.ras->init_ras_controller_interrupt) { 3993 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 3994 if (r) 3995 goto release_con; 3996 } 3997 3998 if (adev->nbio.ras && 3999 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4000 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4001 if (r) 4002 goto release_con; 4003 } 4004 4005 /* Packed socket_id to ras feature mask bits[31:29] */ 4006 if (adev->smuio.funcs && 4007 adev->smuio.funcs->get_socket_id) 4008 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4009 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4010 4011 /* Get RAS schema for particular SOC */ 4012 con->schema = amdgpu_get_ras_schema(adev); 4013 4014 amdgpu_ras_init_reserved_vram_size(adev); 4015 4016 if (amdgpu_ras_fs_init(adev)) { 4017 r = -EINVAL; 4018 goto release_con; 4019 } 4020 4021 if (amdgpu_ras_aca_is_supported(adev)) { 4022 if (amdgpu_aca_is_enabled(adev)) 4023 r = amdgpu_aca_init(adev); 4024 else 4025 r = amdgpu_mca_init(adev); 4026 if (r) 4027 goto release_con; 4028 } 4029 4030 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4031 "hardware ability[%x] ras_mask[%x]\n", 4032 adev->ras_hw_enabled, adev->ras_enabled); 4033 4034 return 0; 4035 release_con: 4036 amdgpu_ras_set_context(adev, NULL); 4037 kfree(con); 4038 4039 return r; 4040 } 4041 4042 int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4043 { 4044 if (adev->gmc.xgmi.connected_to_cpu || 4045 adev->gmc.is_app_apu) 4046 return 1; 4047 return 0; 4048 } 4049 4050 static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4051 struct ras_common_if *ras_block) 4052 { 4053 struct ras_query_if info = { 4054 .head = *ras_block, 4055 }; 4056 4057 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4058 return 0; 4059 4060 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4061 DRM_WARN("RAS init harvest failure"); 4062 4063 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4064 DRM_WARN("RAS init harvest reset failure"); 4065 4066 return 0; 4067 } 4068 4069 bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4070 { 4071 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4072 4073 if (!con) 4074 return false; 4075 4076 return con->poison_supported; 4077 } 4078 4079 /* helper function to handle common stuff in ip late init phase */ 4080 int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4081 struct ras_common_if *ras_block) 4082 { 4083 struct amdgpu_ras_block_object *ras_obj = NULL; 4084 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4085 struct ras_query_if *query_info; 4086 unsigned long ue_count, ce_count; 4087 int r; 4088 4089 /* disable RAS feature per IP block if it is not supported */ 4090 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4091 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4092 return 0; 4093 } 4094 4095 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4096 if (r) { 4097 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4098 /* in resume phase, if fail to enable ras, 4099 * clean up all ras fs nodes, and disable ras */ 4100 goto cleanup; 4101 } else 4102 return r; 4103 } 4104 4105 /* check for errors on warm reset edc persisant supported ASIC */ 4106 amdgpu_persistent_edc_harvesting(adev, ras_block); 4107 4108 /* in resume phase, no need to create ras fs node */ 4109 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4110 return 0; 4111 4112 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4113 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4114 (ras_obj->hw_ops->query_poison_status || 4115 ras_obj->hw_ops->handle_poison_consumption))) { 4116 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4117 if (r) 4118 goto cleanup; 4119 } 4120 4121 if (ras_obj->hw_ops && 4122 (ras_obj->hw_ops->query_ras_error_count || 4123 ras_obj->hw_ops->query_ras_error_status)) { 4124 r = amdgpu_ras_sysfs_create(adev, ras_block); 4125 if (r) 4126 goto interrupt; 4127 4128 /* Those are the cached values at init. 4129 */ 4130 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4131 if (!query_info) 4132 return -ENOMEM; 4133 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4134 4135 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4136 atomic_set(&con->ras_ce_count, ce_count); 4137 atomic_set(&con->ras_ue_count, ue_count); 4138 } 4139 4140 kfree(query_info); 4141 } 4142 4143 return 0; 4144 4145 interrupt: 4146 if (ras_obj->ras_cb) 4147 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4148 cleanup: 4149 amdgpu_ras_feature_enable(adev, ras_block, 0); 4150 return r; 4151 } 4152 4153 static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4154 struct ras_common_if *ras_block) 4155 { 4156 return amdgpu_ras_block_late_init(adev, ras_block); 4157 } 4158 4159 /* helper function to remove ras fs node and interrupt handler */ 4160 void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4161 struct ras_common_if *ras_block) 4162 { 4163 struct amdgpu_ras_block_object *ras_obj; 4164 if (!ras_block) 4165 return; 4166 4167 amdgpu_ras_sysfs_remove(adev, ras_block); 4168 4169 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4170 if (ras_obj->ras_cb) 4171 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4172 } 4173 4174 static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4175 struct ras_common_if *ras_block) 4176 { 4177 return amdgpu_ras_block_late_fini(adev, ras_block); 4178 } 4179 4180 /* do some init work after IP late init as dependence. 4181 * and it runs in resume/gpu reset/booting up cases. 4182 */ 4183 void amdgpu_ras_resume(struct amdgpu_device *adev) 4184 { 4185 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4186 struct ras_manager *obj, *tmp; 4187 4188 if (!adev->ras_enabled || !con) { 4189 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4190 amdgpu_release_ras_context(adev); 4191 4192 return; 4193 } 4194 4195 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4196 /* Set up all other IPs which are not implemented. There is a 4197 * tricky thing that IP's actual ras error type should be 4198 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4199 * ERROR_NONE make sense anyway. 4200 */ 4201 amdgpu_ras_enable_all_features(adev, 1); 4202 4203 /* We enable ras on all hw_supported block, but as boot 4204 * parameter might disable some of them and one or more IP has 4205 * not implemented yet. So we disable them on behalf. 4206 */ 4207 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4208 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4209 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4210 /* there should be no any reference. */ 4211 WARN_ON(alive_obj(obj)); 4212 } 4213 } 4214 } 4215 } 4216 4217 void amdgpu_ras_suspend(struct amdgpu_device *adev) 4218 { 4219 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4220 4221 if (!adev->ras_enabled || !con) 4222 return; 4223 4224 amdgpu_ras_disable_all_features(adev, 0); 4225 /* Make sure all ras objects are disabled. */ 4226 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4227 amdgpu_ras_disable_all_features(adev, 1); 4228 } 4229 4230 int amdgpu_ras_late_init(struct amdgpu_device *adev) 4231 { 4232 struct amdgpu_ras_block_list *node, *tmp; 4233 struct amdgpu_ras_block_object *obj; 4234 int r; 4235 4236 amdgpu_ras_event_mgr_init(adev); 4237 4238 if (amdgpu_ras_aca_is_supported(adev)) { 4239 if (amdgpu_reset_in_recovery(adev)) { 4240 if (amdgpu_aca_is_enabled(adev)) 4241 r = amdgpu_aca_reset(adev); 4242 else 4243 r = amdgpu_mca_reset(adev); 4244 if (r) 4245 return r; 4246 } 4247 4248 if (!amdgpu_sriov_vf(adev)) { 4249 if (amdgpu_aca_is_enabled(adev)) 4250 amdgpu_ras_set_aca_debug_mode(adev, false); 4251 else 4252 amdgpu_ras_set_mca_debug_mode(adev, false); 4253 } 4254 } 4255 4256 /* Guest side doesn't need init ras feature */ 4257 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4258 return 0; 4259 4260 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4261 obj = node->ras_obj; 4262 if (!obj) { 4263 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4264 continue; 4265 } 4266 4267 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4268 continue; 4269 4270 if (obj->ras_late_init) { 4271 r = obj->ras_late_init(adev, &obj->ras_comm); 4272 if (r) { 4273 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4274 obj->ras_comm.name, r); 4275 return r; 4276 } 4277 } else 4278 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4279 } 4280 4281 return 0; 4282 } 4283 4284 /* do some fini work before IP fini as dependence */ 4285 int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4286 { 4287 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4288 4289 if (!adev->ras_enabled || !con) 4290 return 0; 4291 4292 4293 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4294 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4295 amdgpu_ras_disable_all_features(adev, 0); 4296 amdgpu_ras_recovery_fini(adev); 4297 return 0; 4298 } 4299 4300 int amdgpu_ras_fini(struct amdgpu_device *adev) 4301 { 4302 struct amdgpu_ras_block_list *ras_node, *tmp; 4303 struct amdgpu_ras_block_object *obj = NULL; 4304 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4305 4306 if (!adev->ras_enabled || !con) 4307 return 0; 4308 4309 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4310 if (ras_node->ras_obj) { 4311 obj = ras_node->ras_obj; 4312 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4313 obj->ras_fini) 4314 obj->ras_fini(adev, &obj->ras_comm); 4315 else 4316 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4317 } 4318 4319 /* Clear ras blocks from ras_list and free ras block list node */ 4320 list_del(&ras_node->node); 4321 kfree(ras_node); 4322 } 4323 4324 amdgpu_ras_fs_fini(adev); 4325 amdgpu_ras_interrupt_remove_all(adev); 4326 4327 if (amdgpu_ras_aca_is_supported(adev)) { 4328 if (amdgpu_aca_is_enabled(adev)) 4329 amdgpu_aca_fini(adev); 4330 else 4331 amdgpu_mca_fini(adev); 4332 } 4333 4334 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4335 4336 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4337 amdgpu_ras_disable_all_features(adev, 0); 4338 4339 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4340 4341 amdgpu_ras_set_context(adev, NULL); 4342 kfree(con); 4343 4344 return 0; 4345 } 4346 4347 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4348 { 4349 struct amdgpu_ras *ras; 4350 4351 ras = amdgpu_ras_get_context(adev); 4352 if (!ras) 4353 return false; 4354 4355 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4356 } 4357 4358 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4359 { 4360 struct amdgpu_ras *ras; 4361 4362 ras = amdgpu_ras_get_context(adev); 4363 if (ras) { 4364 if (status) 4365 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4366 else 4367 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4368 } 4369 } 4370 4371 void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4372 { 4373 struct amdgpu_ras *ras; 4374 4375 ras = amdgpu_ras_get_context(adev); 4376 if (ras) 4377 ras->ras_err_state = 0; 4378 } 4379 4380 void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4381 enum amdgpu_ras_block block) 4382 { 4383 struct amdgpu_ras *ras; 4384 4385 ras = amdgpu_ras_get_context(adev); 4386 if (ras) 4387 set_bit(block, &ras->ras_err_state); 4388 } 4389 4390 bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4391 { 4392 struct amdgpu_ras *ras; 4393 4394 ras = amdgpu_ras_get_context(adev); 4395 if (ras) { 4396 if (block == AMDGPU_RAS_BLOCK__ANY) 4397 return (ras->ras_err_state != 0); 4398 else 4399 return test_bit(block, &ras->ras_err_state) || 4400 test_bit(AMDGPU_RAS_BLOCK__LAST, 4401 &ras->ras_err_state); 4402 } 4403 4404 return false; 4405 } 4406 4407 static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4408 { 4409 struct amdgpu_ras *ras; 4410 4411 ras = amdgpu_ras_get_context(adev); 4412 if (!ras) 4413 return NULL; 4414 4415 return ras->event_mgr; 4416 } 4417 4418 int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4419 const void *caller) 4420 { 4421 struct ras_event_manager *event_mgr; 4422 struct ras_event_state *event_state; 4423 int ret = 0; 4424 4425 if (type >= RAS_EVENT_TYPE_COUNT) { 4426 ret = -EINVAL; 4427 goto out; 4428 } 4429 4430 event_mgr = __get_ras_event_mgr(adev); 4431 if (!event_mgr) { 4432 ret = -EINVAL; 4433 goto out; 4434 } 4435 4436 event_state = &event_mgr->event_state[type]; 4437 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4438 atomic64_inc(&event_state->count); 4439 4440 out: 4441 if (ret && caller) 4442 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4443 (int)type, caller, ret); 4444 4445 return ret; 4446 } 4447 4448 u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4449 { 4450 struct ras_event_manager *event_mgr; 4451 u64 id; 4452 4453 if (type >= RAS_EVENT_TYPE_COUNT) 4454 return RAS_EVENT_INVALID_ID; 4455 4456 switch (type) { 4457 case RAS_EVENT_TYPE_FATAL: 4458 case RAS_EVENT_TYPE_POISON_CREATION: 4459 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4460 event_mgr = __get_ras_event_mgr(adev); 4461 if (!event_mgr) 4462 return RAS_EVENT_INVALID_ID; 4463 4464 id = event_mgr->event_state[type].last_seqno; 4465 break; 4466 case RAS_EVENT_TYPE_INVALID: 4467 default: 4468 id = RAS_EVENT_INVALID_ID; 4469 break; 4470 } 4471 4472 return id; 4473 } 4474 4475 void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4476 { 4477 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4478 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4479 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4480 u64 event_id; 4481 4482 if (amdgpu_ras_mark_ras_event(adev, type)) 4483 return; 4484 4485 event_id = amdgpu_ras_acquire_event_id(adev, type); 4486 4487 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4488 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4489 4490 amdgpu_ras_set_fed(adev, true); 4491 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4492 amdgpu_ras_reset_gpu(adev); 4493 } 4494 } 4495 4496 bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4497 { 4498 if (adev->asic_type == CHIP_VEGA20 && 4499 adev->pm.fw_version <= 0x283400) { 4500 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4501 amdgpu_ras_intr_triggered(); 4502 } 4503 4504 return false; 4505 } 4506 4507 void amdgpu_release_ras_context(struct amdgpu_device *adev) 4508 { 4509 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4510 4511 if (!con) 4512 return; 4513 4514 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4515 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4516 amdgpu_ras_set_context(adev, NULL); 4517 kfree(con); 4518 } 4519 } 4520 4521 #ifdef CONFIG_X86_MCE_AMD 4522 static struct amdgpu_device *find_adev(uint32_t node_id) 4523 { 4524 int i; 4525 struct amdgpu_device *adev = NULL; 4526 4527 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4528 adev = mce_adev_list.devs[i]; 4529 4530 if (adev && adev->gmc.xgmi.connected_to_cpu && 4531 adev->gmc.xgmi.physical_node_id == node_id) 4532 break; 4533 adev = NULL; 4534 } 4535 4536 return adev; 4537 } 4538 4539 #define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4540 #define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4541 #define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4542 #define GPU_ID_OFFSET 8 4543 4544 static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4545 unsigned long val, void *data) 4546 { 4547 struct mce *m = (struct mce *)data; 4548 struct amdgpu_device *adev = NULL; 4549 uint32_t gpu_id = 0; 4550 uint32_t umc_inst = 0, ch_inst = 0; 4551 4552 /* 4553 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4554 * and error occurred in DramECC (Extended error code = 0) then only 4555 * process the error, else bail out. 4556 */ 4557 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4558 (XEC(m->status, 0x3f) == 0x0))) 4559 return NOTIFY_DONE; 4560 4561 /* 4562 * If it is correctable error, return. 4563 */ 4564 if (mce_is_correctable(m)) 4565 return NOTIFY_OK; 4566 4567 /* 4568 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4569 */ 4570 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4571 4572 adev = find_adev(gpu_id); 4573 if (!adev) { 4574 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4575 gpu_id); 4576 return NOTIFY_DONE; 4577 } 4578 4579 /* 4580 * If it is uncorrectable error, then find out UMC instance and 4581 * channel index. 4582 */ 4583 umc_inst = GET_UMC_INST(m->ipid); 4584 ch_inst = GET_CHAN_INDEX(m->ipid); 4585 4586 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4587 umc_inst, ch_inst); 4588 4589 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4590 return NOTIFY_OK; 4591 else 4592 return NOTIFY_DONE; 4593 } 4594 4595 static struct notifier_block amdgpu_bad_page_nb = { 4596 .notifier_call = amdgpu_bad_page_notifier, 4597 .priority = MCE_PRIO_UC, 4598 }; 4599 4600 static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4601 { 4602 /* 4603 * Add the adev to the mce_adev_list. 4604 * During mode2 reset, amdgpu device is temporarily 4605 * removed from the mgpu_info list which can cause 4606 * page retirement to fail. 4607 * Use this list instead of mgpu_info to find the amdgpu 4608 * device on which the UMC error was reported. 4609 */ 4610 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4611 4612 /* 4613 * Register the x86 notifier only once 4614 * with MCE subsystem. 4615 */ 4616 if (notifier_registered == false) { 4617 mce_register_decode_chain(&amdgpu_bad_page_nb); 4618 notifier_registered = true; 4619 } 4620 } 4621 #endif 4622 4623 struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 4624 { 4625 if (!adev) 4626 return NULL; 4627 4628 return adev->psp.ras_context.ras; 4629 } 4630 4631 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 4632 { 4633 if (!adev) 4634 return -EINVAL; 4635 4636 adev->psp.ras_context.ras = ras_con; 4637 return 0; 4638 } 4639 4640 /* check if ras is supported on block, say, sdma, gfx */ 4641 int amdgpu_ras_is_supported(struct amdgpu_device *adev, 4642 unsigned int block) 4643 { 4644 int ret = 0; 4645 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4646 4647 if (block >= AMDGPU_RAS_BLOCK_COUNT) 4648 return 0; 4649 4650 ret = ras && (adev->ras_enabled & (1 << block)); 4651 4652 /* For the special asic with mem ecc enabled but sram ecc 4653 * not enabled, even if the ras block is not supported on 4654 * .ras_enabled, if the asic supports poison mode and the 4655 * ras block has ras configuration, it can be considered 4656 * that the ras block supports ras function. 4657 */ 4658 if (!ret && 4659 (block == AMDGPU_RAS_BLOCK__GFX || 4660 block == AMDGPU_RAS_BLOCK__SDMA || 4661 block == AMDGPU_RAS_BLOCK__VCN || 4662 block == AMDGPU_RAS_BLOCK__JPEG) && 4663 (amdgpu_ras_mask & (1 << block)) && 4664 amdgpu_ras_is_poison_mode_supported(adev) && 4665 amdgpu_ras_get_ras_block(adev, block, 0)) 4666 ret = 1; 4667 4668 return ret; 4669 } 4670 4671 int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 4672 { 4673 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4674 4675 /* mode1 is the only selection for RMA status */ 4676 if (amdgpu_ras_is_rma(adev)) { 4677 ras->gpu_reset_flags = 0; 4678 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4679 } 4680 4681 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 4682 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4683 int hive_ras_recovery = 0; 4684 4685 if (hive) { 4686 hive_ras_recovery = atomic_read(&hive->ras_recovery); 4687 amdgpu_put_xgmi_hive(hive); 4688 } 4689 /* In the case of multiple GPUs, after a GPU has started 4690 * resetting all GPUs on hive, other GPUs do not need to 4691 * trigger GPU reset again. 4692 */ 4693 if (!hive_ras_recovery) 4694 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4695 else 4696 atomic_set(&ras->in_recovery, 0); 4697 } else { 4698 flush_work(&ras->recovery_work); 4699 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 4700 } 4701 4702 return 0; 4703 } 4704 4705 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 4706 { 4707 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4708 int ret = 0; 4709 4710 if (con) { 4711 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4712 if (!ret) 4713 con->is_aca_debug_mode = enable; 4714 } 4715 4716 return ret; 4717 } 4718 4719 int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 4720 { 4721 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4722 int ret = 0; 4723 4724 if (con) { 4725 if (amdgpu_aca_is_enabled(adev)) 4726 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 4727 else 4728 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 4729 if (!ret) 4730 con->is_aca_debug_mode = enable; 4731 } 4732 4733 return ret; 4734 } 4735 4736 bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 4737 { 4738 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4739 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4740 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4741 4742 if (!con) 4743 return false; 4744 4745 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 4746 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 4747 return con->is_aca_debug_mode; 4748 else 4749 return true; 4750 } 4751 4752 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 4753 unsigned int *error_query_mode) 4754 { 4755 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4756 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 4757 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 4758 4759 if (!con) { 4760 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 4761 return false; 4762 } 4763 4764 if (amdgpu_sriov_vf(adev)) { 4765 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 4766 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 4767 *error_query_mode = 4768 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 4769 } else { 4770 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 4771 } 4772 4773 return true; 4774 } 4775 4776 /* Register each ip ras block into amdgpu ras */ 4777 int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 4778 struct amdgpu_ras_block_object *ras_block_obj) 4779 { 4780 struct amdgpu_ras_block_list *ras_node; 4781 if (!adev || !ras_block_obj) 4782 return -EINVAL; 4783 4784 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 4785 if (!ras_node) 4786 return -ENOMEM; 4787 4788 INIT_LIST_HEAD(&ras_node->node); 4789 ras_node->ras_obj = ras_block_obj; 4790 list_add_tail(&ras_node->node, &adev->ras_list); 4791 4792 return 0; 4793 } 4794 4795 void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 4796 { 4797 if (!err_type_name) 4798 return; 4799 4800 switch (err_type) { 4801 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 4802 sprintf(err_type_name, "correctable"); 4803 break; 4804 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 4805 sprintf(err_type_name, "uncorrectable"); 4806 break; 4807 default: 4808 sprintf(err_type_name, "unknown"); 4809 break; 4810 } 4811 } 4812 4813 bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 4814 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4815 uint32_t instance, 4816 uint32_t *memory_id) 4817 { 4818 uint32_t err_status_lo_data, err_status_lo_offset; 4819 4820 if (!reg_entry) 4821 return false; 4822 4823 err_status_lo_offset = 4824 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4825 reg_entry->seg_lo, reg_entry->reg_lo); 4826 err_status_lo_data = RREG32(err_status_lo_offset); 4827 4828 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 4829 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 4830 return false; 4831 4832 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 4833 4834 return true; 4835 } 4836 4837 bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 4838 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 4839 uint32_t instance, 4840 unsigned long *err_cnt) 4841 { 4842 uint32_t err_status_hi_data, err_status_hi_offset; 4843 4844 if (!reg_entry) 4845 return false; 4846 4847 err_status_hi_offset = 4848 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 4849 reg_entry->seg_hi, reg_entry->reg_hi); 4850 err_status_hi_data = RREG32(err_status_hi_offset); 4851 4852 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 4853 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 4854 /* keep the check here in case we need to refer to the result later */ 4855 dev_dbg(adev->dev, "Invalid err_info field\n"); 4856 4857 /* read err count */ 4858 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 4859 4860 return true; 4861 } 4862 4863 void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 4864 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4865 uint32_t reg_list_size, 4866 const struct amdgpu_ras_memory_id_entry *mem_list, 4867 uint32_t mem_list_size, 4868 uint32_t instance, 4869 uint32_t err_type, 4870 unsigned long *err_count) 4871 { 4872 uint32_t memory_id; 4873 unsigned long err_cnt; 4874 char err_type_name[16]; 4875 uint32_t i, j; 4876 4877 for (i = 0; i < reg_list_size; i++) { 4878 /* query memory_id from err_status_lo */ 4879 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 4880 instance, &memory_id)) 4881 continue; 4882 4883 /* query err_cnt from err_status_hi */ 4884 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 4885 instance, &err_cnt) || 4886 !err_cnt) 4887 continue; 4888 4889 *err_count += err_cnt; 4890 4891 /* log the errors */ 4892 amdgpu_ras_get_error_type_name(err_type, err_type_name); 4893 if (!mem_list) { 4894 /* memory_list is not supported */ 4895 dev_info(adev->dev, 4896 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 4897 err_cnt, err_type_name, 4898 reg_list[i].block_name, 4899 instance, memory_id); 4900 } else { 4901 for (j = 0; j < mem_list_size; j++) { 4902 if (memory_id == mem_list[j].memory_id) { 4903 dev_info(adev->dev, 4904 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 4905 err_cnt, err_type_name, 4906 reg_list[i].block_name, 4907 instance, mem_list[j].name); 4908 break; 4909 } 4910 } 4911 } 4912 } 4913 } 4914 4915 void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 4916 const struct amdgpu_ras_err_status_reg_entry *reg_list, 4917 uint32_t reg_list_size, 4918 uint32_t instance) 4919 { 4920 uint32_t err_status_lo_offset, err_status_hi_offset; 4921 uint32_t i; 4922 4923 for (i = 0; i < reg_list_size; i++) { 4924 err_status_lo_offset = 4925 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4926 reg_list[i].seg_lo, reg_list[i].reg_lo); 4927 err_status_hi_offset = 4928 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 4929 reg_list[i].seg_hi, reg_list[i].reg_hi); 4930 WREG32(err_status_lo_offset, 0); 4931 WREG32(err_status_hi_offset, 0); 4932 } 4933 } 4934 4935 int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 4936 { 4937 memset(err_data, 0, sizeof(*err_data)); 4938 4939 INIT_LIST_HEAD(&err_data->err_node_list); 4940 4941 return 0; 4942 } 4943 4944 static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 4945 { 4946 if (!err_node) 4947 return; 4948 4949 list_del(&err_node->node); 4950 kvfree(err_node); 4951 } 4952 4953 void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 4954 { 4955 struct ras_err_node *err_node, *tmp; 4956 4957 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 4958 amdgpu_ras_error_node_release(err_node); 4959 } 4960 4961 static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 4962 struct amdgpu_smuio_mcm_config_info *mcm_info) 4963 { 4964 struct ras_err_node *err_node; 4965 struct amdgpu_smuio_mcm_config_info *ref_id; 4966 4967 if (!err_data || !mcm_info) 4968 return NULL; 4969 4970 for_each_ras_error(err_node, err_data) { 4971 ref_id = &err_node->err_info.mcm_info; 4972 4973 if (mcm_info->socket_id == ref_id->socket_id && 4974 mcm_info->die_id == ref_id->die_id) 4975 return err_node; 4976 } 4977 4978 return NULL; 4979 } 4980 4981 static struct ras_err_node *amdgpu_ras_error_node_new(void) 4982 { 4983 struct ras_err_node *err_node; 4984 4985 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 4986 if (!err_node) 4987 return NULL; 4988 4989 INIT_LIST_HEAD(&err_node->node); 4990 4991 return err_node; 4992 } 4993 4994 static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 4995 { 4996 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 4997 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 4998 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 4999 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5000 5001 if (unlikely(infoa->socket_id != infob->socket_id)) 5002 return infoa->socket_id - infob->socket_id; 5003 else 5004 return infoa->die_id - infob->die_id; 5005 5006 return 0; 5007 } 5008 5009 static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5010 struct amdgpu_smuio_mcm_config_info *mcm_info) 5011 { 5012 struct ras_err_node *err_node; 5013 5014 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5015 if (err_node) 5016 return &err_node->err_info; 5017 5018 err_node = amdgpu_ras_error_node_new(); 5019 if (!err_node) 5020 return NULL; 5021 5022 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5023 5024 err_data->err_list_count++; 5025 list_add_tail(&err_node->node, &err_data->err_node_list); 5026 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5027 5028 return &err_node->err_info; 5029 } 5030 5031 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5032 struct amdgpu_smuio_mcm_config_info *mcm_info, 5033 u64 count) 5034 { 5035 struct ras_err_info *err_info; 5036 5037 if (!err_data || !mcm_info) 5038 return -EINVAL; 5039 5040 if (!count) 5041 return 0; 5042 5043 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5044 if (!err_info) 5045 return -EINVAL; 5046 5047 err_info->ue_count += count; 5048 err_data->ue_count += count; 5049 5050 return 0; 5051 } 5052 5053 int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5054 struct amdgpu_smuio_mcm_config_info *mcm_info, 5055 u64 count) 5056 { 5057 struct ras_err_info *err_info; 5058 5059 if (!err_data || !mcm_info) 5060 return -EINVAL; 5061 5062 if (!count) 5063 return 0; 5064 5065 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5066 if (!err_info) 5067 return -EINVAL; 5068 5069 err_info->ce_count += count; 5070 err_data->ce_count += count; 5071 5072 return 0; 5073 } 5074 5075 int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5076 struct amdgpu_smuio_mcm_config_info *mcm_info, 5077 u64 count) 5078 { 5079 struct ras_err_info *err_info; 5080 5081 if (!err_data || !mcm_info) 5082 return -EINVAL; 5083 5084 if (!count) 5085 return 0; 5086 5087 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5088 if (!err_info) 5089 return -EINVAL; 5090 5091 err_info->de_count += count; 5092 err_data->de_count += count; 5093 5094 return 0; 5095 } 5096 5097 #define mmMP0_SMN_C2PMSG_92 0x1609C 5098 #define mmMP0_SMN_C2PMSG_126 0x160BE 5099 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5100 u32 instance) 5101 { 5102 u32 socket_id, aid_id, hbm_id; 5103 u32 fw_status; 5104 u32 boot_error; 5105 u64 reg_addr; 5106 5107 /* The pattern for smn addressing in other SOC could be different from 5108 * the one for aqua_vanjaram. We should revisit the code if the pattern 5109 * is changed. In such case, replace the aqua_vanjaram implementation 5110 * with more common helper */ 5111 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5112 aqua_vanjaram_encode_ext_smn_addressing(instance); 5113 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5114 5115 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5116 aqua_vanjaram_encode_ext_smn_addressing(instance); 5117 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5118 5119 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5120 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5121 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5122 5123 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5124 dev_info(adev->dev, 5125 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5126 socket_id, aid_id, hbm_id, fw_status); 5127 5128 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5129 dev_info(adev->dev, 5130 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5131 socket_id, aid_id, fw_status); 5132 5133 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5134 dev_info(adev->dev, 5135 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5136 socket_id, aid_id, fw_status); 5137 5138 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5139 dev_info(adev->dev, 5140 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5141 socket_id, aid_id, fw_status); 5142 5143 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5144 dev_info(adev->dev, 5145 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5146 socket_id, aid_id, fw_status); 5147 5148 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5149 dev_info(adev->dev, 5150 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5151 socket_id, aid_id, fw_status); 5152 5153 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5154 dev_info(adev->dev, 5155 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5156 socket_id, aid_id, hbm_id, fw_status); 5157 5158 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5159 dev_info(adev->dev, 5160 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5161 socket_id, aid_id, hbm_id, fw_status); 5162 5163 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5164 dev_info(adev->dev, 5165 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5166 socket_id, aid_id, fw_status); 5167 5168 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5169 dev_info(adev->dev, 5170 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5171 socket_id, aid_id, fw_status); 5172 } 5173 5174 static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5175 u32 instance) 5176 { 5177 u64 reg_addr; 5178 u32 reg_data; 5179 int retry_loop; 5180 5181 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5182 aqua_vanjaram_encode_ext_smn_addressing(instance); 5183 5184 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5185 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5186 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5187 return false; 5188 else 5189 msleep(1); 5190 } 5191 5192 return true; 5193 } 5194 5195 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5196 { 5197 u32 i; 5198 5199 for (i = 0; i < num_instances; i++) { 5200 if (amdgpu_ras_boot_error_detected(adev, i)) 5201 amdgpu_ras_boot_time_error_reporting(adev, i); 5202 } 5203 } 5204 5205 int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5206 { 5207 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5208 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5209 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5210 int ret = 0; 5211 5212 mutex_lock(&con->page_rsv_lock); 5213 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5214 if (ret == -ENOENT) 5215 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5216 mutex_unlock(&con->page_rsv_lock); 5217 5218 return ret; 5219 } 5220 5221 void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5222 const char *fmt, ...) 5223 { 5224 struct va_format vaf; 5225 va_list args; 5226 5227 va_start(args, fmt); 5228 vaf.fmt = fmt; 5229 vaf.va = &args; 5230 5231 if (RAS_EVENT_ID_IS_VALID(event_id)) 5232 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5233 else 5234 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5235 5236 va_end(args); 5237 } 5238 5239 bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5240 { 5241 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5242 5243 if (!con) 5244 return false; 5245 5246 return con->is_rma; 5247 } 5248