1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2025 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include "ras.h" 25 #include "ras_umc.h" 26 #include "ras_umc_v12_0.h" 27 28 #define MAX_ECC_NUM_PER_RETIREMENT 16 29 30 /* bad page timestamp format 31 * yy[31:27] mm[26:23] day[22:17] hh[16:12] mm[11:6] ss[5:0] 32 */ 33 #define EEPROM_TIMESTAMP_MINUTE 6 34 #define EEPROM_TIMESTAMP_HOUR 12 35 #define EEPROM_TIMESTAMP_DAY 17 36 #define EEPROM_TIMESTAMP_MONTH 23 37 #define EEPROM_TIMESTAMP_YEAR 27 38 39 static uint64_t ras_umc_get_eeprom_timestamp(struct ras_core_context *ras_core) 40 { 41 struct ras_time tm = {0}; 42 uint64_t utc_timestamp = 0; 43 uint64_t eeprom_timestamp = 0; 44 45 utc_timestamp = ras_core_get_utc_second_timestamp(ras_core); 46 if (!utc_timestamp) 47 return utc_timestamp; 48 49 ras_core_convert_timestamp_to_time(ras_core, utc_timestamp, &tm); 50 51 /* the year range is 2000 ~ 2031, set the year if not in the range */ 52 if (tm.tm_year < 2000) 53 tm.tm_year = 2000; 54 if (tm.tm_year > 2031) 55 tm.tm_year = 2031; 56 57 tm.tm_year -= 2000; 58 59 eeprom_timestamp = tm.tm_sec + (tm.tm_min << EEPROM_TIMESTAMP_MINUTE) 60 + (tm.tm_hour << EEPROM_TIMESTAMP_HOUR) 61 + (tm.tm_mday << EEPROM_TIMESTAMP_DAY) 62 + (tm.tm_mon << EEPROM_TIMESTAMP_MONTH) 63 + (tm.tm_year << EEPROM_TIMESTAMP_YEAR); 64 eeprom_timestamp &= 0xffffffff; 65 66 return eeprom_timestamp; 67 } 68 69 static const struct ras_umc_ip_func *ras_umc_get_ip_func( 70 struct ras_core_context *ras_core, uint32_t ip_version) 71 { 72 switch (ip_version) { 73 case IP_VERSION(12, 0, 0): 74 case IP_VERSION(12, 5, 0): 75 return &ras_umc_func_v12_0; 76 default: 77 RAS_DEV_ERR(ras_core->dev, 78 "UMC ip version(0x%x) is not supported!\n", ip_version); 79 break; 80 } 81 82 return NULL; 83 } 84 85 int ras_umc_psp_ma2pa(struct ras_core_context *ras_core, 86 struct umc_mca_addr *in, struct umc_phy_addr *out, 87 uint32_t nps) 88 { 89 struct ras_ta_query_address_input addr_in; 90 struct ras_ta_query_address_output addr_out; 91 int ret; 92 93 if (!in) 94 return -EINVAL; 95 96 memset(&addr_in, 0, sizeof(addr_in)); 97 memset(&addr_out, 0, sizeof(addr_out)); 98 99 addr_in.ma.err_addr = in->err_addr; 100 addr_in.ma.ch_inst = in->ch_inst; 101 addr_in.ma.umc_inst = in->umc_inst; 102 addr_in.ma.node_inst = in->node_inst; 103 addr_in.ma.socket_id = in->socket_id; 104 105 addr_in.addr_type = RAS_TA_MCA_TO_PA; 106 107 ret = ras_psp_query_address(ras_core, &addr_in, &addr_out); 108 if (ret) { 109 RAS_DEV_WARN(ras_core->dev, 110 "Failed to query RAS physical address for 0x%llx, ret:%d", 111 in->err_addr, ret); 112 return -EREMOTEIO; 113 } 114 115 if (out) { 116 out->pa = addr_out.pa.pa; 117 out->bank = addr_out.pa.bank; 118 out->channel_idx = addr_out.pa.channel_idx; 119 } 120 121 return 0; 122 } 123 124 static int ras_umc_log_ecc(struct ras_core_context *ras_core, 125 unsigned long idx, void *data) 126 { 127 struct ras_umc *ras_umc = &ras_core->ras_umc; 128 int ret; 129 130 mutex_lock(&ras_umc->tree_lock); 131 ret = radix_tree_insert(&ras_umc->root, idx, data); 132 if (!ret) 133 radix_tree_tag_set(&ras_umc->root, idx, UMC_ECC_NEW_DETECTED_TAG); 134 mutex_unlock(&ras_umc->tree_lock); 135 136 return ret; 137 } 138 139 int ras_umc_clear_logged_ecc(struct ras_core_context *ras_core) 140 { 141 struct ras_umc *ras_umc = &ras_core->ras_umc; 142 uint64_t buf[8] = {0}; 143 void **slot; 144 void *data; 145 void *iter = buf; 146 147 mutex_lock(&ras_umc->tree_lock); 148 radix_tree_for_each_slot(slot, &ras_umc->root, iter, 0) { 149 data = ras_radix_tree_delete_iter(&ras_umc->root, iter); 150 kfree(data); 151 } 152 mutex_unlock(&ras_umc->tree_lock); 153 154 return 0; 155 } 156 157 int ras_umc_convert_record_to_nps_pages(struct ras_core_context *ras_core, 158 struct eeprom_umc_record *record, uint32_t nps, 159 uint64_t *page_pfn, uint32_t max_pages) 160 { 161 int count = 0; 162 struct ras_umc *ras_umc = &ras_core->ras_umc; 163 164 if (!page_pfn || !max_pages) 165 return -EINVAL; 166 167 if (ras_umc->ip_func && ras_umc->ip_func->eeprom_record_to_nps_pages) 168 count = ras_umc->ip_func->eeprom_record_to_nps_pages(ras_core, 169 record, nps, page_pfn, max_pages); 170 171 return count; 172 } 173 174 static void ras_umc_reserve_eeprom_record(struct ras_core_context *ras_core, 175 struct eeprom_umc_record *record) 176 { 177 uint64_t page_pfn[16]; 178 int count = 0, i; 179 180 memset(page_pfn, 0, sizeof(page_pfn)); 181 count = ras_umc_convert_record_to_nps_pages(ras_core, 182 record, record->cur_nps, page_pfn, ARRAY_SIZE(page_pfn)); 183 if (count <= 0) { 184 RAS_DEV_ERR(ras_core->dev, 185 "Fail to convert error address! count:%d\n", count); 186 return; 187 } 188 189 /* Reserve memory */ 190 for (i = 0; i < count; i++) 191 ras_core_event_notify(ras_core, 192 RAS_EVENT_ID__RESERVE_BAD_PAGE, &page_pfn[i]); 193 } 194 195 /* When gpu reset is ongoing, ecc logging operations will be pended. 196 * 197 * The pending list is bounded by RAS_UMC_PENDING_ECC_MAX so that an ECC 198 * storm or repeated UMC error injection cannot make this list (and the 199 * kernel allocations behind it) grow without bound. Once the limit is 200 * reached, additional events are dropped and counted in 201 * pending_ecc_dropped, with a rate-limited warning emitted. 202 */ 203 int ras_umc_log_bad_bank_pending(struct ras_core_context *ras_core, struct ras_bank_ecc *bank) 204 { 205 struct ras_umc *ras_umc = &ras_core->ras_umc; 206 struct ras_bank_ecc_node *ecc_node; 207 208 mutex_lock(&ras_umc->pending_ecc_lock); 209 if (ras_umc->pending_ecc_count >= RAS_UMC_PENDING_ECC_MAX) { 210 ras_umc->pending_ecc_dropped++; 211 mutex_unlock(&ras_umc->pending_ecc_lock); 212 RAS_DEV_WARN_RATELIMITED(ras_core->dev, 213 "pending ECC list full (%u), dropping bad bank event (total dropped:%u)\n", 214 RAS_UMC_PENDING_ECC_MAX, ras_umc->pending_ecc_dropped); 215 return -ENOSPC; 216 } 217 mutex_unlock(&ras_umc->pending_ecc_lock); 218 219 ecc_node = kzalloc_obj(*ecc_node); 220 if (!ecc_node) 221 return -ENOMEM; 222 223 memcpy(&ecc_node->ecc, bank, sizeof(ecc_node->ecc)); 224 225 mutex_lock(&ras_umc->pending_ecc_lock); 226 /* re-check under the lock to honor the cap across concurrent callers */ 227 if (ras_umc->pending_ecc_count >= RAS_UMC_PENDING_ECC_MAX) { 228 ras_umc->pending_ecc_dropped++; 229 mutex_unlock(&ras_umc->pending_ecc_lock); 230 kfree(ecc_node); 231 return -ENOSPC; 232 } 233 list_add_tail(&ecc_node->node, &ras_umc->pending_ecc_list); 234 ras_umc->pending_ecc_count++; 235 mutex_unlock(&ras_umc->pending_ecc_lock); 236 237 return 0; 238 } 239 240 /* After gpu reset is complete, re-log the pending error banks. 241 */ 242 int ras_umc_log_pending_bad_bank(struct ras_core_context *ras_core) 243 { 244 struct ras_umc *ras_umc = &ras_core->ras_umc; 245 struct ras_bank_ecc_node *ecc_node, *tmp; 246 247 mutex_lock(&ras_umc->pending_ecc_lock); 248 list_for_each_entry_safe(ecc_node, 249 tmp, &ras_umc->pending_ecc_list, node){ 250 if (!ras_umc_log_bad_bank(ras_core, &ecc_node->ecc)) { 251 list_del(&ecc_node->node); 252 kfree(ecc_node); 253 if (ras_umc->pending_ecc_count) 254 ras_umc->pending_ecc_count--; 255 } 256 } 257 if (ras_umc->pending_ecc_dropped) { 258 RAS_DEV_WARN(ras_core->dev, 259 "%u pending ECC bad-bank events were dropped during GPU reset\n", 260 ras_umc->pending_ecc_dropped); 261 ras_umc->pending_ecc_dropped = 0; 262 } 263 mutex_unlock(&ras_umc->pending_ecc_lock); 264 265 return 0; 266 } 267 268 int ras_umc_log_bad_bank(struct ras_core_context *ras_core, struct ras_bank_ecc *bank) 269 { 270 struct ras_umc *ras_umc = &ras_core->ras_umc; 271 struct eeprom_umc_record umc_rec; 272 struct eeprom_umc_record *err_rec; 273 int ret; 274 275 memset(&umc_rec, 0, sizeof(umc_rec)); 276 277 mutex_lock(&ras_umc->bank_log_lock); 278 ret = ras_umc->ip_func->bank_to_eeprom_record(ras_core, bank, &umc_rec); 279 if (ret) 280 goto out; 281 282 err_rec = kzalloc_obj(*err_rec); 283 if (!err_rec) { 284 ret = -ENOMEM; 285 goto out; 286 } 287 288 memcpy(err_rec, &umc_rec, sizeof(umc_rec)); 289 ret = ras_umc_log_ecc(ras_core, err_rec->cur_nps_retired_row_pfn, err_rec); 290 if (ret) { 291 if (ret == -EEXIST) { 292 RAS_DEV_INFO(ras_core->dev, "The bad pages have been logged before.\n"); 293 ret = 0; 294 } 295 296 kfree(err_rec); 297 goto out; 298 } 299 300 ras_umc_reserve_eeprom_record(ras_core, err_rec); 301 302 ret = ras_core_event_notify(ras_core, 303 RAS_EVENT_ID__BAD_PAGE_DETECTED, NULL); 304 305 out: 306 mutex_unlock(&ras_umc->bank_log_lock); 307 return ret; 308 } 309 310 int ras_umc_ma2pa(struct ras_core_context *ras_core, 311 struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out, 312 uint32_t nps) 313 { 314 struct ras_umc *ras_umc = &ras_core->ras_umc; 315 int ret; 316 317 if (ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS)) { 318 ret = ras_umc_psp_ma2pa(ras_core, addr_in, addr_out, nps); 319 } else { 320 if (ras_umc->ip_func && ras_umc->ip_func->ma2pa) { 321 ret = ras_umc->ip_func->ma2pa(ras_core, addr_in, addr_out, nps); 322 } else { 323 RAS_DEV_ERR(ras_core->dev, "ma2pa is not supported!\n"); 324 ret = -EOPNOTSUPP; 325 } 326 } 327 328 return ret; 329 } 330 331 static int ras_umc_pa2ma(struct ras_core_context *ras_core, uint64_t pa, 332 uint64_t *mca, uint32_t nps) 333 { 334 struct ras_ta_query_address_input addr_in; 335 struct ras_ta_query_address_output addr_out; 336 int ret; 337 338 if (!ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS)) 339 return -EOPNOTSUPP; 340 341 memset(&addr_in, 0, sizeof(addr_in)); 342 memset(&addr_out, 0, sizeof(addr_out)); 343 /* nps: the pa belongs to, always NPS1 for legacy eeprom data */ 344 addr_in.pa.pa = pa | ((uint64_t)nps << UMC_PA_NPS_SHIFT); 345 addr_in.addr_type = RAS_TA_PA_TO_MCA; 346 ret = ras_psp_query_address(ras_core, &addr_in, &addr_out); 347 if (ret) { 348 RAS_DEV_WARN_RATELIMITED(ras_core->dev, 349 "Failed to query RAS MCA address for 0x%llx, ret:%d\n", pa, ret); 350 351 return -EREMOTEIO; 352 } 353 354 *mca = addr_out.ma.err_addr; 355 return 0; 356 } 357 358 static int __ras_umc_eeprom_rec2nps_addr(struct ras_core_context *ras_core, 359 struct eeprom_umc_record *record, uint64_t *pa, 360 uint32_t nps, uint32_t die_id) 361 { 362 struct device_system_info dev_info = {0}; 363 struct umc_mca_addr addr_in; 364 struct umc_phy_addr addr_out; 365 struct ras_umc *ras_umc = &ras_core->ras_umc; 366 int ret; 367 368 memset(&addr_in, 0, sizeof(addr_in)); 369 memset(&addr_out, 0, sizeof(addr_out)); 370 371 ras_core_get_device_system_info(ras_core, &dev_info); 372 373 addr_in.err_addr = record->address; 374 addr_in.ch_inst = record->mem_channel; 375 addr_in.umc_inst = record->mcumc_id; 376 addr_in.node_inst = die_id; 377 addr_in.socket_id = dev_info.socket_id; 378 379 ret = ras_umc_ma2pa(ras_core, &addr_in, &addr_out, nps); 380 if (ret) 381 return ret; 382 383 if (ras_umc->ip_func && ras_umc->ip_func->nps_pa_to_row_pa) { 384 *pa = ras_umc->ip_func->nps_pa_to_row_pa(ras_core, addr_out.pa, 385 nps, false); 386 } else { 387 RAS_DEV_ERR(ras_core->dev, "nps_pa_to_row_pa is not supported!\n"); 388 return -EOPNOTSUPP; 389 } 390 391 return ret; 392 } 393 394 static int ras_umc_eeprom_rec2nps_addr(struct ras_core_context *ras_core, 395 struct eeprom_umc_record *record, uint64_t *pa, uint32_t nps) 396 { 397 return __ras_umc_eeprom_rec2nps_addr(ras_core, record, pa, nps, 398 UMC_INV_AID_NODE); 399 } 400 401 /* For legacy eeprom data format, the scope of channel index is 402 * limited to umc instance, and die id is not stored, have to 403 * get it from PA 404 */ 405 static int ras_umc_eeprom_rec2nps_addr_legacy(struct ras_core_context *ras_core, 406 struct eeprom_umc_record *record, uint64_t *pa, uint32_t nps) 407 { 408 uint32_t die_id; 409 410 /* the die id is derived from an NPS1-mode PA(legacy-format EEPROMs 411 * only ever existed on NPS1 systems) 412 */ 413 if (ras_core->ras_umc.ip_func && ras_core->ras_umc.ip_func->get_die_id) { 414 die_id = ras_core->ras_umc.ip_func->get_die_id(record->address, 415 RAS_PFN_TO_ADDR(EEPROM_RECORD_UMC_ADDR_PFN(record))); 416 } else { 417 RAS_DEV_ERR(ras_core->dev, "get_die_id is not supported!\n"); 418 return -EOPNOTSUPP; 419 } 420 421 return __ras_umc_eeprom_rec2nps_addr(ras_core, record, pa, nps, die_id); 422 } 423 424 static int ras_umc_eeprom_rec2nps_rec(struct ras_core_context *ras_core, 425 struct eeprom_umc_record *record, uint32_t nps) 426 { 427 uint64_t ch_idx_v2, pa = 0; 428 uint32_t save_nps; 429 int ret = 0; 430 431 save_nps = EEPROM_RECORD_UMC_NPS_MODE(record); 432 /* eeprom v2 has no stored nps, always convert if the flag is set */ 433 ch_idx_v2 = record->retired_row_pfn & UMC_CHANNEL_IDX_V2; 434 record->cur_nps = nps; 435 436 if (save_nps || ch_idx_v2) { 437 if ((nps == save_nps) && !ras_fw_eeprom_supported(ras_core)) { 438 record->cur_nps_retired_row_pfn = 439 EEPROM_RECORD_UMC_ADDR_PFN(record); 440 } else { 441 ret = ras_umc_eeprom_rec2nps_addr(ras_core, record, &pa, nps); 442 if (!ret) 443 record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa); 444 } 445 } else { 446 /* for specific old eeprom data, mca address is not stored(0 is 447 * default value), calc it from pa(it's nps1 in this case, other 448 * nps modes are introduced later) 449 */ 450 if (record->address == 0) { 451 ret = ras_umc_pa2ma(ras_core, 452 RAS_PFN_TO_ADDR(EEPROM_RECORD_UMC_ADDR_PFN(record)), 453 &record->address, UMC_MEMORY_PARTITION_MODE_NPS1); 454 if (ret) 455 return ret; 456 } 457 458 /* old eeprom data format, the scope of channel index is 459 * limited to umc instance 460 */ 461 ret = ras_umc_eeprom_rec2nps_addr_legacy(ras_core, record, &pa, nps); 462 if (!ret) 463 record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa); 464 } 465 466 return ret; 467 } 468 469 static int ras_umc_get_new_records(struct ras_core_context *ras_core, 470 struct eeprom_umc_record *records, u32 num) 471 { 472 struct ras_umc *ras_umc = &ras_core->ras_umc; 473 struct eeprom_umc_record *entries[MAX_ECC_NUM_PER_RETIREMENT]; 474 u32 entry_num = num < MAX_ECC_NUM_PER_RETIREMENT ? num : MAX_ECC_NUM_PER_RETIREMENT; 475 int count = 0; 476 int new_detected, i; 477 478 mutex_lock(&ras_umc->tree_lock); 479 new_detected = radix_tree_gang_lookup_tag(&ras_umc->root, (void **)entries, 480 0, entry_num, UMC_ECC_NEW_DETECTED_TAG); 481 for (i = 0; i < new_detected; i++) { 482 if (!entries[i]) 483 continue; 484 485 memcpy(&records[i], entries[i], sizeof(struct eeprom_umc_record)); 486 count++; 487 radix_tree_tag_clear(&ras_umc->root, 488 entries[i]->cur_nps_retired_row_pfn, UMC_ECC_NEW_DETECTED_TAG); 489 } 490 mutex_unlock(&ras_umc->tree_lock); 491 492 return count; 493 } 494 495 static bool ras_umc_check_retired_record(struct ras_core_context *ras_core, 496 struct eeprom_umc_record *record, bool from_eeprom) 497 { 498 struct ras_umc *ras_umc = &ras_core->ras_umc; 499 struct eeprom_store_record *data = &ras_umc->umc_err_data.rom_data; 500 uint32_t nps = 0; 501 int i, ret; 502 503 if (from_eeprom) { 504 nps = ras_umc->umc_err_data.umc_nps_mode; 505 ret = ras_umc_eeprom_rec2nps_rec(ras_core, record, nps); 506 if (ret) 507 RAS_DEV_WARN_RATELIMITED(ras_core->dev, 508 "Failed to adjust eeprom record, ret:%d", ret); 509 510 return false; 511 } 512 513 for (i = 0; i < data->count; i++) { 514 if ((data->bps[i].retired_row_pfn == record->retired_row_pfn) && 515 (data->bps[i].cur_nps_retired_row_pfn == record->cur_nps_retired_row_pfn)) 516 return true; 517 } 518 519 return false; 520 } 521 522 /* alloc/realloc bps array */ 523 static int ras_umc_realloc_err_data_space(struct ras_core_context *ras_core, 524 struct eeprom_store_record *data, int pages) 525 { 526 unsigned int old_space = data->count + data->space_left; 527 unsigned int new_space = old_space + pages; 528 unsigned int align_space = ALIGN(new_space, 512); 529 void *bps = kzalloc(align_space * sizeof(*data->bps), GFP_KERNEL); 530 531 if (!bps) 532 return -ENOMEM; 533 534 if (data->bps) { 535 memcpy(bps, data->bps, 536 data->count * sizeof(*data->bps)); 537 kfree(data->bps); 538 } 539 540 data->bps = bps; 541 data->space_left += align_space - old_space; 542 return 0; 543 } 544 545 static int ras_umc_update_eeprom_rom_data(struct ras_core_context *ras_core, 546 struct eeprom_umc_record *bps) 547 { 548 struct eeprom_store_record *data = &ras_core->ras_umc.umc_err_data.rom_data; 549 550 if (!data->space_left && 551 ras_umc_realloc_err_data_space(ras_core, data, 256)) { 552 return -ENOMEM; 553 } 554 555 memcpy(&data->bps[data->count], bps, sizeof(*data->bps)); 556 data->count++; 557 data->space_left--; 558 return 0; 559 } 560 561 static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core, 562 struct eeprom_umc_record *bps) 563 { 564 struct ras_umc *ras_umc = &ras_core->ras_umc; 565 struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data; 566 uint64_t page_pfn[16]; 567 int count = 0, i, j; 568 569 if (!data->space_left && 570 ras_umc_realloc_err_data_space(ras_core, data, 256)) { 571 return -ENOMEM; 572 } 573 574 memset(page_pfn, 0, sizeof(page_pfn)); 575 count = ras_umc_convert_record_to_nps_pages(ras_core, 576 bps, bps->cur_nps, page_pfn, ARRAY_SIZE(page_pfn)); 577 if (count > 0) { 578 for (j = 0; j < count; j++) { 579 if (ras_core_check_address_sanity(ras_core, 580 page_pfn[j] << AMDGPU_GPU_PAGE_SHIFT)) { 581 582 for (i = 0; i < data->count; i++) 583 if (page_pfn[j] == data->bps[i].cur_nps_retired_row_pfn) 584 break; 585 data->bps[data->count].cur_nps_retired_row_pfn = U64_MAX; 586 data->count++; 587 data->space_left--; 588 continue; 589 } 590 591 bps->cur_nps_retired_row_pfn = page_pfn[j]; 592 memcpy(&data->bps[data->count], bps, sizeof(*data->bps)); 593 data->count++; 594 data->space_left--; 595 data->bad_page_num++; 596 } 597 } else { 598 RAS_DEV_ERR(ras_core->dev, "Failed to convert record to nps pages!"); 599 return -EINVAL; 600 } 601 602 return 0; 603 } 604 605 static void ras_umc_update_bad_pages(struct ras_core_context *ras_core) 606 { 607 struct ras_umc *ras_umc = &ras_core->ras_umc; 608 struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data; 609 610 data->bad_page_num_old = data->bad_page_num; 611 } 612 613 /* it deal with vram only. */ 614 static int ras_umc_add_bad_pages(struct ras_core_context *ras_core, 615 struct eeprom_umc_record *bps, 616 int pages, bool from_eeprom) 617 { 618 struct ras_umc *ras_umc = &ras_core->ras_umc; 619 struct ras_umc_err_data *data = &ras_umc->umc_err_data; 620 int i, ret = 0; 621 622 if (!bps || pages <= 0) 623 return 0; 624 625 mutex_lock(&ras_umc->umc_lock); 626 for (i = 0; i < pages; i++) { 627 if (ras_umc_check_retired_record(ras_core, &bps[i], from_eeprom)) 628 continue; 629 630 ret = ras_umc_update_eeprom_rom_data(ras_core, &bps[i]); 631 if (ret) 632 goto out; 633 634 if (data->last_retired_pfn == bps[i].cur_nps_retired_row_pfn) 635 continue; 636 637 data->last_retired_pfn = bps[i].cur_nps_retired_row_pfn; 638 639 if (from_eeprom) 640 ras_umc_reserve_eeprom_record(ras_core, &bps[i]); 641 642 ret = ras_umc_update_eeprom_ram_data(ras_core, &bps[i]); 643 if (ret) 644 goto out; 645 } 646 out: 647 mutex_unlock(&ras_umc->umc_lock); 648 649 return ret; 650 } 651 652 /* 653 * read error record array in eeprom and reserve enough space for 654 * storing new bad pages 655 */ 656 int ras_umc_load_bad_pages(struct ras_core_context *ras_core) 657 { 658 struct eeprom_umc_record *bps; 659 uint32_t ras_num_recs; 660 int ret; 661 662 if (ras_fw_eeprom_supported(ras_core)) { 663 ras_num_recs = ras_fw_eeprom_get_record_count(ras_core); 664 /* no bad page record, skip eeprom access */ 665 if (!ras_num_recs || 666 ras_core->ras_fw_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE) 667 return 0; 668 } else { 669 ras_num_recs = ras_eeprom_get_record_count(ras_core); 670 if (!ras_num_recs || 671 ras_core->ras_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE) 672 return 0; 673 } 674 675 bps = kzalloc_objs(*bps, ras_num_recs); 676 if (!bps) 677 return -ENOMEM; 678 679 if (ras_fw_eeprom_supported(ras_core)) 680 ret = ras_fw_eeprom_read_idx(ras_core, bps, 0, 0, ras_num_recs); 681 else 682 ret = ras_eeprom_read(ras_core, bps, ras_num_recs); 683 if (ret) { 684 RAS_DEV_ERR(ras_core->dev, "Failed to load EEPROM table records!"); 685 } else { 686 ras_core->ras_umc.umc_err_data.last_retired_pfn = UMC_INV_MEM_PFN; 687 ret = ras_umc_add_bad_pages(ras_core, bps, ras_num_recs, true); 688 ras_umc_update_bad_pages(ras_core); 689 } 690 691 kfree(bps); 692 return ret; 693 } 694 695 /* 696 * write error record array to eeprom, the function should be 697 * protected by recovery_lock 698 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 699 */ 700 static int ras_umc_save_bad_pages(struct ras_core_context *ras_core) 701 { 702 struct ras_umc *ras_umc = &ras_core->ras_umc; 703 struct eeprom_store_record *data = &ras_umc->umc_err_data.rom_data; 704 struct eeprom_store_record *ram_data = &ras_umc->umc_err_data.ram_data; 705 uint32_t eeprom_record_num, logical_count = 0; 706 int save_count; 707 int ret = 0; 708 709 if (!data->bps) 710 return 0; 711 712 if (ras_fw_eeprom_supported(ras_core)) 713 eeprom_record_num = ras_fw_eeprom_get_record_count(ras_core); 714 else 715 eeprom_record_num = ras_eeprom_get_record_count(ras_core); 716 mutex_lock(&ras_umc->umc_lock); 717 save_count = data->count - eeprom_record_num; 718 logical_count = ram_data->bad_page_num - ram_data->bad_page_num_old; 719 /* only new entries are saved */ 720 if (save_count > 0) { 721 if (ras_fw_eeprom_supported(ras_core)) 722 ret = ras_fw_eeprom_append(ras_core, &data->bps[eeprom_record_num], 723 save_count); 724 else 725 ret = ras_eeprom_append(ras_core, &data->bps[eeprom_record_num], 726 save_count); 727 if (ret) { 728 RAS_DEV_ERR(ras_core->dev, "Failed to save EEPROM table data!"); 729 ret = -EIO; 730 goto exit; 731 } 732 ras_umc_update_bad_pages(ras_core); 733 RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n", logical_count); 734 } 735 736 exit: 737 mutex_unlock(&ras_umc->umc_lock); 738 return ret; 739 } 740 741 int ras_umc_handle_bad_pages(struct ras_core_context *ras_core, void *data) 742 { 743 struct eeprom_umc_record *records; 744 int count, ret; 745 746 records = kzalloc_objs(*records, MAX_ECC_NUM_PER_RETIREMENT); 747 if (!records) 748 return -ENOMEM; 749 750 count = ras_umc_get_new_records(ras_core, records, 751 MAX_ECC_NUM_PER_RETIREMENT); 752 if (count <= 0) { 753 ret = -ENODATA; 754 goto out; 755 } 756 757 ret = ras_umc_add_bad_pages(ras_core, records, count, false); 758 if (ret) { 759 RAS_DEV_ERR(ras_core->dev, "Failed to add ras bad page!\n"); 760 ret = -EINVAL; 761 goto out; 762 } 763 764 ret = ras_umc_save_bad_pages(ras_core); 765 if (ret) { 766 RAS_DEV_ERR(ras_core->dev, "Failed to save ras bad page\n"); 767 ret = -EINVAL; 768 goto out; 769 } 770 771 ret = 0; 772 773 out: 774 kfree(records); 775 return ret; 776 } 777 778 int ras_umc_sw_init(struct ras_core_context *ras_core) 779 { 780 struct ras_umc *ras_umc = &ras_core->ras_umc; 781 782 memset(ras_umc, 0, sizeof(*ras_umc)); 783 784 INIT_LIST_HEAD(&ras_umc->pending_ecc_list); 785 786 INIT_RADIX_TREE(&ras_umc->root, GFP_KERNEL); 787 788 mutex_init(&ras_umc->tree_lock); 789 mutex_init(&ras_umc->pending_ecc_lock); 790 mutex_init(&ras_umc->umc_lock); 791 mutex_init(&ras_umc->bank_log_lock); 792 793 return 0; 794 } 795 796 int ras_umc_sw_fini(struct ras_core_context *ras_core) 797 { 798 struct ras_umc *ras_umc = &ras_core->ras_umc; 799 struct ras_umc_err_data *umc_err_data = &ras_umc->umc_err_data; 800 struct ras_bank_ecc_node *ecc_node, *tmp; 801 802 mutex_destroy(&ras_umc->umc_lock); 803 mutex_destroy(&ras_umc->bank_log_lock); 804 805 if (umc_err_data->rom_data.bps) { 806 umc_err_data->rom_data.count = 0; 807 kfree(umc_err_data->rom_data.bps); 808 umc_err_data->rom_data.bps = NULL; 809 umc_err_data->rom_data.space_left = 0; 810 } 811 812 if (umc_err_data->ram_data.bps) { 813 umc_err_data->ram_data.count = 0; 814 kfree(umc_err_data->ram_data.bps); 815 umc_err_data->ram_data.bps = NULL; 816 umc_err_data->ram_data.space_left = 0; 817 } 818 819 ras_umc_clear_logged_ecc(ras_core); 820 821 mutex_lock(&ras_umc->pending_ecc_lock); 822 list_for_each_entry_safe(ecc_node, 823 tmp, &ras_umc->pending_ecc_list, node){ 824 list_del(&ecc_node->node); 825 kfree(ecc_node); 826 } 827 ras_umc->pending_ecc_count = 0; 828 ras_umc->pending_ecc_dropped = 0; 829 mutex_unlock(&ras_umc->pending_ecc_lock); 830 831 mutex_destroy(&ras_umc->tree_lock); 832 mutex_destroy(&ras_umc->pending_ecc_lock); 833 834 return 0; 835 } 836 837 int ras_umc_hw_init(struct ras_core_context *ras_core) 838 { 839 struct ras_umc *ras_umc = &ras_core->ras_umc; 840 uint32_t nps; 841 842 nps = ras_core_get_curr_nps_mode(ras_core); 843 844 if (!nps || (nps >= UMC_MEMORY_PARTITION_MODE_UNKNOWN)) { 845 RAS_DEV_ERR(ras_core->dev, "Invalid memory NPS mode: %u!\n", nps); 846 return -ENODATA; 847 } 848 849 ras_umc->umc_err_data.umc_nps_mode = nps; 850 851 ras_umc->umc_vram_type = ras_core->config->umc_cfg.umc_vram_type; 852 ras_umc->num_umc = ras_core->config->umc_cfg.num_umc; 853 if (!ras_umc->umc_vram_type) { 854 RAS_DEV_ERR(ras_core->dev, "Invalid UMC VRAM Type: %u!\n", 855 ras_umc->umc_vram_type); 856 return -ENODATA; 857 } 858 859 ras_umc->umc_ip_version = ras_core->config->umc_ip_version; 860 ras_umc->ip_func = ras_umc_get_ip_func(ras_core, ras_umc->umc_ip_version); 861 if (!ras_umc->ip_func) 862 return -EINVAL; 863 864 return 0; 865 } 866 867 int ras_umc_hw_fini(struct ras_core_context *ras_core) 868 { 869 return 0; 870 } 871 872 int ras_umc_clean_badpage_data(struct ras_core_context *ras_core) 873 { 874 struct ras_umc_err_data *data = &ras_core->ras_umc.umc_err_data; 875 876 mutex_lock(&ras_core->ras_umc.umc_lock); 877 878 kfree(data->rom_data.bps); 879 kfree(data->ram_data.bps); 880 881 memset(data, 0, sizeof(*data)); 882 mutex_unlock(&ras_core->ras_umc.umc_lock); 883 884 return 0; 885 } 886 887 int ras_umc_fill_eeprom_record(struct ras_core_context *ras_core, 888 uint64_t err_addr, uint32_t umc_inst, struct umc_phy_addr *cur_nps_addr, 889 enum umc_memory_partition_mode cur_nps, struct eeprom_umc_record *record) 890 { 891 struct eeprom_umc_record *err_rec = record; 892 893 /* Set bad page pfn and nps mode */ 894 EEPROM_RECORD_SETUP_UMC_ADDR_AND_NPS(err_rec, 895 RAS_ADDR_TO_PFN(cur_nps_addr->pa), cur_nps); 896 897 err_rec->address = err_addr; 898 err_rec->ts = ras_umc_get_eeprom_timestamp(ras_core); 899 err_rec->err_type = RAS_EEPROM_ERR_NON_RECOVERABLE; 900 err_rec->cu = 0; 901 err_rec->mem_channel = cur_nps_addr->channel_idx; 902 err_rec->mcumc_id = umc_inst; 903 err_rec->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(cur_nps_addr->pa); 904 err_rec->cur_nps_bank = cur_nps_addr->bank; 905 err_rec->cur_nps = cur_nps; 906 return 0; 907 } 908 909 int ras_umc_get_saved_eeprom_count(struct ras_core_context *ras_core) 910 { 911 struct ras_umc_err_data *err_data = &ras_core->ras_umc.umc_err_data; 912 913 return err_data->rom_data.count; 914 } 915 916 int ras_umc_get_badpage_count(struct ras_core_context *ras_core) 917 { 918 struct eeprom_store_record *data = &ras_core->ras_umc.umc_err_data.ram_data; 919 920 return data->count; 921 } 922 923 int ras_umc_get_badpage_record(struct ras_core_context *ras_core, uint32_t index, void *record) 924 { 925 struct eeprom_store_record *data = &ras_core->ras_umc.umc_err_data.ram_data; 926 927 if (index >= data->count) 928 return -EINVAL; 929 930 memcpy(record, &data->bps[index], sizeof(struct eeprom_umc_record)); 931 return 0; 932 } 933 934 bool ras_umc_check_retired_addr(struct ras_core_context *ras_core, uint64_t addr) 935 { 936 struct ras_umc *ras_umc = &ras_core->ras_umc; 937 struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data; 938 uint64_t page_pfn = RAS_ADDR_TO_PFN(addr); 939 int i, ret = false; 940 941 mutex_lock(&ras_umc->umc_lock); 942 for (i = 0; i < data->count; i++) { 943 if (data->bps[i].cur_nps_retired_row_pfn == page_pfn) { 944 ret = true; 945 break; 946 } 947 } 948 mutex_unlock(&ras_umc->umc_lock); 949 950 return ret; 951 } 952 953 int ras_umc_translate_soc_pa_and_bank(struct ras_core_context *ras_core, 954 uint64_t *soc_pa, struct umc_bank_addr *bank_addr, bool bank_to_pa) 955 { 956 struct ras_umc *ras_umc = &ras_core->ras_umc; 957 int ret = 0; 958 959 if (bank_to_pa) 960 ret = ras_umc->ip_func->bank_to_soc_pa(ras_core, *bank_addr, soc_pa); 961 else 962 ret = ras_umc->ip_func->soc_pa_to_bank(ras_core, *soc_pa, bank_addr); 963 964 return ret; 965 } 966 967 uint32_t ras_umc_bit_wise_xor(uint32_t val) 968 { 969 uint32_t result = 0; 970 int i; 971 972 for (i = 0; i < 32; i++) 973 result = result ^ ((val >> i) & 0x1); 974 975 return result; 976 } 977