1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/bitmap.h> 25 #include <linux/sort.h> 26 #include "amdgpu.h" 27 #include "umc_v6_7.h" 28 #include "amdgpu_ras_mgr.h" 29 #define MAX_UMC_POISON_POLLING_TIME_SYNC 20 //ms 30 31 #define MAX_UMC_HASH_STRING_SIZE 256 32 33 static int amdgpu_umc_convert_error_address(struct amdgpu_device *adev, 34 struct ras_err_data *err_data, uint64_t err_addr, 35 uint32_t ch_inst, uint32_t umc_inst) 36 { 37 switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) { 38 case IP_VERSION(6, 7, 0): 39 umc_v6_7_convert_error_address(adev, 40 err_data, err_addr, ch_inst, umc_inst); 41 break; 42 default: 43 dev_warn(adev->dev, 44 "UMC address to Physical address translation is not supported\n"); 45 return AMDGPU_RAS_FAIL; 46 } 47 48 return AMDGPU_RAS_SUCCESS; 49 } 50 51 int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev, 52 uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst) 53 { 54 struct ras_err_data err_data; 55 int ret; 56 57 ret = amdgpu_ras_error_data_init(&err_data); 58 if (ret) 59 return ret; 60 61 err_data.err_addr = 62 kzalloc_objs(struct eeprom_table_record, 63 adev->umc.max_ras_err_cnt_per_query); 64 if (!err_data.err_addr) { 65 dev_warn(adev->dev, 66 "Failed to alloc memory for umc error record in MCA notifier!\n"); 67 ret = AMDGPU_RAS_FAIL; 68 goto out_fini_err_data; 69 } 70 71 err_data.err_addr_len = adev->umc.max_ras_err_cnt_per_query; 72 73 /* 74 * Translate UMC channel address to Physical address 75 */ 76 ret = amdgpu_umc_convert_error_address(adev, &err_data, err_addr, 77 ch_inst, umc_inst); 78 if (ret) 79 goto out_free_err_addr; 80 81 if (amdgpu_bad_page_threshold != 0) { 82 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 83 err_data.err_addr_cnt, false); 84 amdgpu_ras_save_bad_pages(adev, NULL); 85 } 86 87 out_free_err_addr: 88 kfree(err_data.err_addr); 89 90 out_fini_err_data: 91 amdgpu_ras_error_data_fini(&err_data); 92 93 return ret; 94 } 95 96 void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, 97 void *ras_error_status) 98 { 99 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 100 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 101 unsigned int error_query_mode; 102 int ret = 0; 103 unsigned long err_count; 104 105 amdgpu_ras_get_error_query_mode(adev, &error_query_mode); 106 107 err_data->err_addr = 108 kzalloc_objs(struct eeprom_table_record, 109 adev->umc.max_ras_err_cnt_per_query); 110 111 /* still call query_ras_error_address to clear error status 112 * even NOMEM error is encountered 113 */ 114 if (!err_data->err_addr) 115 dev_warn(adev->dev, 116 "Failed to alloc memory for umc error address record!\n"); 117 else 118 err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query; 119 120 mutex_lock(&con->page_retirement_lock); 121 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc)); 122 if (ret == -EOPNOTSUPP && 123 error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 124 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 125 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 126 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, 127 ras_error_status); 128 129 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 130 adev->umc.ras->ras_block.hw_ops->query_ras_error_address && 131 adev->umc.max_ras_err_cnt_per_query) { 132 err_data->err_addr = 133 kzalloc_objs(struct eeprom_table_record, 134 adev->umc.max_ras_err_cnt_per_query); 135 136 /* still call query_ras_error_address to clear error status 137 * even NOMEM error is encountered 138 */ 139 if (!err_data->err_addr) 140 dev_warn(adev->dev, 141 "Failed to alloc memory for umc error address record!\n"); 142 else 143 err_data->err_addr_len = 144 adev->umc.max_ras_err_cnt_per_query; 145 146 /* umc query_ras_error_address is also responsible for clearing 147 * error status 148 */ 149 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, 150 ras_error_status); 151 } 152 } else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY || 153 (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) { 154 if (adev->umc.ras && 155 adev->umc.ras->ecc_info_query_ras_error_count) 156 adev->umc.ras->ecc_info_query_ras_error_count(adev, 157 ras_error_status); 158 159 if (adev->umc.ras && 160 adev->umc.ras->ecc_info_query_ras_error_address && 161 adev->umc.max_ras_err_cnt_per_query) { 162 err_data->err_addr = 163 kzalloc_objs(struct eeprom_table_record, 164 adev->umc.max_ras_err_cnt_per_query); 165 166 /* still call query_ras_error_address to clear error status 167 * even NOMEM error is encountered 168 */ 169 if (!err_data->err_addr) 170 dev_warn(adev->dev, 171 "Failed to alloc memory for umc error address record!\n"); 172 else 173 err_data->err_addr_len = 174 adev->umc.max_ras_err_cnt_per_query; 175 176 /* umc query_ras_error_address is also responsible for clearing 177 * error status 178 */ 179 adev->umc.ras->ecc_info_query_ras_error_address(adev, 180 ras_error_status); 181 } 182 } 183 184 /* only uncorrectable error needs gpu reset */ 185 if (err_data->ue_count || err_data->de_count) { 186 err_count = err_data->ue_count + err_data->de_count; 187 if ((amdgpu_bad_page_threshold != 0) && 188 err_data->err_addr_cnt) { 189 amdgpu_ras_add_bad_pages(adev, err_data->err_addr, 190 err_data->err_addr_cnt, false); 191 amdgpu_ras_save_bad_pages(adev, &err_count); 192 193 amdgpu_dpm_send_hbm_bad_pages_num(adev, 194 con->eeprom_control.ras_num_bad_pages); 195 196 if (con->update_channel_flag == true) { 197 amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap); 198 con->update_channel_flag = false; 199 } 200 } 201 } 202 203 kfree(err_data->err_addr); 204 err_data->err_addr = NULL; 205 206 mutex_unlock(&con->page_retirement_lock); 207 } 208 209 static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev, 210 void *ras_error_status, 211 struct amdgpu_iv_entry *entry, 212 uint32_t reset) 213 { 214 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 215 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 216 217 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 218 amdgpu_umc_handle_bad_pages(adev, ras_error_status); 219 220 if ((err_data->ue_count || err_data->de_count) && 221 (reset || amdgpu_ras_is_rma(adev))) { 222 con->gpu_reset_flags |= reset; 223 amdgpu_ras_reset_gpu(adev); 224 } 225 226 return AMDGPU_RAS_SUCCESS; 227 } 228 229 int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev, 230 enum amdgpu_ras_block block, uint16_t pasid, 231 pasid_notify pasid_fn, void *data, uint32_t reset) 232 { 233 int ret = AMDGPU_RAS_SUCCESS; 234 struct ras_ih_info ih_info = { 235 .block = block, 236 .pasid = pasid, 237 .reset = reset, 238 .pasid_fn = pasid_fn, 239 .data = data, 240 }; 241 242 if (adev->gmc.xgmi.connected_to_cpu || 243 adev->gmc.is_app_apu) { 244 if (reset) { 245 /* MCA poison handler is only responsible for GPU reset, 246 * let MCA notifier do page retirement. 247 */ 248 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 249 amdgpu_ras_reset_gpu(adev); 250 } 251 return ret; 252 } 253 254 if (!amdgpu_sriov_vf(adev)) { 255 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) { 256 struct ras_err_data err_data; 257 struct ras_common_if head = { 258 .block = AMDGPU_RAS_BLOCK__UMC, 259 }; 260 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head); 261 262 ret = amdgpu_ras_error_data_init(&err_data); 263 if (ret) 264 return ret; 265 266 ret = amdgpu_umc_do_page_retirement(adev, &err_data, NULL, reset); 267 268 if (ret == AMDGPU_RAS_SUCCESS && obj) { 269 obj->err_data.ue_count += err_data.ue_count; 270 obj->err_data.ce_count += err_data.ce_count; 271 } 272 273 amdgpu_ras_error_data_fini(&err_data); 274 } else { 275 amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info); 276 } 277 } else { 278 if (amdgpu_uniras_enabled(adev) && 279 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) 280 amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info); 281 else if (adev->virt.ops && adev->virt.ops->ras_poison_handler) 282 adev->virt.ops->ras_poison_handler(adev, block); 283 else 284 dev_warn(adev->dev, 285 "No ras_poison_handler interface in SRIOV!\n"); 286 } 287 288 return ret; 289 } 290 291 int amdgpu_umc_poison_handler(struct amdgpu_device *adev, 292 enum amdgpu_ras_block block, uint32_t reset) 293 { 294 return amdgpu_umc_pasid_poison_handler(adev, 295 block, 0, NULL, NULL, reset); 296 } 297 298 int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev, 299 void *ras_error_status, 300 struct amdgpu_iv_entry *entry) 301 { 302 return amdgpu_umc_do_page_retirement(adev, ras_error_status, entry, 303 AMDGPU_RAS_GPU_RESET_MODE1_RESET); 304 } 305 306 int amdgpu_umc_ras_sw_init(struct amdgpu_device *adev) 307 { 308 int err; 309 struct amdgpu_umc_ras *ras; 310 311 if (!adev->umc.ras) 312 return 0; 313 314 ras = adev->umc.ras; 315 316 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); 317 if (err) { 318 dev_err(adev->dev, "Failed to register umc ras block!\n"); 319 return err; 320 } 321 322 strcpy(adev->umc.ras->ras_block.ras_comm.name, "umc"); 323 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC; 324 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 325 adev->umc.ras_if = &ras->ras_block.ras_comm; 326 327 if (!ras->ras_block.ras_late_init) 328 ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init; 329 330 if (!ras->ras_block.ras_cb) 331 ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb; 332 333 return 0; 334 } 335 336 int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 337 { 338 int r; 339 340 r = amdgpu_ras_block_late_init(adev, ras_block); 341 if (r) 342 return r; 343 344 if (amdgpu_sriov_vf(adev)) 345 return r; 346 347 if (amdgpu_ras_is_supported(adev, ras_block->block)) { 348 r = amdgpu_irq_get(adev, &adev->gmc.ecc_irq, 0); 349 if (r) 350 goto late_fini; 351 } 352 353 /* ras init of specific umc version */ 354 if (adev->umc.ras && 355 adev->umc.ras->err_cnt_init) 356 adev->umc.ras->err_cnt_init(adev); 357 358 return 0; 359 360 late_fini: 361 amdgpu_ras_block_late_fini(adev, ras_block); 362 return r; 363 } 364 365 int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev, 366 struct amdgpu_irq_src *source, 367 struct amdgpu_iv_entry *entry) 368 { 369 struct ras_common_if *ras_if = adev->umc.ras_if; 370 struct ras_dispatch_if ih_data = { 371 .entry = entry, 372 }; 373 374 if (!ras_if) 375 return 0; 376 377 ih_data.head = *ras_if; 378 379 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 380 return 0; 381 } 382 383 int amdgpu_umc_uniras_process_ecc_irq(struct amdgpu_device *adev, 384 struct amdgpu_irq_src *source, 385 struct amdgpu_iv_entry *entry) 386 { 387 struct ras_ih_info ih_info = {0}; 388 389 ih_info.block = RAS_BLOCK_ID__UMC; 390 amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info); 391 return 0; 392 } 393 394 int amdgpu_umc_fill_error_record(struct ras_err_data *err_data, 395 uint64_t err_addr, 396 uint64_t retired_page, 397 uint32_t channel_index, 398 uint32_t umc_inst) 399 { 400 struct eeprom_table_record *err_rec; 401 402 if (!err_data || 403 !err_data->err_addr || 404 (err_data->err_addr_cnt >= err_data->err_addr_len)) 405 return -EINVAL; 406 407 err_rec = &err_data->err_addr[err_data->err_addr_cnt]; 408 409 err_rec->address = err_addr; 410 /* page frame address is saved */ 411 err_rec->retired_page = retired_page >> AMDGPU_GPU_PAGE_SHIFT; 412 err_rec->ts = (uint64_t)ktime_get_real_seconds(); 413 err_rec->err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE; 414 err_rec->cu = 0; 415 err_rec->mem_channel = channel_index; 416 err_rec->mcumc_id = umc_inst; 417 418 err_data->err_addr_cnt++; 419 420 return 0; 421 } 422 423 static int amdgpu_umc_loop_all_aid(struct amdgpu_device *adev, umc_func func, 424 void *data) 425 { 426 uint32_t umc_node_inst; 427 uint32_t node_inst; 428 uint32_t umc_inst; 429 uint32_t ch_inst; 430 DECLARE_BITMAP(umc_bitmap, 64); 431 int ret; 432 433 /* 434 * This loop is done based on the following - 435 * umc.active mask = mask of active umc instances across all nodes 436 * umc.umc_inst_num = maximum number of umc instancess per node 437 * umc.node_inst_num = maximum number of node instances 438 * Channel instances are not assumed to be harvested. 439 */ 440 dev_dbg(adev->dev, "active umcs :%llx umc_inst per node: %d", 441 adev->umc.active_mask, adev->umc.umc_inst_num); 442 bitmap_from_u64(umc_bitmap, adev->umc.active_mask); 443 for_each_set_bit(umc_node_inst, umc_bitmap, 444 adev->umc.node_inst_num * adev->umc.umc_inst_num) { 445 node_inst = umc_node_inst / adev->umc.umc_inst_num; 446 umc_inst = umc_node_inst % adev->umc.umc_inst_num; 447 LOOP_UMC_CH_INST(ch_inst) { 448 dev_dbg(adev->dev, 449 "node_inst :%d umc_inst: %d ch_inst: %d", 450 node_inst, umc_inst, ch_inst); 451 ret = func(adev, node_inst, umc_inst, ch_inst, data); 452 if (ret) { 453 dev_err(adev->dev, 454 "Node %d umc %d ch %d func returns %d\n", 455 node_inst, umc_inst, ch_inst, ret); 456 return ret; 457 } 458 } 459 } 460 461 return 0; 462 } 463 464 int amdgpu_umc_loop_channels(struct amdgpu_device *adev, 465 umc_func func, void *data) 466 { 467 uint32_t node_inst = 0; 468 uint32_t umc_inst = 0; 469 uint32_t ch_inst = 0; 470 int ret = 0; 471 472 if (adev->aid_mask) 473 return amdgpu_umc_loop_all_aid(adev, func, data); 474 475 if (adev->umc.node_inst_num) { 476 DECLARE_BITMAP(umc_bitmap, 64); 477 478 bitmap_from_u64(umc_bitmap, adev->umc.active_mask); 479 for_each_set_bit(node_inst, umc_bitmap, adev->umc.node_inst_num) { 480 LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { 481 ret = func(adev, node_inst, umc_inst, ch_inst, data); 482 if (ret) { 483 dev_err(adev->dev, 484 "Node %d umc %d ch %d func returns %d\n", 485 node_inst, umc_inst, ch_inst, ret); 486 return ret; 487 } 488 } 489 } 490 } else { 491 LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) { 492 ret = func(adev, 0, umc_inst, ch_inst, data); 493 if (ret) { 494 dev_err(adev->dev, "Umc %d ch %d func returns %d\n", 495 umc_inst, ch_inst, ret); 496 return ret; 497 } 498 } 499 } 500 501 return 0; 502 } 503