xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c (revision 466423c6dd8af23ebb3a69d43434d01aed0db356) 
1 /*
2  * Copyright 2021 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include "amdgpu_ras.h"
24 #include "amdgpu.h"
25 #include "amdgpu_mca.h"
26 
27 #include "umc/umc_6_7_0_offset.h"
28 #include "umc/umc_6_7_0_sh_mask.h"
29 
30 static bool amdgpu_mca_is_deferred_error(struct amdgpu_device *adev,
31 					uint64_t mc_status)
32 {
33 	if (adev->umc.ras->check_ecc_err_status)
34 		return adev->umc.ras->check_ecc_err_status(adev,
35 				AMDGPU_MCA_ERROR_TYPE_DE, &mc_status);
36 
37 	return false;
38 }
39 
40 void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev,
41 					      uint64_t mc_status_addr,
42 					      unsigned long *error_count)
43 {
44 	uint64_t mc_status = RREG64_PCIE(mc_status_addr);
45 
46 	if (REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
47 	    REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)
48 		*error_count += 1;
49 }
50 
51 void amdgpu_mca_query_uncorrectable_error_count(struct amdgpu_device *adev,
52 						uint64_t mc_status_addr,
53 						unsigned long *error_count)
54 {
55 	uint64_t mc_status = RREG64_PCIE(mc_status_addr);
56 
57 	if ((REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
58 	    (REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1 ||
59 	    REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
60 	    REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 ||
61 	    REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 ||
62 	    REG_GET_FIELD(mc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC) == 1))
63 		*error_count += 1;
64 }
65 
66 void amdgpu_mca_reset_error_count(struct amdgpu_device *adev,
67 				  uint64_t mc_status_addr)
68 {
69 	WREG64_PCIE(mc_status_addr, 0x0ULL);
70 }
71 
72 void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev,
73 				      uint64_t mc_status_addr,
74 				      void *ras_error_status)
75 {
76 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
77 
78 	amdgpu_mca_query_correctable_error_count(adev, mc_status_addr, &(err_data->ce_count));
79 	amdgpu_mca_query_uncorrectable_error_count(adev, mc_status_addr, &(err_data->ue_count));
80 
81 	amdgpu_mca_reset_error_count(adev, mc_status_addr);
82 }
83 
84 int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev)
85 {
86 	int err;
87 	struct amdgpu_mca_ras_block *ras;
88 
89 	if (!adev->mca.mp0.ras)
90 		return 0;
91 
92 	ras = adev->mca.mp0.ras;
93 
94 	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
95 	if (err) {
96 		dev_err(adev->dev, "Failed to register mca.mp0 ras block!\n");
97 		return err;
98 	}
99 
100 	strcpy(ras->ras_block.ras_comm.name, "mca.mp0");
101 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
102 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
103 	adev->mca.mp0.ras_if = &ras->ras_block.ras_comm;
104 
105 	return 0;
106 }
107 
108 int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev)
109 {
110 	int err;
111 	struct amdgpu_mca_ras_block *ras;
112 
113 	if (!adev->mca.mp1.ras)
114 		return 0;
115 
116 	ras = adev->mca.mp1.ras;
117 
118 	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
119 	if (err) {
120 		dev_err(adev->dev, "Failed to register mca.mp1 ras block!\n");
121 		return err;
122 	}
123 
124 	strcpy(ras->ras_block.ras_comm.name, "mca.mp1");
125 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
126 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
127 	adev->mca.mp1.ras_if = &ras->ras_block.ras_comm;
128 
129 	return 0;
130 }
131 
132 int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev)
133 {
134 	int err;
135 	struct amdgpu_mca_ras_block *ras;
136 
137 	if (!adev->mca.mpio.ras)
138 		return 0;
139 
140 	ras = adev->mca.mpio.ras;
141 
142 	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
143 	if (err) {
144 		dev_err(adev->dev, "Failed to register mca.mpio ras block!\n");
145 		return err;
146 	}
147 
148 	strcpy(ras->ras_block.ras_comm.name, "mca.mpio");
149 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
150 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
151 	adev->mca.mpio.ras_if = &ras->ras_block.ras_comm;
152 
153 	return 0;
154 }
155 
156 static void amdgpu_mca_bank_set_init(struct mca_bank_set *mca_set)
157 {
158 	if (!mca_set)
159 		return;
160 
161 	memset(mca_set, 0, sizeof(*mca_set));
162 	INIT_LIST_HEAD(&mca_set->list);
163 }
164 
165 static int amdgpu_mca_bank_set_add_entry(struct mca_bank_set *mca_set, struct mca_bank_entry *entry)
166 {
167 	struct mca_bank_node *node;
168 
169 	if (!entry)
170 		return -EINVAL;
171 
172 	node = kvzalloc(sizeof(*node), GFP_KERNEL);
173 	if (!node)
174 		return -ENOMEM;
175 
176 	memcpy(&node->entry, entry, sizeof(*entry));
177 
178 	INIT_LIST_HEAD(&node->node);
179 	list_add_tail(&node->node, &mca_set->list);
180 
181 	mca_set->nr_entries++;
182 
183 	return 0;
184 }
185 
186 static int amdgpu_mca_bank_set_merge(struct mca_bank_set *mca_set, struct mca_bank_set *new)
187 {
188 	struct mca_bank_node *node;
189 
190 	list_for_each_entry(node, &new->list, node)
191 		amdgpu_mca_bank_set_add_entry(mca_set, &node->entry);
192 
193 	return 0;
194 }
195 
196 static void amdgpu_mca_bank_set_remove_node(struct mca_bank_set *mca_set, struct mca_bank_node *node)
197 {
198 	if (!node)
199 		return;
200 
201 	list_del(&node->node);
202 	kvfree(node);
203 
204 	mca_set->nr_entries--;
205 }
206 
207 static void amdgpu_mca_bank_set_release(struct mca_bank_set *mca_set)
208 {
209 	struct mca_bank_node *node, *tmp;
210 
211 	if (list_empty(&mca_set->list))
212 		return;
213 
214 	list_for_each_entry_safe(node, tmp, &mca_set->list, node)
215 		amdgpu_mca_bank_set_remove_node(mca_set, node);
216 }
217 
218 void amdgpu_mca_smu_init_funcs(struct amdgpu_device *adev, const struct amdgpu_mca_smu_funcs *mca_funcs)
219 {
220 	struct amdgpu_mca *mca = &adev->mca;
221 
222 	mca->mca_funcs = mca_funcs;
223 }
224 
225 int amdgpu_mca_init(struct amdgpu_device *adev)
226 {
227 	struct amdgpu_mca *mca = &adev->mca;
228 	struct mca_bank_cache *mca_cache;
229 	int i;
230 
231 	atomic_set(&mca->ue_update_flag, 0);
232 
233 	for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) {
234 		mca_cache = &mca->mca_caches[i];
235 		spin_lock_init(&mca_cache->lock);
236 		amdgpu_mca_bank_set_init(&mca_cache->mca_set);
237 	}
238 
239 	return 0;
240 }
241 
242 void amdgpu_mca_fini(struct amdgpu_device *adev)
243 {
244 	struct amdgpu_mca *mca = &adev->mca;
245 	struct mca_bank_cache *mca_cache;
246 	int i;
247 
248 	atomic_set(&mca->ue_update_flag, 0);
249 
250 	for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) {
251 		mca_cache = &mca->mca_caches[i];
252 		amdgpu_mca_bank_set_release(&mca_cache->mca_set);
253 	}
254 }
255 
256 int amdgpu_mca_reset(struct amdgpu_device *adev)
257 {
258 	amdgpu_mca_fini(adev);
259 
260 	return amdgpu_mca_init(adev);
261 }
262 
263 int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
264 {
265 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
266 
267 	if (mca_funcs && mca_funcs->mca_set_debug_mode)
268 		return mca_funcs->mca_set_debug_mode(adev, enable);
269 
270 	return -EOPNOTSUPP;
271 }
272 
273 static void amdgpu_mca_smu_mca_bank_dump(struct amdgpu_device *adev, int idx, struct mca_bank_entry *entry,
274 					 struct ras_query_context *qctx)
275 {
276 	u64 event_id = qctx->event_id;
277 
278 	RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n");
279 	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].STATUS=0x%016llx\n",
280 		      idx, entry->regs[MCA_REG_IDX_STATUS]);
281 	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].ADDR=0x%016llx\n",
282 		      idx, entry->regs[MCA_REG_IDX_ADDR]);
283 	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].MISC0=0x%016llx\n",
284 		      idx, entry->regs[MCA_REG_IDX_MISC0]);
285 	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].IPID=0x%016llx\n",
286 		      idx, entry->regs[MCA_REG_IDX_IPID]);
287 	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].SYND=0x%016llx\n",
288 		      idx, entry->regs[MCA_REG_IDX_SYND]);
289 }
290 
291 static int amdgpu_mca_smu_get_valid_mca_count(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, uint32_t *count)
292 {
293 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
294 
295 	if (!count)
296 		return -EINVAL;
297 
298 	if (mca_funcs && mca_funcs->mca_get_valid_mca_count)
299 		return mca_funcs->mca_get_valid_mca_count(adev, type, count);
300 
301 	return -EOPNOTSUPP;
302 }
303 
304 static int amdgpu_mca_smu_get_mca_entry(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
305 					int idx, struct mca_bank_entry *entry)
306 {
307 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
308 	int count;
309 
310 	if (!mca_funcs || !mca_funcs->mca_get_mca_entry)
311 		return -EOPNOTSUPP;
312 
313 	switch (type) {
314 	case AMDGPU_MCA_ERROR_TYPE_UE:
315 		count = mca_funcs->max_ue_count;
316 		break;
317 	case AMDGPU_MCA_ERROR_TYPE_CE:
318 		count = mca_funcs->max_ce_count;
319 		break;
320 	default:
321 		return -EINVAL;
322 	}
323 
324 	if (idx >= count)
325 		return -EINVAL;
326 
327 	return mca_funcs->mca_get_mca_entry(adev, type, idx, entry);
328 }
329 
330 static bool amdgpu_mca_bank_should_update(struct amdgpu_device *adev, enum amdgpu_mca_error_type type)
331 {
332 	struct amdgpu_mca *mca = &adev->mca;
333 	bool ret = true;
334 
335 	/*
336 	 * Because the UE Valid MCA count will only be cleared after reset,
337 	 * in order to avoid repeated counting of the error count,
338 	 * the aca bank is only updated once during the gpu recovery stage.
339 	 */
340 	if (type == AMDGPU_MCA_ERROR_TYPE_UE) {
341 		if (amdgpu_ras_intr_triggered())
342 			ret = atomic_cmpxchg(&mca->ue_update_flag, 0, 1) == 0;
343 		else
344 			atomic_set(&mca->ue_update_flag, 0);
345 	}
346 
347 	return ret;
348 }
349 
350 static int amdgpu_mca_smu_get_mca_set(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *mca_set,
351 				      struct ras_query_context *qctx)
352 {
353 	struct mca_bank_entry entry;
354 	uint32_t count = 0, i;
355 	int ret;
356 
357 	if (!mca_set)
358 		return -EINVAL;
359 
360 	if (!amdgpu_mca_bank_should_update(adev, type))
361 		return 0;
362 
363 	ret = amdgpu_mca_smu_get_valid_mca_count(adev, type, &count);
364 	if (ret)
365 		return ret;
366 
367 	for (i = 0; i < count; i++) {
368 		memset(&entry, 0, sizeof(entry));
369 		ret = amdgpu_mca_smu_get_mca_entry(adev, type, i, &entry);
370 		if (ret)
371 			return ret;
372 
373 		amdgpu_mca_bank_set_add_entry(mca_set, &entry);
374 
375 		amdgpu_mca_smu_mca_bank_dump(adev, i, &entry, qctx);
376 	}
377 
378 	return 0;
379 }
380 
381 static int amdgpu_mca_smu_parse_mca_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
382 						enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count)
383 {
384 	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
385 
386 	if (!count || !entry)
387 		return -EINVAL;
388 
389 	if (!mca_funcs || !mca_funcs->mca_parse_mca_error_count)
390 		return -EOPNOTSUPP;
391 
392 	return mca_funcs->mca_parse_mca_error_count(adev, blk, type, entry, count);
393 }
394 
395 static int amdgpu_mca_dispatch_mca_set(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
396 				       struct mca_bank_set *mca_set, struct ras_err_data *err_data)
397 {
398 	struct ras_err_addr err_addr;
399 	struct amdgpu_smuio_mcm_config_info mcm_info;
400 	struct mca_bank_node *node, *tmp;
401 	struct mca_bank_entry *entry;
402 	uint32_t count;
403 	int ret;
404 
405 	if (!mca_set)
406 		return -EINVAL;
407 
408 	if (!mca_set->nr_entries)
409 		return 0;
410 
411 	list_for_each_entry_safe(node, tmp, &mca_set->list, node) {
412 		entry = &node->entry;
413 
414 		count = 0;
415 		ret = amdgpu_mca_smu_parse_mca_error_count(adev, blk, type, entry, &count);
416 		if (ret && ret != -EOPNOTSUPP)
417 			return ret;
418 
419 		if (!count)
420 			continue;
421 
422 		memset(&mcm_info, 0, sizeof(mcm_info));
423 		memset(&err_addr, 0, sizeof(err_addr));
424 
425 		mcm_info.socket_id = entry->info.socket_id;
426 		mcm_info.die_id = entry->info.aid;
427 
428 		if (blk == AMDGPU_RAS_BLOCK__UMC) {
429 			err_addr.err_status = entry->regs[MCA_REG_IDX_STATUS];
430 			err_addr.err_ipid = entry->regs[MCA_REG_IDX_IPID];
431 			err_addr.err_addr = entry->regs[MCA_REG_IDX_ADDR];
432 		}
433 
434 		if (type == AMDGPU_MCA_ERROR_TYPE_UE) {
435 			amdgpu_ras_error_statistic_ue_count(err_data,
436 							    &mcm_info, &err_addr, (uint64_t)count);
437 		} else {
438 			if (amdgpu_mca_is_deferred_error(adev, entry->regs[MCA_REG_IDX_STATUS]))
439 				amdgpu_ras_error_statistic_de_count(err_data,
440 								    &mcm_info, &err_addr, (uint64_t)count);
441 			else
442 				amdgpu_ras_error_statistic_ce_count(err_data,
443 								    &mcm_info, &err_addr, (uint64_t)count);
444 		}
445 
446 		amdgpu_mca_bank_set_remove_node(mca_set, node);
447 	}
448 
449 	return 0;
450 }
451 
452 static int amdgpu_mca_add_mca_set_to_cache(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *new)
453 {
454 	struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type];
455 	int ret;
456 
457 	spin_lock(&mca_cache->lock);
458 	ret = amdgpu_mca_bank_set_merge(&mca_cache->mca_set, new);
459 	spin_unlock(&mca_cache->lock);
460 
461 	return ret;
462 }
463 
464 int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
465 				 struct ras_err_data *err_data, struct ras_query_context *qctx)
466 {
467 	struct mca_bank_set mca_set;
468 	struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type];
469 	int ret;
470 
471 	amdgpu_mca_bank_set_init(&mca_set);
472 
473 	ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, qctx);
474 	if (ret)
475 		goto out_mca_release;
476 
477 	ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_set, err_data);
478 	if (ret)
479 		goto out_mca_release;
480 
481 	/* add remain mca bank to mca cache */
482 	if (mca_set.nr_entries) {
483 		ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set);
484 		if (ret)
485 			goto out_mca_release;
486 	}
487 
488 	/* dispatch mca set again if mca cache has valid data */
489 	spin_lock(&mca_cache->lock);
490 	if (mca_cache->mca_set.nr_entries)
491 		ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_cache->mca_set, err_data);
492 	spin_unlock(&mca_cache->lock);
493 
494 out_mca_release:
495 	amdgpu_mca_bank_set_release(&mca_set);
496 
497 	return ret;
498 }
499 
500 #if defined(CONFIG_DEBUG_FS)
501 static int amdgpu_mca_smu_debug_mode_set(void *data, u64 val)
502 {
503 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
504 	int ret;
505 
506 	ret = amdgpu_ras_set_mca_debug_mode(adev, val ? true : false);
507 	if (ret)
508 		return ret;
509 
510 	dev_info(adev->dev, "amdgpu set smu mca debug mode %s success\n", val ? "on" : "off");
511 
512 	return 0;
513 }
514 
515 static void mca_dump_entry(struct seq_file *m, struct mca_bank_entry *entry)
516 {
517 	int i, idx = entry->idx;
518 	int reg_idx_array[] = {
519 		MCA_REG_IDX_STATUS,
520 		MCA_REG_IDX_ADDR,
521 		MCA_REG_IDX_MISC0,
522 		MCA_REG_IDX_IPID,
523 		MCA_REG_IDX_SYND,
524 	};
525 
526 	seq_printf(m, "mca entry[%d].type: %s\n", idx, entry->type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE");
527 	seq_printf(m, "mca entry[%d].ip: %d\n", idx, entry->ip);
528 	seq_printf(m, "mca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n",
529 		   idx, entry->info.socket_id, entry->info.aid, entry->info.hwid, entry->info.mcatype);
530 
531 	for (i = 0; i < ARRAY_SIZE(reg_idx_array); i++)
532 		seq_printf(m, "mca entry[%d].regs[%d]: 0x%016llx\n", idx, reg_idx_array[i], entry->regs[reg_idx_array[i]]);
533 }
534 
535 static int mca_dump_show(struct seq_file *m, enum amdgpu_mca_error_type type)
536 {
537 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
538 	struct mca_bank_node *node;
539 	struct mca_bank_set mca_set;
540 	struct ras_query_context qctx;
541 	int ret;
542 
543 	amdgpu_mca_bank_set_init(&mca_set);
544 
545 	qctx.event_id = 0ULL;
546 	ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, &qctx);
547 	if (ret)
548 		goto err_free_mca_set;
549 
550 	seq_printf(m, "amdgpu smu %s valid mca count: %d\n",
551 		   type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE", mca_set.nr_entries);
552 
553 	if (!mca_set.nr_entries)
554 		goto err_free_mca_set;
555 
556 	list_for_each_entry(node, &mca_set.list, node)
557 		mca_dump_entry(m, &node->entry);
558 
559 	/* add mca bank to mca bank cache */
560 	ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set);
561 
562 err_free_mca_set:
563 	amdgpu_mca_bank_set_release(&mca_set);
564 
565 	return ret;
566 }
567 
568 static int mca_dump_ce_show(struct seq_file *m, void *unused)
569 {
570 	return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_CE);
571 }
572 
573 static int mca_dump_ce_open(struct inode *inode, struct file *file)
574 {
575 	return single_open(file, mca_dump_ce_show, inode->i_private);
576 }
577 
578 static const struct file_operations mca_ce_dump_debug_fops = {
579 	.owner = THIS_MODULE,
580 	.open = mca_dump_ce_open,
581 	.read = seq_read,
582 	.llseek = seq_lseek,
583 	.release = single_release,
584 };
585 
586 static int mca_dump_ue_show(struct seq_file *m, void *unused)
587 {
588 	return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_UE);
589 }
590 
591 static int mca_dump_ue_open(struct inode *inode, struct file *file)
592 {
593 	return single_open(file, mca_dump_ue_show, inode->i_private);
594 }
595 
596 static const struct file_operations mca_ue_dump_debug_fops = {
597 	.owner = THIS_MODULE,
598 	.open = mca_dump_ue_open,
599 	.read = seq_read,
600 	.llseek = seq_lseek,
601 	.release = single_release,
602 };
603 
604 DEFINE_DEBUGFS_ATTRIBUTE(mca_debug_mode_fops, NULL, amdgpu_mca_smu_debug_mode_set, "%llu\n");
605 #endif
606 
607 void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root)
608 {
609 #if defined(CONFIG_DEBUG_FS)
610 	if (!root)
611 		return;
612 
613 	debugfs_create_file("mca_debug_mode", 0200, root, adev, &mca_debug_mode_fops);
614 	debugfs_create_file("mca_ue_dump", 0400, root, adev, &mca_ue_dump_debug_fops);
615 	debugfs_create_file("mca_ce_dump", 0400, root, adev, &mca_ce_dump_debug_fops);
616 #endif
617 }
618 
619