xref: /linux/drivers/gpu/drm/xe/xe_devcoredump.c (revision e04e2b760ddbe3d7b283a05898c3a029085cd8cd)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2023 Intel Corporation
4  */
5 
6 #include "xe_devcoredump.h"
7 #include "xe_devcoredump_types.h"
8 
9 #include <linux/devcoredump.h>
10 #include <generated/utsrelease.h>
11 
12 #include <drm/drm_managed.h>
13 
14 #include "xe_device.h"
15 #include "xe_exec_queue.h"
16 #include "xe_force_wake.h"
17 #include "xe_gt.h"
18 #include "xe_gt_printk.h"
19 #include "xe_guc_ct.h"
20 #include "xe_guc_submit.h"
21 #include "xe_hw_engine.h"
22 #include "xe_sched_job.h"
23 #include "xe_vm.h"
24 
25 /**
26  * DOC: Xe device coredump
27  *
28  * Devices overview:
29  * Xe uses dev_coredump infrastructure for exposing the crash errors in a
30  * standardized way.
31  * devcoredump exposes a temporary device under /sys/class/devcoredump/
32  * which is linked with our card device directly.
33  * The core dump can be accessed either from
34  * /sys/class/drm/card<n>/device/devcoredump/ or from
35  * /sys/class/devcoredump/devcd<m> where
36  * /sys/class/devcoredump/devcd<m>/failing_device is a link to
37  * /sys/class/drm/card<n>/device/.
38  *
39  * Snapshot at hang:
40  * The 'data' file is printed with a drm_printer pointer at devcoredump read
41  * time. For this reason, we need to take snapshots from when the hang has
42  * happened, and not only when the user is reading the file. Otherwise the
43  * information is outdated since the resets might have happened in between.
44  *
45  * 'First' failure snapshot:
46  * In general, the first hang is the most critical one since the following hangs
47  * can be a consequence of the initial hang. For this reason we only take the
48  * snapshot of the 'first' failure and ignore subsequent calls of this function,
49  * at least while the coredump device is alive. Dev_coredump has a delayed work
50  * queue that will eventually delete the device and free all the dump
51  * information.
52  */
53 
54 #ifdef CONFIG_DEV_COREDUMP
55 
56 /* 1 hour timeout */
57 #define XE_COREDUMP_TIMEOUT_JIFFIES (60 * 60 * HZ)
58 
59 static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump)
60 {
61 	return container_of(coredump, struct xe_device, devcoredump);
62 }
63 
64 static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q)
65 {
66 	return &q->gt->uc.guc;
67 }
68 
69 static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
70 {
71 	struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work);
72 
73 	/* keep going if fw fails as we still want to save the memory and SW data */
74 	if (xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL))
75 		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
76 	xe_vm_snapshot_capture_delayed(ss->vm);
77 	xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
78 	xe_force_wake_put(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
79 }
80 
81 static ssize_t xe_devcoredump_read(char *buffer, loff_t offset,
82 				   size_t count, void *data, size_t datalen)
83 {
84 	struct xe_devcoredump *coredump = data;
85 	struct xe_device *xe;
86 	struct xe_devcoredump_snapshot *ss;
87 	struct drm_printer p;
88 	struct drm_print_iterator iter;
89 	struct timespec64 ts;
90 	int i;
91 
92 	if (!coredump)
93 		return -ENODEV;
94 
95 	xe = coredump_to_xe(coredump);
96 	ss = &coredump->snapshot;
97 
98 	/* Ensure delayed work is captured before continuing */
99 	flush_work(&ss->work);
100 
101 	iter.data = buffer;
102 	iter.offset = 0;
103 	iter.start = offset;
104 	iter.remain = count;
105 
106 	p = drm_coredump_printer(&iter);
107 
108 	drm_printf(&p, "**** Xe Device Coredump ****\n");
109 	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
110 	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
111 
112 	ts = ktime_to_timespec64(ss->snapshot_time);
113 	drm_printf(&p, "Snapshot time: %lld.%09ld\n", ts.tv_sec, ts.tv_nsec);
114 	ts = ktime_to_timespec64(ss->boot_time);
115 	drm_printf(&p, "Uptime: %lld.%09ld\n", ts.tv_sec, ts.tv_nsec);
116 	drm_printf(&p, "Process: %s\n", ss->process_name);
117 	xe_device_snapshot_print(xe, &p);
118 
119 	drm_printf(&p, "\n**** GuC CT ****\n");
120 	xe_guc_ct_snapshot_print(coredump->snapshot.ct, &p);
121 	xe_guc_exec_queue_snapshot_print(coredump->snapshot.ge, &p);
122 
123 	drm_printf(&p, "\n**** Job ****\n");
124 	xe_sched_job_snapshot_print(coredump->snapshot.job, &p);
125 
126 	drm_printf(&p, "\n**** HW Engines ****\n");
127 	for (i = 0; i < XE_NUM_HW_ENGINES; i++)
128 		if (coredump->snapshot.hwe[i])
129 			xe_hw_engine_snapshot_print(coredump->snapshot.hwe[i],
130 						    &p);
131 	drm_printf(&p, "\n**** VM state ****\n");
132 	xe_vm_snapshot_print(coredump->snapshot.vm, &p);
133 
134 	return count - iter.remain;
135 }
136 
137 static void xe_devcoredump_free(void *data)
138 {
139 	struct xe_devcoredump *coredump = data;
140 	int i;
141 
142 	/* Our device is gone. Nothing to do... */
143 	if (!data || !coredump_to_xe(coredump))
144 		return;
145 
146 	cancel_work_sync(&coredump->snapshot.work);
147 
148 	xe_guc_ct_snapshot_free(coredump->snapshot.ct);
149 	xe_guc_exec_queue_snapshot_free(coredump->snapshot.ge);
150 	xe_sched_job_snapshot_free(coredump->snapshot.job);
151 	for (i = 0; i < XE_NUM_HW_ENGINES; i++)
152 		if (coredump->snapshot.hwe[i])
153 			xe_hw_engine_snapshot_free(coredump->snapshot.hwe[i]);
154 	xe_vm_snapshot_free(coredump->snapshot.vm);
155 
156 	/* To prevent stale data on next snapshot, clear everything */
157 	memset(&coredump->snapshot, 0, sizeof(coredump->snapshot));
158 	coredump->captured = false;
159 	drm_info(&coredump_to_xe(coredump)->drm,
160 		 "Xe device coredump has been deleted.\n");
161 }
162 
163 static void devcoredump_snapshot(struct xe_devcoredump *coredump,
164 				 struct xe_sched_job *job)
165 {
166 	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
167 	struct xe_exec_queue *q = job->q;
168 	struct xe_guc *guc = exec_queue_to_guc(q);
169 	struct xe_hw_engine *hwe;
170 	enum xe_hw_engine_id id;
171 	u32 adj_logical_mask = q->logical_mask;
172 	u32 width_mask = (0x1 << q->width) - 1;
173 	const char *process_name = "no process";
174 	struct task_struct *task = NULL;
175 
176 	int i;
177 	bool cookie;
178 
179 	ss->snapshot_time = ktime_get_real();
180 	ss->boot_time = ktime_get_boottime();
181 
182 	if (q->vm && q->vm->xef) {
183 		task = get_pid_task(q->vm->xef->drm->pid, PIDTYPE_PID);
184 		if (task)
185 			process_name = task->comm;
186 	}
187 	strscpy(ss->process_name, process_name);
188 	if (task)
189 		put_task_struct(task);
190 
191 	ss->gt = q->gt;
192 	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
193 
194 	cookie = dma_fence_begin_signalling();
195 	for (i = 0; q->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) {
196 		if (adj_logical_mask & BIT(i)) {
197 			adj_logical_mask |= width_mask << i;
198 			i += q->width;
199 		} else {
200 			++i;
201 		}
202 	}
203 
204 	/* keep going if fw fails as we still want to save the memory and SW data */
205 	if (xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL))
206 		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
207 
208 	coredump->snapshot.ct = xe_guc_ct_snapshot_capture(&guc->ct, true);
209 	coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(q);
210 	coredump->snapshot.job = xe_sched_job_snapshot_capture(job);
211 	coredump->snapshot.vm = xe_vm_snapshot_capture(q->vm);
212 
213 	for_each_hw_engine(hwe, q->gt, id) {
214 		if (hwe->class != q->hwe->class ||
215 		    !(BIT(hwe->logical_instance) & adj_logical_mask)) {
216 			coredump->snapshot.hwe[id] = NULL;
217 			continue;
218 		}
219 		coredump->snapshot.hwe[id] = xe_hw_engine_snapshot_capture(hwe);
220 	}
221 
222 	queue_work(system_unbound_wq, &ss->work);
223 
224 	xe_force_wake_put(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
225 	dma_fence_end_signalling(cookie);
226 }
227 
228 /**
229  * xe_devcoredump - Take the required snapshots and initialize coredump device.
230  * @job: The faulty xe_sched_job, where the issue was detected.
231  *
232  * This function should be called at the crash time within the serialized
233  * gt_reset. It is skipped if we still have the core dump device available
234  * with the information of the 'first' snapshot.
235  */
236 void xe_devcoredump(struct xe_sched_job *job)
237 {
238 	struct xe_device *xe = gt_to_xe(job->q->gt);
239 	struct xe_devcoredump *coredump = &xe->devcoredump;
240 
241 	if (coredump->captured) {
242 		drm_dbg(&xe->drm, "Multiple hangs are occurring, but only the first snapshot was taken\n");
243 		return;
244 	}
245 
246 	coredump->captured = true;
247 	devcoredump_snapshot(coredump, job);
248 
249 	drm_info(&xe->drm, "Xe device coredump has been created\n");
250 	drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n",
251 		 xe->drm.primary->index);
252 
253 	dev_coredumpm_timeout(xe->drm.dev, THIS_MODULE, coredump, 0, GFP_KERNEL,
254 			      xe_devcoredump_read, xe_devcoredump_free,
255 			      XE_COREDUMP_TIMEOUT_JIFFIES);
256 }
257 
258 static void xe_driver_devcoredump_fini(void *arg)
259 {
260 	struct drm_device *drm = arg;
261 
262 	dev_coredump_put(drm->dev);
263 }
264 
265 int xe_devcoredump_init(struct xe_device *xe)
266 {
267 	return devm_add_action_or_reset(xe->drm.dev, xe_driver_devcoredump_fini, &xe->drm);
268 }
269 #endif
270