1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_dm_debugfs.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_rap.h"
36 #include "amdgpu_securedisplay.h"
37 #include "amdgpu_fw_attestation.h"
38 #include "amdgpu_umr.h"
39
40 #include "amdgpu_reset.h"
41 #include "amdgpu_psp_ta.h"
42
43 #if defined(CONFIG_DEBUG_FS)
44
45 /**
46 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47 *
48 * @read: True if reading
49 * @f: open file handle
50 * @buf: User buffer to write/read to
51 * @size: Number of bytes to write/read
52 * @pos: Offset to seek to
53 *
54 * This debugfs entry has special meaning on the offset being sought.
55 * Various bits have different meanings:
56 *
57 * Bit 62: Indicates a GRBM bank switch is needed
58 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
59 * zero)
60 * Bits 24..33: The SE or ME selector if needed
61 * Bits 34..43: The SH (or SA) or PIPE selector if needed
62 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63 *
64 * Bit 23: Indicates that the PM power gating lock should be held
65 * This is necessary to read registers that might be
66 * unreliable during a power gating transistion.
67 *
68 * The lower bits are the BYTE offset of the register to read. This
69 * allows reading multiple registers in a single call and having
70 * the returned size reflect that.
71 */
amdgpu_debugfs_process_reg_op(bool read,struct file * f,char __user * buf,size_t size,loff_t * pos)72 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 char __user *buf, size_t size, loff_t *pos)
74 {
75 struct amdgpu_device *adev = file_inode(f)->i_private;
76 ssize_t result = 0;
77 int r;
78 bool pm_pg_lock, use_bank, use_ring;
79 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80
81 pm_pg_lock = use_bank = use_ring = false;
82 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83
84 if (size & 0x3 || *pos & 0x3 ||
85 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 return -EINVAL;
87
88 /* are we reading registers for which a PG lock is necessary? */
89 pm_pg_lock = (*pos >> 23) & 1;
90
91 if (*pos & (1ULL << 62)) {
92 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95
96 if (se_bank == 0x3FF)
97 se_bank = 0xFFFFFFFF;
98 if (sh_bank == 0x3FF)
99 sh_bank = 0xFFFFFFFF;
100 if (instance_bank == 0x3FF)
101 instance_bank = 0xFFFFFFFF;
102 use_bank = true;
103 } else if (*pos & (1ULL << 61)) {
104
105 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109
110 use_ring = true;
111 } else {
112 use_bank = use_ring = false;
113 }
114
115 *pos &= (1UL << 22) - 1;
116
117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 if (r < 0) {
119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 return r;
121 }
122
123 r = amdgpu_virt_enable_access_debugfs(adev);
124 if (r < 0) {
125 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 return r;
127 }
128
129 if (use_bank) {
130 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 amdgpu_virt_disable_access_debugfs(adev);
135 return -EINVAL;
136 }
137 mutex_lock(&adev->grbm_idx_mutex);
138 amdgpu_gfx_select_se_sh(adev, se_bank,
139 sh_bank, instance_bank, 0);
140 } else if (use_ring) {
141 mutex_lock(&adev->srbm_mutex);
142 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
143 }
144
145 if (pm_pg_lock)
146 mutex_lock(&adev->pm.mutex);
147
148 while (size) {
149 uint32_t value;
150
151 if (read) {
152 value = RREG32(*pos >> 2);
153 r = put_user(value, (uint32_t *)buf);
154 } else {
155 r = get_user(value, (uint32_t *)buf);
156 if (!r)
157 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
158 }
159 if (r) {
160 result = r;
161 goto end;
162 }
163
164 result += 4;
165 buf += 4;
166 *pos += 4;
167 size -= 4;
168 }
169
170 end:
171 if (use_bank) {
172 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
173 mutex_unlock(&adev->grbm_idx_mutex);
174 } else if (use_ring) {
175 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
176 mutex_unlock(&adev->srbm_mutex);
177 }
178
179 if (pm_pg_lock)
180 mutex_unlock(&adev->pm.mutex);
181
182 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184
185 amdgpu_virt_disable_access_debugfs(adev);
186 return result;
187 }
188
189 /*
190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191 */
amdgpu_debugfs_regs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 size_t size, loff_t *pos)
194 {
195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196 }
197
198 /*
199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200 */
amdgpu_debugfs_regs_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 size_t size, loff_t *pos)
203 {
204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205 }
206
amdgpu_debugfs_regs2_open(struct inode * inode,struct file * file)207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208 {
209 struct amdgpu_debugfs_regs2_data *rd;
210
211 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
212 if (!rd)
213 return -ENOMEM;
214 rd->adev = file_inode(file)->i_private;
215 file->private_data = rd;
216 mutex_init(&rd->lock);
217
218 return 0;
219 }
220
amdgpu_debugfs_regs2_release(struct inode * inode,struct file * file)221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222 {
223 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224
225 mutex_destroy(&rd->lock);
226 kfree(file->private_data);
227 return 0;
228 }
229
amdgpu_debugfs_regs2_op(struct file * f,char __user * buf,u32 offset,size_t size,int write_en)230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
231 {
232 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
233 struct amdgpu_device *adev = rd->adev;
234 ssize_t result = 0;
235 int r;
236 uint32_t value;
237
238 if (size & 0x3 || offset & 0x3)
239 return -EINVAL;
240
241 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
242 if (r < 0) {
243 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
244 return r;
245 }
246
247 r = amdgpu_virt_enable_access_debugfs(adev);
248 if (r < 0) {
249 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
250 return r;
251 }
252
253 mutex_lock(&rd->lock);
254
255 if (rd->id.use_grbm) {
256 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
257 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
258 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
259 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
260 amdgpu_virt_disable_access_debugfs(adev);
261 mutex_unlock(&rd->lock);
262 return -EINVAL;
263 }
264 mutex_lock(&adev->grbm_idx_mutex);
265 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
266 rd->id.grbm.sh,
267 rd->id.grbm.instance, rd->id.xcc_id);
268 }
269
270 if (rd->id.use_srbm) {
271 mutex_lock(&adev->srbm_mutex);
272 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
273 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
274 }
275
276 if (rd->id.pg_lock)
277 mutex_lock(&adev->pm.mutex);
278
279 while (size) {
280 if (!write_en) {
281 value = RREG32(offset >> 2);
282 r = put_user(value, (uint32_t *)buf);
283 } else {
284 r = get_user(value, (uint32_t *)buf);
285 if (!r)
286 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
287 }
288 if (r) {
289 result = r;
290 goto end;
291 }
292 offset += 4;
293 size -= 4;
294 result += 4;
295 buf += 4;
296 }
297 end:
298 if (rd->id.use_grbm) {
299 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
300 mutex_unlock(&adev->grbm_idx_mutex);
301 }
302
303 if (rd->id.use_srbm) {
304 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
305 mutex_unlock(&adev->srbm_mutex);
306 }
307
308 if (rd->id.pg_lock)
309 mutex_unlock(&adev->pm.mutex);
310
311 mutex_unlock(&rd->lock);
312
313 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
314 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
315
316 amdgpu_virt_disable_access_debugfs(adev);
317 return result;
318 }
319
amdgpu_debugfs_regs2_ioctl(struct file * f,unsigned int cmd,unsigned long data)320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
321 {
322 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
323 struct amdgpu_debugfs_regs2_iocdata v1_data;
324 int r;
325
326 mutex_lock(&rd->lock);
327
328 switch (cmd) {
329 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
330 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
331 sizeof(rd->id));
332 if (r)
333 r = -EINVAL;
334 goto done;
335 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
336 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
337 sizeof(v1_data));
338 if (r) {
339 r = -EINVAL;
340 goto done;
341 }
342 goto v1_copy;
343 default:
344 r = -EINVAL;
345 goto done;
346 }
347
348 v1_copy:
349 rd->id.use_srbm = v1_data.use_srbm;
350 rd->id.use_grbm = v1_data.use_grbm;
351 rd->id.pg_lock = v1_data.pg_lock;
352 rd->id.grbm.se = v1_data.grbm.se;
353 rd->id.grbm.sh = v1_data.grbm.sh;
354 rd->id.grbm.instance = v1_data.grbm.instance;
355 rd->id.srbm.me = v1_data.srbm.me;
356 rd->id.srbm.pipe = v1_data.srbm.pipe;
357 rd->id.srbm.queue = v1_data.srbm.queue;
358 rd->id.xcc_id = 0;
359 done:
360 mutex_unlock(&rd->lock);
361 return r;
362 }
363
amdgpu_debugfs_regs2_read(struct file * f,char __user * buf,size_t size,loff_t * pos)364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
365 {
366 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
367 }
368
amdgpu_debugfs_regs2_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
370 {
371 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
372 }
373
amdgpu_debugfs_gprwave_open(struct inode * inode,struct file * file)374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
375 {
376 struct amdgpu_debugfs_gprwave_data *rd;
377
378 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
379 if (!rd)
380 return -ENOMEM;
381 rd->adev = file_inode(file)->i_private;
382 file->private_data = rd;
383 mutex_init(&rd->lock);
384
385 return 0;
386 }
387
amdgpu_debugfs_gprwave_release(struct inode * inode,struct file * file)388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
389 {
390 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
391
392 mutex_destroy(&rd->lock);
393 kfree(file->private_data);
394 return 0;
395 }
396
amdgpu_debugfs_gprwave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)397 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
398 {
399 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
400 struct amdgpu_device *adev = rd->adev;
401 ssize_t result = 0;
402 int r;
403 uint32_t *data, x;
404
405 if (size & 0x3 || *pos & 0x3)
406 return -EINVAL;
407
408 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
409 if (r < 0) {
410 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
411 return r;
412 }
413
414 r = amdgpu_virt_enable_access_debugfs(adev);
415 if (r < 0) {
416 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
417 return r;
418 }
419
420 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
421 if (!data) {
422 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
423 amdgpu_virt_disable_access_debugfs(adev);
424 return -ENOMEM;
425 }
426
427 /* switch to the specific se/sh/cu */
428 mutex_lock(&adev->grbm_idx_mutex);
429 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
430
431 if (!rd->id.gpr_or_wave) {
432 x = 0;
433 if (adev->gfx.funcs->read_wave_data)
434 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
435 } else {
436 x = size >> 2;
437 if (rd->id.gpr.vpgr_or_sgpr) {
438 if (adev->gfx.funcs->read_wave_vgprs)
439 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
440 } else {
441 if (adev->gfx.funcs->read_wave_sgprs)
442 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
443 }
444 }
445
446 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
447 mutex_unlock(&adev->grbm_idx_mutex);
448
449 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
450 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
451
452 if (!x) {
453 result = -EINVAL;
454 goto done;
455 }
456
457 while (size && (*pos < x * 4)) {
458 uint32_t value;
459
460 value = data[*pos >> 2];
461 r = put_user(value, (uint32_t *)buf);
462 if (r) {
463 result = r;
464 goto done;
465 }
466
467 result += 4;
468 buf += 4;
469 *pos += 4;
470 size -= 4;
471 }
472
473 done:
474 amdgpu_virt_disable_access_debugfs(adev);
475 kfree(data);
476 return result;
477 }
478
amdgpu_debugfs_gprwave_ioctl(struct file * f,unsigned int cmd,unsigned long data)479 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
480 {
481 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
482 int r = 0;
483
484 mutex_lock(&rd->lock);
485
486 switch (cmd) {
487 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
488 if (copy_from_user(&rd->id,
489 (struct amdgpu_debugfs_gprwave_iocdata *)data,
490 sizeof(rd->id)))
491 r = -EFAULT;
492 goto done;
493 default:
494 r = -EINVAL;
495 goto done;
496 }
497
498 done:
499 mutex_unlock(&rd->lock);
500 return r;
501 }
502
503
504
505
506 /**
507 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
508 *
509 * @f: open file handle
510 * @buf: User buffer to store read data in
511 * @size: Number of bytes to read
512 * @pos: Offset to seek to
513 *
514 * The lower bits are the BYTE offset of the register to read. This
515 * allows reading multiple registers in a single call and having
516 * the returned size reflect that.
517 */
amdgpu_debugfs_regs_pcie_read(struct file * f,char __user * buf,size_t size,loff_t * pos)518 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
519 size_t size, loff_t *pos)
520 {
521 struct amdgpu_device *adev = file_inode(f)->i_private;
522 ssize_t result = 0;
523 int r;
524
525 if (size & 0x3 || *pos & 0x3)
526 return -EINVAL;
527
528 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
529 if (r < 0) {
530 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
531 return r;
532 }
533
534 r = amdgpu_virt_enable_access_debugfs(adev);
535 if (r < 0) {
536 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
537 return r;
538 }
539
540 while (size) {
541 uint32_t value;
542
543 if (upper_32_bits(*pos))
544 value = RREG32_PCIE_EXT(*pos);
545 else
546 value = RREG32_PCIE(*pos);
547
548 r = put_user(value, (uint32_t *)buf);
549 if (r)
550 goto out;
551
552 result += 4;
553 buf += 4;
554 *pos += 4;
555 size -= 4;
556 }
557
558 r = result;
559 out:
560 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
561 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
562 amdgpu_virt_disable_access_debugfs(adev);
563 return r;
564 }
565
566 /**
567 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
568 *
569 * @f: open file handle
570 * @buf: User buffer to write data from
571 * @size: Number of bytes to write
572 * @pos: Offset to seek to
573 *
574 * The lower bits are the BYTE offset of the register to write. This
575 * allows writing multiple registers in a single call and having
576 * the returned size reflect that.
577 */
amdgpu_debugfs_regs_pcie_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)578 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
579 size_t size, loff_t *pos)
580 {
581 struct amdgpu_device *adev = file_inode(f)->i_private;
582 ssize_t result = 0;
583 int r;
584
585 if (size & 0x3 || *pos & 0x3)
586 return -EINVAL;
587
588 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
589 if (r < 0) {
590 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
591 return r;
592 }
593
594 r = amdgpu_virt_enable_access_debugfs(adev);
595 if (r < 0) {
596 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
597 return r;
598 }
599
600 while (size) {
601 uint32_t value;
602
603 r = get_user(value, (uint32_t *)buf);
604 if (r)
605 goto out;
606
607 if (upper_32_bits(*pos))
608 WREG32_PCIE_EXT(*pos, value);
609 else
610 WREG32_PCIE(*pos, value);
611
612 result += 4;
613 buf += 4;
614 *pos += 4;
615 size -= 4;
616 }
617
618 r = result;
619 out:
620 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
621 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
622 amdgpu_virt_disable_access_debugfs(adev);
623 return r;
624 }
625
626 /**
627 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
628 *
629 * @f: open file handle
630 * @buf: User buffer to store read data in
631 * @size: Number of bytes to read
632 * @pos: Offset to seek to
633 *
634 * The lower bits are the BYTE offset of the register to read. This
635 * allows reading multiple registers in a single call and having
636 * the returned size reflect that.
637 */
amdgpu_debugfs_regs_didt_read(struct file * f,char __user * buf,size_t size,loff_t * pos)638 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
639 size_t size, loff_t *pos)
640 {
641 struct amdgpu_device *adev = file_inode(f)->i_private;
642 ssize_t result = 0;
643 int r;
644
645 if (size & 0x3 || *pos & 0x3)
646 return -EINVAL;
647
648 if (!adev->didt_rreg)
649 return -EOPNOTSUPP;
650
651 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
652 if (r < 0) {
653 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
654 return r;
655 }
656
657 r = amdgpu_virt_enable_access_debugfs(adev);
658 if (r < 0) {
659 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
660 return r;
661 }
662
663 while (size) {
664 uint32_t value;
665
666 value = RREG32_DIDT(*pos >> 2);
667 r = put_user(value, (uint32_t *)buf);
668 if (r)
669 goto out;
670
671 result += 4;
672 buf += 4;
673 *pos += 4;
674 size -= 4;
675 }
676
677 r = result;
678 out:
679 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
680 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
681 amdgpu_virt_disable_access_debugfs(adev);
682 return r;
683 }
684
685 /**
686 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
687 *
688 * @f: open file handle
689 * @buf: User buffer to write data from
690 * @size: Number of bytes to write
691 * @pos: Offset to seek to
692 *
693 * The lower bits are the BYTE offset of the register to write. This
694 * allows writing multiple registers in a single call and having
695 * the returned size reflect that.
696 */
amdgpu_debugfs_regs_didt_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)697 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
698 size_t size, loff_t *pos)
699 {
700 struct amdgpu_device *adev = file_inode(f)->i_private;
701 ssize_t result = 0;
702 int r;
703
704 if (size & 0x3 || *pos & 0x3)
705 return -EINVAL;
706
707 if (!adev->didt_wreg)
708 return -EOPNOTSUPP;
709
710 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
711 if (r < 0) {
712 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
713 return r;
714 }
715
716 r = amdgpu_virt_enable_access_debugfs(adev);
717 if (r < 0) {
718 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
719 return r;
720 }
721
722 while (size) {
723 uint32_t value;
724
725 r = get_user(value, (uint32_t *)buf);
726 if (r)
727 goto out;
728
729 WREG32_DIDT(*pos >> 2, value);
730
731 result += 4;
732 buf += 4;
733 *pos += 4;
734 size -= 4;
735 }
736
737 r = result;
738 out:
739 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
740 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
741 amdgpu_virt_disable_access_debugfs(adev);
742 return r;
743 }
744
745 /**
746 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
747 *
748 * @f: open file handle
749 * @buf: User buffer to store read data in
750 * @size: Number of bytes to read
751 * @pos: Offset to seek to
752 *
753 * The lower bits are the BYTE offset of the register to read. This
754 * allows reading multiple registers in a single call and having
755 * the returned size reflect that.
756 */
amdgpu_debugfs_regs_smc_read(struct file * f,char __user * buf,size_t size,loff_t * pos)757 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
758 size_t size, loff_t *pos)
759 {
760 struct amdgpu_device *adev = file_inode(f)->i_private;
761 ssize_t result = 0;
762 int r;
763
764 if (!adev->smc_rreg)
765 return -EOPNOTSUPP;
766
767 if (size & 0x3 || *pos & 0x3)
768 return -EINVAL;
769
770 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
771 if (r < 0) {
772 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
773 return r;
774 }
775
776 r = amdgpu_virt_enable_access_debugfs(adev);
777 if (r < 0) {
778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
779 return r;
780 }
781
782 while (size) {
783 uint32_t value;
784
785 value = RREG32_SMC(*pos);
786 r = put_user(value, (uint32_t *)buf);
787 if (r)
788 goto out;
789
790 result += 4;
791 buf += 4;
792 *pos += 4;
793 size -= 4;
794 }
795
796 r = result;
797 out:
798 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
799 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
800 amdgpu_virt_disable_access_debugfs(adev);
801 return r;
802 }
803
804 /**
805 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
806 *
807 * @f: open file handle
808 * @buf: User buffer to write data from
809 * @size: Number of bytes to write
810 * @pos: Offset to seek to
811 *
812 * The lower bits are the BYTE offset of the register to write. This
813 * allows writing multiple registers in a single call and having
814 * the returned size reflect that.
815 */
amdgpu_debugfs_regs_smc_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)816 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
817 size_t size, loff_t *pos)
818 {
819 struct amdgpu_device *adev = file_inode(f)->i_private;
820 ssize_t result = 0;
821 int r;
822
823 if (!adev->smc_wreg)
824 return -EOPNOTSUPP;
825
826 if (size & 0x3 || *pos & 0x3)
827 return -EINVAL;
828
829 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
830 if (r < 0) {
831 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
832 return r;
833 }
834
835 r = amdgpu_virt_enable_access_debugfs(adev);
836 if (r < 0) {
837 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
838 return r;
839 }
840
841 while (size) {
842 uint32_t value;
843
844 r = get_user(value, (uint32_t *)buf);
845 if (r)
846 goto out;
847
848 WREG32_SMC(*pos, value);
849
850 result += 4;
851 buf += 4;
852 *pos += 4;
853 size -= 4;
854 }
855
856 r = result;
857 out:
858 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
859 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
860 amdgpu_virt_disable_access_debugfs(adev);
861 return r;
862 }
863
864 /**
865 * amdgpu_debugfs_gca_config_read - Read from gfx config data
866 *
867 * @f: open file handle
868 * @buf: User buffer to store read data in
869 * @size: Number of bytes to read
870 * @pos: Offset to seek to
871 *
872 * This file is used to access configuration data in a somewhat
873 * stable fashion. The format is a series of DWORDs with the first
874 * indicating which revision it is. New content is appended to the
875 * end so that older software can still read the data.
876 */
877
amdgpu_debugfs_gca_config_read(struct file * f,char __user * buf,size_t size,loff_t * pos)878 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
879 size_t size, loff_t *pos)
880 {
881 struct amdgpu_device *adev = file_inode(f)->i_private;
882 ssize_t result = 0;
883 int r;
884 uint32_t *config, no_regs = 0;
885
886 if (size & 0x3 || *pos & 0x3)
887 return -EINVAL;
888
889 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
890 if (!config)
891 return -ENOMEM;
892
893 /* version, increment each time something is added */
894 config[no_regs++] = 5;
895 config[no_regs++] = adev->gfx.config.max_shader_engines;
896 config[no_regs++] = adev->gfx.config.max_tile_pipes;
897 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
898 config[no_regs++] = adev->gfx.config.max_sh_per_se;
899 config[no_regs++] = adev->gfx.config.max_backends_per_se;
900 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
901 config[no_regs++] = adev->gfx.config.max_gprs;
902 config[no_regs++] = adev->gfx.config.max_gs_threads;
903 config[no_regs++] = adev->gfx.config.max_hw_contexts;
904 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
905 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
906 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
907 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
908 config[no_regs++] = adev->gfx.config.num_tile_pipes;
909 config[no_regs++] = adev->gfx.config.backend_enable_mask;
910 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
911 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
912 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
913 config[no_regs++] = adev->gfx.config.num_gpus;
914 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
915 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
916 config[no_regs++] = adev->gfx.config.gb_addr_config;
917 config[no_regs++] = adev->gfx.config.num_rbs;
918
919 /* rev==1 */
920 config[no_regs++] = adev->rev_id;
921 config[no_regs++] = adev->pg_flags;
922 config[no_regs++] = lower_32_bits(adev->cg_flags);
923
924 /* rev==2 */
925 config[no_regs++] = adev->family;
926 config[no_regs++] = adev->external_rev_id;
927
928 /* rev==3 */
929 config[no_regs++] = adev->pdev->device;
930 config[no_regs++] = adev->pdev->revision;
931 config[no_regs++] = adev->pdev->subsystem_device;
932 config[no_regs++] = adev->pdev->subsystem_vendor;
933
934 /* rev==4 APU flag */
935 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
936
937 /* rev==5 PG/CG flag upper 32bit */
938 config[no_regs++] = 0;
939 config[no_regs++] = upper_32_bits(adev->cg_flags);
940
941 while (size && (*pos < no_regs * 4)) {
942 uint32_t value;
943
944 value = config[*pos >> 2];
945 r = put_user(value, (uint32_t *)buf);
946 if (r) {
947 kfree(config);
948 return r;
949 }
950
951 result += 4;
952 buf += 4;
953 *pos += 4;
954 size -= 4;
955 }
956
957 kfree(config);
958 return result;
959 }
960
961 /**
962 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
963 *
964 * @f: open file handle
965 * @buf: User buffer to store read data in
966 * @size: Number of bytes to read
967 * @pos: Offset to seek to
968 *
969 * The offset is treated as the BYTE address of one of the sensors
970 * enumerated in amd/include/kgd_pp_interface.h under the
971 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
972 * you would use the offset 3 * 4 = 12.
973 */
amdgpu_debugfs_sensor_read(struct file * f,char __user * buf,size_t size,loff_t * pos)974 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
975 size_t size, loff_t *pos)
976 {
977 struct amdgpu_device *adev = file_inode(f)->i_private;
978 int idx, x, outsize, r, valuesize;
979 uint32_t values[16];
980
981 if (size & 3 || *pos & 0x3)
982 return -EINVAL;
983
984 if (!adev->pm.dpm_enabled)
985 return -EINVAL;
986
987 /* convert offset to sensor number */
988 idx = *pos >> 2;
989
990 valuesize = sizeof(values);
991
992 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
993 if (r < 0) {
994 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
995 return r;
996 }
997
998 r = amdgpu_virt_enable_access_debugfs(adev);
999 if (r < 0) {
1000 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1001 return r;
1002 }
1003
1004 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
1005
1006 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1007 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1008
1009 if (r) {
1010 amdgpu_virt_disable_access_debugfs(adev);
1011 return r;
1012 }
1013
1014 if (size > valuesize) {
1015 amdgpu_virt_disable_access_debugfs(adev);
1016 return -EINVAL;
1017 }
1018
1019 outsize = 0;
1020 x = 0;
1021 if (!r) {
1022 while (size) {
1023 r = put_user(values[x++], (int32_t *)buf);
1024 buf += 4;
1025 size -= 4;
1026 outsize += 4;
1027 }
1028 }
1029
1030 amdgpu_virt_disable_access_debugfs(adev);
1031 return !r ? outsize : r;
1032 }
1033
1034 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1035 *
1036 * @f: open file handle
1037 * @buf: User buffer to store read data in
1038 * @size: Number of bytes to read
1039 * @pos: Offset to seek to
1040 *
1041 * The offset being sought changes which wave that the status data
1042 * will be returned for. The bits are used as follows:
1043 *
1044 * Bits 0..6: Byte offset into data
1045 * Bits 7..14: SE selector
1046 * Bits 15..22: SH/SA selector
1047 * Bits 23..30: CU/{WGP+SIMD} selector
1048 * Bits 31..36: WAVE ID selector
1049 * Bits 37..44: SIMD ID selector
1050 *
1051 * The returned data begins with one DWORD of version information
1052 * Followed by WAVE STATUS registers relevant to the GFX IP version
1053 * being used. See gfx_v8_0_read_wave_data() for an example output.
1054 */
amdgpu_debugfs_wave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1055 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1056 size_t size, loff_t *pos)
1057 {
1058 struct amdgpu_device *adev = f->f_inode->i_private;
1059 int r, x;
1060 ssize_t result = 0;
1061 uint32_t offset, se, sh, cu, wave, simd, data[32];
1062
1063 if (size & 3 || *pos & 3)
1064 return -EINVAL;
1065
1066 /* decode offset */
1067 offset = (*pos & GENMASK_ULL(6, 0));
1068 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1069 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1070 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1071 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1072 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1073
1074 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1075 if (r < 0) {
1076 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1077 return r;
1078 }
1079
1080 r = amdgpu_virt_enable_access_debugfs(adev);
1081 if (r < 0) {
1082 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1083 return r;
1084 }
1085
1086 /* switch to the specific se/sh/cu */
1087 mutex_lock(&adev->grbm_idx_mutex);
1088 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1089
1090 x = 0;
1091 if (adev->gfx.funcs->read_wave_data)
1092 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1093
1094 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1095 mutex_unlock(&adev->grbm_idx_mutex);
1096
1097 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1098 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1099
1100 if (!x) {
1101 amdgpu_virt_disable_access_debugfs(adev);
1102 return -EINVAL;
1103 }
1104
1105 while (size && (offset < x * 4)) {
1106 uint32_t value;
1107
1108 value = data[offset >> 2];
1109 r = put_user(value, (uint32_t *)buf);
1110 if (r) {
1111 amdgpu_virt_disable_access_debugfs(adev);
1112 return r;
1113 }
1114
1115 result += 4;
1116 buf += 4;
1117 offset += 4;
1118 size -= 4;
1119 }
1120
1121 amdgpu_virt_disable_access_debugfs(adev);
1122 return result;
1123 }
1124
1125 /** amdgpu_debugfs_gpr_read - Read wave gprs
1126 *
1127 * @f: open file handle
1128 * @buf: User buffer to store read data in
1129 * @size: Number of bytes to read
1130 * @pos: Offset to seek to
1131 *
1132 * The offset being sought changes which wave that the status data
1133 * will be returned for. The bits are used as follows:
1134 *
1135 * Bits 0..11: Byte offset into data
1136 * Bits 12..19: SE selector
1137 * Bits 20..27: SH/SA selector
1138 * Bits 28..35: CU/{WGP+SIMD} selector
1139 * Bits 36..43: WAVE ID selector
1140 * Bits 37..44: SIMD ID selector
1141 * Bits 52..59: Thread selector
1142 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1143 *
1144 * The return data comes from the SGPR or VGPR register bank for
1145 * the selected operational unit.
1146 */
amdgpu_debugfs_gpr_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1147 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1148 size_t size, loff_t *pos)
1149 {
1150 struct amdgpu_device *adev = f->f_inode->i_private;
1151 int r;
1152 ssize_t result = 0;
1153 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1154
1155 if (size > 4096 || size & 3 || *pos & 3)
1156 return -EINVAL;
1157
1158 /* decode offset */
1159 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1160 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1161 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1162 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1163 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1164 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1165 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1166 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1167
1168 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1169 if (!data)
1170 return -ENOMEM;
1171
1172 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1173 if (r < 0)
1174 goto err;
1175
1176 r = amdgpu_virt_enable_access_debugfs(adev);
1177 if (r < 0)
1178 goto err;
1179
1180 /* switch to the specific se/sh/cu */
1181 mutex_lock(&adev->grbm_idx_mutex);
1182 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1183
1184 if (bank == 0) {
1185 if (adev->gfx.funcs->read_wave_vgprs)
1186 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1187 } else {
1188 if (adev->gfx.funcs->read_wave_sgprs)
1189 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1190 }
1191
1192 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1193 mutex_unlock(&adev->grbm_idx_mutex);
1194
1195 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1196 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1197
1198 while (size) {
1199 uint32_t value;
1200
1201 value = data[result >> 2];
1202 r = put_user(value, (uint32_t *)buf);
1203 if (r) {
1204 amdgpu_virt_disable_access_debugfs(adev);
1205 goto err;
1206 }
1207
1208 result += 4;
1209 buf += 4;
1210 size -= 4;
1211 }
1212
1213 kfree(data);
1214 amdgpu_virt_disable_access_debugfs(adev);
1215 return result;
1216
1217 err:
1218 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1219 kfree(data);
1220 return r;
1221 }
1222
1223 /**
1224 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1225 *
1226 * @f: open file handle
1227 * @buf: User buffer to store read data in
1228 * @size: Number of bytes to read
1229 * @pos: Offset to seek to
1230 *
1231 * Read the last residency value logged. It doesn't auto update, one needs to
1232 * stop logging before getting the current value.
1233 */
amdgpu_debugfs_gfxoff_residency_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1234 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1235 size_t size, loff_t *pos)
1236 {
1237 struct amdgpu_device *adev = file_inode(f)->i_private;
1238 ssize_t result = 0;
1239 int r;
1240
1241 if (size & 0x3 || *pos & 0x3)
1242 return -EINVAL;
1243
1244 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1245 if (r < 0) {
1246 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1247 return r;
1248 }
1249
1250 while (size) {
1251 uint32_t value;
1252
1253 r = amdgpu_get_gfx_off_residency(adev, &value);
1254 if (r)
1255 goto out;
1256
1257 r = put_user(value, (uint32_t *)buf);
1258 if (r)
1259 goto out;
1260
1261 result += 4;
1262 buf += 4;
1263 *pos += 4;
1264 size -= 4;
1265 }
1266
1267 r = result;
1268 out:
1269 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1270 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1271
1272 return r;
1273 }
1274
1275 /**
1276 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1277 *
1278 * @f: open file handle
1279 * @buf: User buffer to write data from
1280 * @size: Number of bytes to write
1281 * @pos: Offset to seek to
1282 *
1283 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1284 */
amdgpu_debugfs_gfxoff_residency_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1285 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1286 size_t size, loff_t *pos)
1287 {
1288 struct amdgpu_device *adev = file_inode(f)->i_private;
1289 ssize_t result = 0;
1290 int r;
1291
1292 if (size & 0x3 || *pos & 0x3)
1293 return -EINVAL;
1294
1295 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1296 if (r < 0) {
1297 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1298 return r;
1299 }
1300
1301 while (size) {
1302 u32 value;
1303
1304 r = get_user(value, (uint32_t *)buf);
1305 if (r)
1306 goto out;
1307
1308 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1309
1310 result += 4;
1311 buf += 4;
1312 *pos += 4;
1313 size -= 4;
1314 }
1315
1316 r = result;
1317 out:
1318 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1319 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1320
1321 return r;
1322 }
1323
1324
1325 /**
1326 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1327 *
1328 * @f: open file handle
1329 * @buf: User buffer to store read data in
1330 * @size: Number of bytes to read
1331 * @pos: Offset to seek to
1332 */
amdgpu_debugfs_gfxoff_count_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1333 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1334 size_t size, loff_t *pos)
1335 {
1336 struct amdgpu_device *adev = file_inode(f)->i_private;
1337 ssize_t result = 0;
1338 int r;
1339
1340 if (size & 0x3 || *pos & 0x3)
1341 return -EINVAL;
1342
1343 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1344 if (r < 0) {
1345 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1346 return r;
1347 }
1348
1349 while (size) {
1350 u64 value = 0;
1351
1352 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1353 if (r)
1354 goto out;
1355
1356 r = put_user(value, (u64 *)buf);
1357 if (r)
1358 goto out;
1359
1360 result += 4;
1361 buf += 4;
1362 *pos += 4;
1363 size -= 4;
1364 }
1365
1366 r = result;
1367 out:
1368 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1369 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1370
1371 return r;
1372 }
1373
1374 /**
1375 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1376 *
1377 * @f: open file handle
1378 * @buf: User buffer to write data from
1379 * @size: Number of bytes to write
1380 * @pos: Offset to seek to
1381 *
1382 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1383 */
amdgpu_debugfs_gfxoff_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1384 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1385 size_t size, loff_t *pos)
1386 {
1387 struct amdgpu_device *adev = file_inode(f)->i_private;
1388 ssize_t result = 0;
1389 int r;
1390
1391 if (size & 0x3 || *pos & 0x3)
1392 return -EINVAL;
1393
1394 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1395 if (r < 0) {
1396 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1397 return r;
1398 }
1399
1400 while (size) {
1401 uint32_t value;
1402
1403 r = get_user(value, (uint32_t *)buf);
1404 if (r)
1405 goto out;
1406
1407 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1408
1409 result += 4;
1410 buf += 4;
1411 *pos += 4;
1412 size -= 4;
1413 }
1414
1415 r = result;
1416 out:
1417 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1418 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1419
1420 return r;
1421 }
1422
1423
1424 /**
1425 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1426 *
1427 * @f: open file handle
1428 * @buf: User buffer to store read data in
1429 * @size: Number of bytes to read
1430 * @pos: Offset to seek to
1431 */
amdgpu_debugfs_gfxoff_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1432 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1433 size_t size, loff_t *pos)
1434 {
1435 struct amdgpu_device *adev = file_inode(f)->i_private;
1436 ssize_t result = 0;
1437 int r;
1438
1439 if (size & 0x3 || *pos & 0x3)
1440 return -EINVAL;
1441
1442 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1443 if (r < 0) {
1444 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1445 return r;
1446 }
1447
1448 while (size) {
1449 u32 value = adev->gfx.gfx_off_state;
1450
1451 r = put_user(value, (u32 *)buf);
1452 if (r)
1453 goto out;
1454
1455 result += 4;
1456 buf += 4;
1457 *pos += 4;
1458 size -= 4;
1459 }
1460
1461 r = result;
1462 out:
1463 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1464 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1465
1466 return r;
1467 }
1468
amdgpu_debugfs_gfxoff_status_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1469 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1470 size_t size, loff_t *pos)
1471 {
1472 struct amdgpu_device *adev = file_inode(f)->i_private;
1473 ssize_t result = 0;
1474 int r;
1475
1476 if (size & 0x3 || *pos & 0x3)
1477 return -EINVAL;
1478
1479 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1480 if (r < 0) {
1481 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1482 return r;
1483 }
1484
1485 while (size) {
1486 u32 value;
1487
1488 r = amdgpu_get_gfx_off_status(adev, &value);
1489 if (r)
1490 goto out;
1491
1492 r = put_user(value, (u32 *)buf);
1493 if (r)
1494 goto out;
1495
1496 result += 4;
1497 buf += 4;
1498 *pos += 4;
1499 size -= 4;
1500 }
1501
1502 r = result;
1503 out:
1504 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1505 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1506
1507 return r;
1508 }
1509
1510 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1511 .owner = THIS_MODULE,
1512 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1513 .read = amdgpu_debugfs_regs2_read,
1514 .write = amdgpu_debugfs_regs2_write,
1515 .open = amdgpu_debugfs_regs2_open,
1516 .release = amdgpu_debugfs_regs2_release,
1517 .llseek = default_llseek
1518 };
1519
1520 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1521 .owner = THIS_MODULE,
1522 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1523 .read = amdgpu_debugfs_gprwave_read,
1524 .open = amdgpu_debugfs_gprwave_open,
1525 .release = amdgpu_debugfs_gprwave_release,
1526 .llseek = default_llseek
1527 };
1528
1529 static const struct file_operations amdgpu_debugfs_regs_fops = {
1530 .owner = THIS_MODULE,
1531 .read = amdgpu_debugfs_regs_read,
1532 .write = amdgpu_debugfs_regs_write,
1533 .llseek = default_llseek
1534 };
1535 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1536 .owner = THIS_MODULE,
1537 .read = amdgpu_debugfs_regs_didt_read,
1538 .write = amdgpu_debugfs_regs_didt_write,
1539 .llseek = default_llseek
1540 };
1541 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1542 .owner = THIS_MODULE,
1543 .read = amdgpu_debugfs_regs_pcie_read,
1544 .write = amdgpu_debugfs_regs_pcie_write,
1545 .llseek = default_llseek
1546 };
1547 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1548 .owner = THIS_MODULE,
1549 .read = amdgpu_debugfs_regs_smc_read,
1550 .write = amdgpu_debugfs_regs_smc_write,
1551 .llseek = default_llseek
1552 };
1553
1554 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1555 .owner = THIS_MODULE,
1556 .read = amdgpu_debugfs_gca_config_read,
1557 .llseek = default_llseek
1558 };
1559
1560 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1561 .owner = THIS_MODULE,
1562 .read = amdgpu_debugfs_sensor_read,
1563 .llseek = default_llseek
1564 };
1565
1566 static const struct file_operations amdgpu_debugfs_wave_fops = {
1567 .owner = THIS_MODULE,
1568 .read = amdgpu_debugfs_wave_read,
1569 .llseek = default_llseek
1570 };
1571 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1572 .owner = THIS_MODULE,
1573 .read = amdgpu_debugfs_gpr_read,
1574 .llseek = default_llseek
1575 };
1576
1577 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1578 .owner = THIS_MODULE,
1579 .read = amdgpu_debugfs_gfxoff_read,
1580 .write = amdgpu_debugfs_gfxoff_write,
1581 .llseek = default_llseek
1582 };
1583
1584 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1585 .owner = THIS_MODULE,
1586 .read = amdgpu_debugfs_gfxoff_status_read,
1587 .llseek = default_llseek
1588 };
1589
1590 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1591 .owner = THIS_MODULE,
1592 .read = amdgpu_debugfs_gfxoff_count_read,
1593 .llseek = default_llseek
1594 };
1595
1596 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1597 .owner = THIS_MODULE,
1598 .read = amdgpu_debugfs_gfxoff_residency_read,
1599 .write = amdgpu_debugfs_gfxoff_residency_write,
1600 .llseek = default_llseek
1601 };
1602
1603 static const struct file_operations *debugfs_regs[] = {
1604 &amdgpu_debugfs_regs_fops,
1605 &amdgpu_debugfs_regs2_fops,
1606 &amdgpu_debugfs_gprwave_fops,
1607 &amdgpu_debugfs_regs_didt_fops,
1608 &amdgpu_debugfs_regs_pcie_fops,
1609 &amdgpu_debugfs_regs_smc_fops,
1610 &amdgpu_debugfs_gca_config_fops,
1611 &amdgpu_debugfs_sensors_fops,
1612 &amdgpu_debugfs_wave_fops,
1613 &amdgpu_debugfs_gpr_fops,
1614 &amdgpu_debugfs_gfxoff_fops,
1615 &amdgpu_debugfs_gfxoff_status_fops,
1616 &amdgpu_debugfs_gfxoff_count_fops,
1617 &amdgpu_debugfs_gfxoff_residency_fops,
1618 };
1619
1620 static const char * const debugfs_regs_names[] = {
1621 "amdgpu_regs",
1622 "amdgpu_regs2",
1623 "amdgpu_gprwave",
1624 "amdgpu_regs_didt",
1625 "amdgpu_regs_pcie",
1626 "amdgpu_regs_smc",
1627 "amdgpu_gca_config",
1628 "amdgpu_sensors",
1629 "amdgpu_wave",
1630 "amdgpu_gpr",
1631 "amdgpu_gfxoff",
1632 "amdgpu_gfxoff_status",
1633 "amdgpu_gfxoff_count",
1634 "amdgpu_gfxoff_residency",
1635 };
1636
1637 /**
1638 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1639 * register access.
1640 *
1641 * @adev: The device to attach the debugfs entries to
1642 */
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)1643 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1644 {
1645 struct drm_minor *minor = adev_to_drm(adev)->primary;
1646 struct dentry *ent, *root = minor->debugfs_root;
1647 unsigned int i;
1648
1649 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1650 ent = debugfs_create_file(debugfs_regs_names[i],
1651 S_IFREG | 0444, root,
1652 adev, debugfs_regs[i]);
1653 if (!i && !IS_ERR_OR_NULL(ent))
1654 i_size_write(ent->d_inode, adev->rmmio_size);
1655 }
1656
1657 return 0;
1658 }
1659
amdgpu_debugfs_test_ib_show(struct seq_file * m,void * unused)1660 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1661 {
1662 struct amdgpu_device *adev = m->private;
1663 struct drm_device *dev = adev_to_drm(adev);
1664 int r = 0, i;
1665
1666 r = pm_runtime_get_sync(dev->dev);
1667 if (r < 0) {
1668 pm_runtime_put_autosuspend(dev->dev);
1669 return r;
1670 }
1671
1672 /* Avoid accidently unparking the sched thread during GPU reset */
1673 r = down_write_killable(&adev->reset_domain->sem);
1674 if (r)
1675 return r;
1676
1677 /* hold on the scheduler */
1678 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1679 struct amdgpu_ring *ring = adev->rings[i];
1680
1681 if (!amdgpu_ring_sched_ready(ring))
1682 continue;
1683 drm_sched_wqueue_stop(&ring->sched);
1684 }
1685
1686 seq_puts(m, "run ib test:\n");
1687 r = amdgpu_ib_ring_tests(adev);
1688 if (r)
1689 seq_printf(m, "ib ring tests failed (%d).\n", r);
1690 else
1691 seq_puts(m, "ib ring tests passed.\n");
1692
1693 /* go on the scheduler */
1694 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1695 struct amdgpu_ring *ring = adev->rings[i];
1696
1697 if (!amdgpu_ring_sched_ready(ring))
1698 continue;
1699 drm_sched_wqueue_start(&ring->sched);
1700 }
1701
1702 up_write(&adev->reset_domain->sem);
1703
1704 pm_runtime_mark_last_busy(dev->dev);
1705 pm_runtime_put_autosuspend(dev->dev);
1706
1707 return 0;
1708 }
1709
amdgpu_debugfs_evict_vram(void * data,u64 * val)1710 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1711 {
1712 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1713 struct drm_device *dev = adev_to_drm(adev);
1714 int r;
1715
1716 r = pm_runtime_get_sync(dev->dev);
1717 if (r < 0) {
1718 pm_runtime_put_autosuspend(dev->dev);
1719 return r;
1720 }
1721
1722 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1723
1724 pm_runtime_mark_last_busy(dev->dev);
1725 pm_runtime_put_autosuspend(dev->dev);
1726
1727 return 0;
1728 }
1729
1730
amdgpu_debugfs_evict_gtt(void * data,u64 * val)1731 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1732 {
1733 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1734 struct drm_device *dev = adev_to_drm(adev);
1735 int r;
1736
1737 r = pm_runtime_get_sync(dev->dev);
1738 if (r < 0) {
1739 pm_runtime_put_autosuspend(dev->dev);
1740 return r;
1741 }
1742
1743 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1744
1745 pm_runtime_mark_last_busy(dev->dev);
1746 pm_runtime_put_autosuspend(dev->dev);
1747
1748 return 0;
1749 }
1750
amdgpu_debugfs_benchmark(void * data,u64 val)1751 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1752 {
1753 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1754 struct drm_device *dev = adev_to_drm(adev);
1755 int r;
1756
1757 r = pm_runtime_get_sync(dev->dev);
1758 if (r < 0) {
1759 pm_runtime_put_autosuspend(dev->dev);
1760 return r;
1761 }
1762
1763 r = amdgpu_benchmark(adev, val);
1764
1765 pm_runtime_mark_last_busy(dev->dev);
1766 pm_runtime_put_autosuspend(dev->dev);
1767
1768 return r;
1769 }
1770
amdgpu_debugfs_vm_info_show(struct seq_file * m,void * unused)1771 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1772 {
1773 struct amdgpu_device *adev = m->private;
1774 struct drm_device *dev = adev_to_drm(adev);
1775 struct drm_file *file;
1776 int r;
1777
1778 r = mutex_lock_interruptible(&dev->filelist_mutex);
1779 if (r)
1780 return r;
1781
1782 list_for_each_entry(file, &dev->filelist, lhead) {
1783 struct amdgpu_fpriv *fpriv = file->driver_priv;
1784 struct amdgpu_vm *vm = &fpriv->vm;
1785 struct amdgpu_task_info *ti;
1786
1787 ti = amdgpu_vm_get_task_info_vm(vm);
1788 if (ti) {
1789 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->pid, ti->process_name);
1790 amdgpu_vm_put_task_info(ti);
1791 }
1792
1793 r = amdgpu_bo_reserve(vm->root.bo, true);
1794 if (r)
1795 break;
1796 amdgpu_debugfs_vm_bo_info(vm, m);
1797 amdgpu_bo_unreserve(vm->root.bo);
1798 }
1799
1800 mutex_unlock(&dev->filelist_mutex);
1801
1802 return r;
1803 }
1804
1805 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1806 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1807 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1808 NULL, "%lld\n");
1809 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1810 NULL, "%lld\n");
1811 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1812 "%lld\n");
1813
amdgpu_ib_preempt_fences_swap(struct amdgpu_ring * ring,struct dma_fence ** fences)1814 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1815 struct dma_fence **fences)
1816 {
1817 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1818 uint32_t sync_seq, last_seq;
1819
1820 last_seq = atomic_read(&ring->fence_drv.last_seq);
1821 sync_seq = ring->fence_drv.sync_seq;
1822
1823 last_seq &= drv->num_fences_mask;
1824 sync_seq &= drv->num_fences_mask;
1825
1826 do {
1827 struct dma_fence *fence, **ptr;
1828
1829 ++last_seq;
1830 last_seq &= drv->num_fences_mask;
1831 ptr = &drv->fences[last_seq];
1832
1833 fence = rcu_dereference_protected(*ptr, 1);
1834 RCU_INIT_POINTER(*ptr, NULL);
1835
1836 if (!fence)
1837 continue;
1838
1839 fences[last_seq] = fence;
1840
1841 } while (last_seq != sync_seq);
1842 }
1843
amdgpu_ib_preempt_signal_fences(struct dma_fence ** fences,int length)1844 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1845 int length)
1846 {
1847 int i;
1848 struct dma_fence *fence;
1849
1850 for (i = 0; i < length; i++) {
1851 fence = fences[i];
1852 if (!fence)
1853 continue;
1854 dma_fence_signal(fence);
1855 dma_fence_put(fence);
1856 }
1857 }
1858
amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler * sched)1859 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1860 {
1861 struct drm_sched_job *s_job;
1862 struct dma_fence *fence;
1863
1864 spin_lock(&sched->job_list_lock);
1865 list_for_each_entry(s_job, &sched->pending_list, list) {
1866 fence = sched->ops->run_job(s_job);
1867 dma_fence_put(fence);
1868 }
1869 spin_unlock(&sched->job_list_lock);
1870 }
1871
amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring * ring)1872 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1873 {
1874 struct amdgpu_job *job;
1875 struct drm_sched_job *s_job, *tmp;
1876 uint32_t preempt_seq;
1877 struct dma_fence *fence, **ptr;
1878 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1879 struct drm_gpu_scheduler *sched = &ring->sched;
1880 bool preempted = true;
1881
1882 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1883 return;
1884
1885 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1886 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1887 preempted = false;
1888 goto no_preempt;
1889 }
1890
1891 preempt_seq &= drv->num_fences_mask;
1892 ptr = &drv->fences[preempt_seq];
1893 fence = rcu_dereference_protected(*ptr, 1);
1894
1895 no_preempt:
1896 spin_lock(&sched->job_list_lock);
1897 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1898 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1899 /* remove job from ring_mirror_list */
1900 list_del_init(&s_job->list);
1901 sched->ops->free_job(s_job);
1902 continue;
1903 }
1904 job = to_amdgpu_job(s_job);
1905 if (preempted && (&job->hw_fence) == fence)
1906 /* mark the job as preempted */
1907 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1908 }
1909 spin_unlock(&sched->job_list_lock);
1910 }
1911
amdgpu_debugfs_ib_preempt(void * data,u64 val)1912 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1913 {
1914 int r, length;
1915 struct amdgpu_ring *ring;
1916 struct dma_fence **fences = NULL;
1917 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1918
1919 if (val >= AMDGPU_MAX_RINGS)
1920 return -EINVAL;
1921
1922 ring = adev->rings[val];
1923
1924 if (!amdgpu_ring_sched_ready(ring) ||
1925 !ring->funcs->preempt_ib)
1926 return -EINVAL;
1927
1928 /* the last preemption failed */
1929 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1930 return -EBUSY;
1931
1932 length = ring->fence_drv.num_fences_mask + 1;
1933 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1934 if (!fences)
1935 return -ENOMEM;
1936
1937 /* Avoid accidently unparking the sched thread during GPU reset */
1938 r = down_read_killable(&adev->reset_domain->sem);
1939 if (r)
1940 goto pro_end;
1941
1942 /* stop the scheduler */
1943 drm_sched_wqueue_stop(&ring->sched);
1944
1945 /* preempt the IB */
1946 r = amdgpu_ring_preempt_ib(ring);
1947 if (r) {
1948 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1949 goto failure;
1950 }
1951
1952 amdgpu_fence_process(ring);
1953
1954 if (atomic_read(&ring->fence_drv.last_seq) !=
1955 ring->fence_drv.sync_seq) {
1956 DRM_INFO("ring %d was preempted\n", ring->idx);
1957
1958 amdgpu_ib_preempt_mark_partial_job(ring);
1959
1960 /* swap out the old fences */
1961 amdgpu_ib_preempt_fences_swap(ring, fences);
1962
1963 amdgpu_fence_driver_force_completion(ring);
1964
1965 /* resubmit unfinished jobs */
1966 amdgpu_ib_preempt_job_recovery(&ring->sched);
1967
1968 /* wait for jobs finished */
1969 amdgpu_fence_wait_empty(ring);
1970
1971 /* signal the old fences */
1972 amdgpu_ib_preempt_signal_fences(fences, length);
1973 }
1974
1975 failure:
1976 /* restart the scheduler */
1977 drm_sched_wqueue_start(&ring->sched);
1978
1979 up_read(&adev->reset_domain->sem);
1980
1981 pro_end:
1982 kfree(fences);
1983
1984 return r;
1985 }
1986
amdgpu_debugfs_sclk_set(void * data,u64 val)1987 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1988 {
1989 int ret = 0;
1990 uint32_t max_freq, min_freq;
1991 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1992
1993 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1994 return -EINVAL;
1995
1996 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1997 if (ret < 0) {
1998 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1999 return ret;
2000 }
2001
2002 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
2003 if (ret == -EOPNOTSUPP) {
2004 ret = 0;
2005 goto out;
2006 }
2007 if (ret || val > max_freq || val < min_freq) {
2008 ret = -EINVAL;
2009 goto out;
2010 }
2011
2012 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2013 if (ret)
2014 ret = -EINVAL;
2015
2016 out:
2017 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
2018 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2019
2020 return ret;
2021 }
2022
2023 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2024 amdgpu_debugfs_ib_preempt, "%llu\n");
2025
2026 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2027 amdgpu_debugfs_sclk_set, "%llu\n");
2028
amdgpu_debugfs_init(struct amdgpu_device * adev)2029 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2030 {
2031 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2032 struct dentry *ent;
2033 int r, i;
2034
2035 if (!debugfs_initialized())
2036 return 0;
2037
2038 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2039 &adev->pm.smu_debug_mask);
2040
2041 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2042 &fops_ib_preempt);
2043 if (IS_ERR(ent)) {
2044 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2045 return PTR_ERR(ent);
2046 }
2047
2048 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2049 &fops_sclk_set);
2050 if (IS_ERR(ent)) {
2051 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2052 return PTR_ERR(ent);
2053 }
2054
2055 /* Register debugfs entries for amdgpu_ttm */
2056 amdgpu_ttm_debugfs_init(adev);
2057 amdgpu_debugfs_pm_init(adev);
2058 amdgpu_debugfs_sa_init(adev);
2059 amdgpu_debugfs_fence_init(adev);
2060 amdgpu_debugfs_gem_init(adev);
2061
2062 r = amdgpu_debugfs_regs_init(adev);
2063 if (r)
2064 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2065
2066 amdgpu_debugfs_firmware_init(adev);
2067 amdgpu_ta_if_debugfs_init(adev);
2068
2069 amdgpu_debugfs_mes_event_log_init(adev);
2070
2071 #if defined(CONFIG_DRM_AMD_DC)
2072 if (adev->dc_enabled)
2073 dtn_debugfs_init(adev);
2074 #endif
2075
2076 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2077 struct amdgpu_ring *ring = adev->rings[i];
2078
2079 if (!ring)
2080 continue;
2081
2082 amdgpu_debugfs_ring_init(adev, ring);
2083 }
2084
2085 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2086 if (!amdgpu_vcnfw_log)
2087 break;
2088
2089 if (adev->vcn.harvest_config & (1 << i))
2090 continue;
2091
2092 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2093 }
2094
2095 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog)
2096 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm);
2097
2098 amdgpu_ras_debugfs_create_all(adev);
2099 amdgpu_rap_debugfs_init(adev);
2100 amdgpu_securedisplay_debugfs_init(adev);
2101 amdgpu_fw_attestation_debugfs_init(adev);
2102
2103 debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2104 &amdgpu_evict_vram_fops);
2105 debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2106 &amdgpu_evict_gtt_fops);
2107 debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2108 &amdgpu_debugfs_test_ib_fops);
2109 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2110 &amdgpu_debugfs_vm_info_fops);
2111 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2112 &amdgpu_benchmark_fops);
2113
2114 adev->debugfs_vbios_blob.data = adev->bios;
2115 adev->debugfs_vbios_blob.size = adev->bios_size;
2116 debugfs_create_blob("amdgpu_vbios", 0444, root,
2117 &adev->debugfs_vbios_blob);
2118
2119 adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2120 adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2121 debugfs_create_blob("amdgpu_discovery", 0444, root,
2122 &adev->debugfs_discovery_blob);
2123
2124 return 0;
2125 }
2126
2127 #else
amdgpu_debugfs_init(struct amdgpu_device * adev)2128 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2129 {
2130 return 0;
2131 }
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)2132 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2133 {
2134 return 0;
2135 }
2136 #endif
2137