1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/security.h>
30 #include <linux/pm_runtime.h>
31
32 #include "amdgpu.h"
33 #include "amdgpu_pm.h"
34 #include "amdgpu_dm_debugfs.h"
35 #include "amdgpu_ras.h"
36 #include "amdgpu_rap.h"
37 #include "amdgpu_securedisplay.h"
38 #include "amdgpu_fw_attestation.h"
39 #include "amdgpu_umr.h"
40
41 #include "amdgpu_reset.h"
42 #include "amdgpu_psp_ta.h"
43 #include "amdgpu_userq.h"
44
45 #if defined(CONFIG_DEBUG_FS)
46
47 /* Encode milliwatts in the raw Q24.8 sensor report format used by UMR. */
48 #define AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(power_mw) \
49 DIV_ROUND_CLOSEST_ULL((u64)(power_mw) * BIT(8), \
50 MILLIWATT_PER_WATT)
51
52 /**
53 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
54 *
55 * @read: True if reading
56 * @f: open file handle
57 * @buf: User buffer to write/read to
58 * @size: Number of bytes to write/read
59 * @pos: Offset to seek to
60 *
61 * This debugfs entry has special meaning on the offset being sought.
62 * Various bits have different meanings:
63 *
64 * Bit 62: Indicates a GRBM bank switch is needed
65 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
66 * zero)
67 * Bits 24..33: The SE or ME selector if needed
68 * Bits 34..43: The SH (or SA) or PIPE selector if needed
69 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
70 *
71 * Bit 23: Indicates that the PM power gating lock should be held
72 * This is necessary to read registers that might be
73 * unreliable during a power gating transistion.
74 *
75 * The lower bits are the BYTE offset of the register to read. This
76 * allows reading multiple registers in a single call and having
77 * the returned size reflect that.
78 */
amdgpu_debugfs_process_reg_op(bool read,struct file * f,char __user * buf,size_t size,loff_t * pos)79 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
80 char __user *buf, size_t size, loff_t *pos)
81 {
82 struct amdgpu_device *adev = file_inode(f)->i_private;
83 ssize_t result = 0;
84 int r;
85 bool pm_pg_lock, use_bank, use_ring;
86 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
87
88 pm_pg_lock = use_bank = use_ring = false;
89 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
90
91 if (size & 0x3 || *pos & 0x3 ||
92 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
93 return -EINVAL;
94
95 /* are we reading registers for which a PG lock is necessary? */
96 pm_pg_lock = (*pos >> 23) & 1;
97
98 if (*pos & (1ULL << 62)) {
99 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
100 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
101 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
102
103 if (se_bank == 0x3FF)
104 se_bank = 0xFFFFFFFF;
105 if (sh_bank == 0x3FF)
106 sh_bank = 0xFFFFFFFF;
107 if (instance_bank == 0x3FF)
108 instance_bank = 0xFFFFFFFF;
109 use_bank = true;
110 } else if (*pos & (1ULL << 61)) {
111
112 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
113 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
114 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
115 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
116
117 use_ring = true;
118 } else {
119 use_bank = use_ring = false;
120 }
121
122 *pos &= (1UL << 22) - 1;
123
124 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
125 if (r < 0) {
126 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
127 return r;
128 }
129
130 r = amdgpu_virt_enable_access_debugfs(adev);
131 if (r < 0) {
132 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
133 return r;
134 }
135
136 if (use_bank) {
137 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
138 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
139 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
140 amdgpu_virt_disable_access_debugfs(adev);
141 return -EINVAL;
142 }
143 mutex_lock(&adev->grbm_idx_mutex);
144 amdgpu_gfx_select_se_sh(adev, se_bank,
145 sh_bank, instance_bank, 0);
146 } else if (use_ring) {
147 mutex_lock(&adev->srbm_mutex);
148 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
149 }
150
151 if (pm_pg_lock)
152 mutex_lock(&adev->pm.mutex);
153
154 while (size) {
155 uint32_t value;
156
157 if (read) {
158 value = RREG32(*pos >> 2);
159 r = put_user(value, (uint32_t *)buf);
160 } else {
161 r = get_user(value, (uint32_t *)buf);
162 if (!r)
163 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
164 }
165 if (r) {
166 result = r;
167 goto end;
168 }
169
170 result += 4;
171 buf += 4;
172 *pos += 4;
173 size -= 4;
174 }
175
176 end:
177 if (use_bank) {
178 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
179 mutex_unlock(&adev->grbm_idx_mutex);
180 } else if (use_ring) {
181 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
182 mutex_unlock(&adev->srbm_mutex);
183 }
184
185 if (pm_pg_lock)
186 mutex_unlock(&adev->pm.mutex);
187
188 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
189
190 amdgpu_virt_disable_access_debugfs(adev);
191 return result;
192 }
193
194 /*
195 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
196 */
amdgpu_debugfs_regs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)197 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
198 size_t size, loff_t *pos)
199 {
200 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
201 }
202
203 /*
204 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
205 */
amdgpu_debugfs_regs_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)206 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
207 size_t size, loff_t *pos)
208 {
209 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
210 }
211
amdgpu_debugfs_regs2_open(struct inode * inode,struct file * file)212 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
213 {
214 struct amdgpu_debugfs_regs2_data *rd;
215
216 rd = kzalloc_obj(*rd);
217 if (!rd)
218 return -ENOMEM;
219 rd->adev = file_inode(file)->i_private;
220 file->private_data = rd;
221 mutex_init(&rd->lock);
222
223 return 0;
224 }
225
amdgpu_debugfs_regs2_release(struct inode * inode,struct file * file)226 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
227 {
228 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
229
230 mutex_destroy(&rd->lock);
231 kfree(file->private_data);
232 return 0;
233 }
234
amdgpu_debugfs_regs2_op(struct file * f,char __user * buf,u32 offset,size_t size,int write_en)235 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
236 {
237 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
238 struct amdgpu_device *adev = rd->adev;
239 ssize_t result = 0;
240 int r;
241 uint32_t value;
242
243 if (size & 0x3 || offset & 0x3)
244 return -EINVAL;
245
246 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
247 if (r < 0) {
248 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
249 return r;
250 }
251
252 r = amdgpu_virt_enable_access_debugfs(adev);
253 if (r < 0) {
254 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
255 return r;
256 }
257
258 mutex_lock(&rd->lock);
259
260 if (rd->id.use_grbm) {
261 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
262 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
263 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
264 amdgpu_virt_disable_access_debugfs(adev);
265 mutex_unlock(&rd->lock);
266 return -EINVAL;
267 }
268 mutex_lock(&adev->grbm_idx_mutex);
269 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
270 rd->id.grbm.sh,
271 rd->id.grbm.instance, rd->id.xcc_id);
272 }
273
274 if (rd->id.use_srbm) {
275 mutex_lock(&adev->srbm_mutex);
276 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
277 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
278 }
279
280 if (rd->id.pg_lock)
281 mutex_lock(&adev->pm.mutex);
282
283 while (size) {
284 if (!write_en) {
285 value = RREG32(offset >> 2);
286 r = put_user(value, (uint32_t *)buf);
287 } else {
288 r = get_user(value, (uint32_t *)buf);
289 if (!r)
290 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
291 }
292 if (r) {
293 result = r;
294 goto end;
295 }
296 offset += 4;
297 size -= 4;
298 result += 4;
299 buf += 4;
300 }
301 end:
302 if (rd->id.use_grbm) {
303 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
304 mutex_unlock(&adev->grbm_idx_mutex);
305 }
306
307 if (rd->id.use_srbm) {
308 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
309 mutex_unlock(&adev->srbm_mutex);
310 }
311
312 if (rd->id.pg_lock)
313 mutex_unlock(&adev->pm.mutex);
314
315 mutex_unlock(&rd->lock);
316
317 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
318
319 amdgpu_virt_disable_access_debugfs(adev);
320 return result;
321 }
322
amdgpu_debugfs_regs2_ioctl(struct file * f,unsigned int cmd,unsigned long data)323 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
324 {
325 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
326 struct amdgpu_debugfs_regs2_iocdata v1_data;
327 int r;
328
329 mutex_lock(&rd->lock);
330
331 switch (cmd) {
332 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
333 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
334 sizeof(rd->id));
335 if (r)
336 r = -EINVAL;
337 goto done;
338 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
339 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
340 sizeof(v1_data));
341 if (r) {
342 r = -EINVAL;
343 goto done;
344 }
345 goto v1_copy;
346 default:
347 r = -EINVAL;
348 goto done;
349 }
350
351 v1_copy:
352 rd->id.use_srbm = v1_data.use_srbm;
353 rd->id.use_grbm = v1_data.use_grbm;
354 rd->id.pg_lock = v1_data.pg_lock;
355 rd->id.grbm.se = v1_data.grbm.se;
356 rd->id.grbm.sh = v1_data.grbm.sh;
357 rd->id.grbm.instance = v1_data.grbm.instance;
358 rd->id.srbm.me = v1_data.srbm.me;
359 rd->id.srbm.pipe = v1_data.srbm.pipe;
360 rd->id.srbm.queue = v1_data.srbm.queue;
361 rd->id.xcc_id = 0;
362 done:
363 mutex_unlock(&rd->lock);
364 return r;
365 }
366
amdgpu_debugfs_regs2_read(struct file * f,char __user * buf,size_t size,loff_t * pos)367 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
368 {
369 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
370 }
371
amdgpu_debugfs_regs2_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)372 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
373 {
374 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
375 }
376
amdgpu_debugfs_gprwave_open(struct inode * inode,struct file * file)377 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
378 {
379 struct amdgpu_debugfs_gprwave_data *rd;
380
381 rd = kzalloc_obj(*rd);
382 if (!rd)
383 return -ENOMEM;
384 rd->adev = file_inode(file)->i_private;
385 file->private_data = rd;
386 mutex_init(&rd->lock);
387
388 return 0;
389 }
390
amdgpu_debugfs_gprwave_release(struct inode * inode,struct file * file)391 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
392 {
393 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
394
395 mutex_destroy(&rd->lock);
396 kfree(file->private_data);
397 return 0;
398 }
399
amdgpu_debugfs_gprwave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)400 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
401 {
402 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
403 struct amdgpu_device *adev = rd->adev;
404 ssize_t result = 0;
405 int r;
406 uint32_t *data, x;
407
408 if (size > 4096 || size & 0x3 || *pos & 0x3)
409 return -EINVAL;
410
411 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
412 if (r < 0) {
413 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
414 return r;
415 }
416
417 r = amdgpu_virt_enable_access_debugfs(adev);
418 if (r < 0) {
419 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
420 return r;
421 }
422
423 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
424 if (!data) {
425 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
426 amdgpu_virt_disable_access_debugfs(adev);
427 return -ENOMEM;
428 }
429
430 /* switch to the specific se/sh/cu */
431 mutex_lock(&adev->grbm_idx_mutex);
432 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
433
434 if (!rd->id.gpr_or_wave) {
435 x = 0;
436 if (adev->gfx.funcs->read_wave_data)
437 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
438 } else {
439 x = size >> 2;
440 if (rd->id.gpr.vpgr_or_sgpr) {
441 if (adev->gfx.funcs->read_wave_vgprs)
442 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
443 } else {
444 if (adev->gfx.funcs->read_wave_sgprs)
445 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
446 }
447 }
448
449 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
450 mutex_unlock(&adev->grbm_idx_mutex);
451
452 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
453
454 if (!x) {
455 result = -EINVAL;
456 goto done;
457 }
458
459 while (size && (*pos < x * 4)) {
460 uint32_t value;
461
462 value = data[*pos >> 2];
463 r = put_user(value, (uint32_t *)buf);
464 if (r) {
465 result = r;
466 goto done;
467 }
468
469 result += 4;
470 buf += 4;
471 *pos += 4;
472 size -= 4;
473 }
474
475 done:
476 amdgpu_virt_disable_access_debugfs(adev);
477 kfree(data);
478 return result;
479 }
480
amdgpu_debugfs_gprwave_ioctl(struct file * f,unsigned int cmd,unsigned long data)481 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
482 {
483 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
484 int r = 0;
485
486 mutex_lock(&rd->lock);
487
488 switch (cmd) {
489 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
490 if (copy_from_user(&rd->id,
491 (struct amdgpu_debugfs_gprwave_iocdata *)data,
492 sizeof(rd->id)))
493 r = -EFAULT;
494 goto done;
495 default:
496 r = -EINVAL;
497 goto done;
498 }
499
500 done:
501 mutex_unlock(&rd->lock);
502 return r;
503 }
504
505
506
507
508 /**
509 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
510 *
511 * @f: open file handle
512 * @buf: User buffer to store read data in
513 * @size: Number of bytes to read
514 * @pos: Offset to seek to
515 *
516 * The lower bits are the BYTE offset of the register to read. This
517 * allows reading multiple registers in a single call and having
518 * the returned size reflect that.
519 */
amdgpu_debugfs_regs_pcie_read(struct file * f,char __user * buf,size_t size,loff_t * pos)520 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
521 size_t size, loff_t *pos)
522 {
523 struct amdgpu_device *adev = file_inode(f)->i_private;
524 ssize_t result = 0;
525 int r;
526
527 if (size & 0x3 || *pos & 0x3)
528 return -EINVAL;
529
530 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
531 if (r < 0) {
532 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
533 return r;
534 }
535
536 r = amdgpu_virt_enable_access_debugfs(adev);
537 if (r < 0) {
538 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
539 return r;
540 }
541
542 while (size) {
543 uint32_t value;
544
545 if (upper_32_bits(*pos))
546 value = RREG32_PCIE_EXT(*pos);
547 else
548 value = RREG32_PCIE(*pos);
549
550 r = put_user(value, (uint32_t *)buf);
551 if (r)
552 goto out;
553
554 result += 4;
555 buf += 4;
556 *pos += 4;
557 size -= 4;
558 }
559
560 r = result;
561 out:
562 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
563 amdgpu_virt_disable_access_debugfs(adev);
564 return r;
565 }
566
567 /**
568 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
569 *
570 * @f: open file handle
571 * @buf: User buffer to write data from
572 * @size: Number of bytes to write
573 * @pos: Offset to seek to
574 *
575 * The lower bits are the BYTE offset of the register to write. This
576 * allows writing multiple registers in a single call and having
577 * the returned size reflect that.
578 */
amdgpu_debugfs_regs_pcie_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)579 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
580 size_t size, loff_t *pos)
581 {
582 struct amdgpu_device *adev = file_inode(f)->i_private;
583 ssize_t result = 0;
584 int r;
585
586 if (size & 0x3 || *pos & 0x3)
587 return -EINVAL;
588
589 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
590 if (r < 0) {
591 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
592 return r;
593 }
594
595 r = amdgpu_virt_enable_access_debugfs(adev);
596 if (r < 0) {
597 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
598 return r;
599 }
600
601 while (size) {
602 uint32_t value;
603
604 r = get_user(value, (uint32_t *)buf);
605 if (r)
606 goto out;
607
608 if (upper_32_bits(*pos))
609 WREG32_PCIE_EXT(*pos, value);
610 else
611 WREG32_PCIE(*pos, value);
612
613 result += 4;
614 buf += 4;
615 *pos += 4;
616 size -= 4;
617 }
618
619 r = result;
620 out:
621 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
622 amdgpu_virt_disable_access_debugfs(adev);
623 return r;
624 }
625
626 /**
627 * amdgpu_debugfs_regs_pcie64_read - Read from a 64-bit PCIE register
628 *
629 * @f: open file handle
630 * @buf: User buffer to store read data in
631 * @size: Number of bytes to read
632 * @pos: Offset to seek to
633 */
amdgpu_debugfs_regs_pcie64_read(struct file * f,char __user * buf,size_t size,loff_t * pos)634 static ssize_t amdgpu_debugfs_regs_pcie64_read(struct file *f, char __user *buf,
635 size_t size, loff_t *pos)
636 {
637 struct amdgpu_device *adev = file_inode(f)->i_private;
638 ssize_t result = 0;
639 int r;
640
641 if (size & 0x7 || *pos & 0x7)
642 return -EINVAL;
643
644 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
645 if (r < 0) {
646 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
647 return r;
648 }
649
650 r = amdgpu_virt_enable_access_debugfs(adev);
651 if (r < 0) {
652 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
653 return r;
654 }
655
656 while (size) {
657 uint64_t value;
658
659 value = RREG64_PCIE_EXT(*pos);
660
661 r = put_user(value, (uint64_t *)buf);
662 if (r)
663 goto out;
664
665 result += 8;
666 buf += 8;
667 *pos += 8;
668 size -= 8;
669 }
670
671 r = result;
672 out:
673 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
674 amdgpu_virt_disable_access_debugfs(adev);
675 return r;
676 }
677
678 /**
679 * amdgpu_debugfs_regs_pcie64_write - Write to a 64-bit PCIE register
680 *
681 * @f: open file handle
682 * @buf: User buffer to write data from
683 * @size: Number of bytes to write
684 * @pos: Offset to seek to
685 */
amdgpu_debugfs_regs_pcie64_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)686 static ssize_t amdgpu_debugfs_regs_pcie64_write(struct file *f, const char __user *buf,
687 size_t size, loff_t *pos)
688 {
689 struct amdgpu_device *adev = file_inode(f)->i_private;
690 ssize_t result = 0;
691 int r;
692
693 if (size & 0x7 || *pos & 0x7)
694 return -EINVAL;
695
696 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
697 if (r < 0) {
698 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
699 return r;
700 }
701
702 r = amdgpu_virt_enable_access_debugfs(adev);
703 if (r < 0) {
704 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
705 return r;
706 }
707
708 while (size) {
709 uint64_t value;
710
711 r = get_user(value, (uint64_t *)buf);
712 if (r)
713 goto out;
714
715 WREG64_PCIE_EXT(*pos, value);
716
717 result += 8;
718 buf += 8;
719 *pos += 8;
720 size -= 8;
721 }
722
723 r = result;
724 out:
725 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
726 amdgpu_virt_disable_access_debugfs(adev);
727 return r;
728 }
729
730 /**
731 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
732 *
733 * @f: open file handle
734 * @buf: User buffer to store read data in
735 * @size: Number of bytes to read
736 * @pos: Offset to seek to
737 *
738 * The lower bits are the BYTE offset of the register to read. This
739 * allows reading multiple registers in a single call and having
740 * the returned size reflect that.
741 */
amdgpu_debugfs_regs_didt_read(struct file * f,char __user * buf,size_t size,loff_t * pos)742 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
743 size_t size, loff_t *pos)
744 {
745 struct amdgpu_device *adev = file_inode(f)->i_private;
746 ssize_t result = 0;
747 int r;
748
749 if (size & 0x3 || *pos & 0x3)
750 return -EINVAL;
751
752 if (!adev->reg.didt.rreg)
753 return -EOPNOTSUPP;
754
755 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
756 if (r < 0) {
757 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
758 return r;
759 }
760
761 r = amdgpu_virt_enable_access_debugfs(adev);
762 if (r < 0) {
763 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
764 return r;
765 }
766
767 while (size) {
768 uint32_t value;
769
770 value = RREG32_DIDT(*pos >> 2);
771 r = put_user(value, (uint32_t *)buf);
772 if (r)
773 goto out;
774
775 result += 4;
776 buf += 4;
777 *pos += 4;
778 size -= 4;
779 }
780
781 r = result;
782 out:
783 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
784 amdgpu_virt_disable_access_debugfs(adev);
785 return r;
786 }
787
788 /**
789 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
790 *
791 * @f: open file handle
792 * @buf: User buffer to write data from
793 * @size: Number of bytes to write
794 * @pos: Offset to seek to
795 *
796 * The lower bits are the BYTE offset of the register to write. This
797 * allows writing multiple registers in a single call and having
798 * the returned size reflect that.
799 */
amdgpu_debugfs_regs_didt_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)800 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
801 size_t size, loff_t *pos)
802 {
803 struct amdgpu_device *adev = file_inode(f)->i_private;
804 ssize_t result = 0;
805 int r;
806
807 if (size & 0x3 || *pos & 0x3)
808 return -EINVAL;
809
810 if (!adev->reg.didt.wreg)
811 return -EOPNOTSUPP;
812
813 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
814 if (r < 0) {
815 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
816 return r;
817 }
818
819 r = amdgpu_virt_enable_access_debugfs(adev);
820 if (r < 0) {
821 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
822 return r;
823 }
824
825 while (size) {
826 uint32_t value;
827
828 r = get_user(value, (uint32_t *)buf);
829 if (r)
830 goto out;
831
832 WREG32_DIDT(*pos >> 2, value);
833
834 result += 4;
835 buf += 4;
836 *pos += 4;
837 size -= 4;
838 }
839
840 r = result;
841 out:
842 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
843 amdgpu_virt_disable_access_debugfs(adev);
844 return r;
845 }
846
847 /**
848 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
849 *
850 * @f: open file handle
851 * @buf: User buffer to store read data in
852 * @size: Number of bytes to read
853 * @pos: Offset to seek to
854 *
855 * The lower bits are the BYTE offset of the register to read. This
856 * allows reading multiple registers in a single call and having
857 * the returned size reflect that.
858 */
amdgpu_debugfs_regs_smc_read(struct file * f,char __user * buf,size_t size,loff_t * pos)859 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
860 size_t size, loff_t *pos)
861 {
862 struct amdgpu_device *adev = file_inode(f)->i_private;
863 ssize_t result = 0;
864 int r;
865
866 if (!adev->reg.smc.rreg)
867 return -EOPNOTSUPP;
868
869 if (size & 0x3 || *pos & 0x3)
870 return -EINVAL;
871
872 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
873 if (r < 0) {
874 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
875 return r;
876 }
877
878 r = amdgpu_virt_enable_access_debugfs(adev);
879 if (r < 0) {
880 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
881 return r;
882 }
883
884 while (size) {
885 uint32_t value;
886
887 value = RREG32_SMC(*pos);
888 r = put_user(value, (uint32_t *)buf);
889 if (r)
890 goto out;
891
892 result += 4;
893 buf += 4;
894 *pos += 4;
895 size -= 4;
896 }
897
898 r = result;
899 out:
900 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
901 amdgpu_virt_disable_access_debugfs(adev);
902 return r;
903 }
904
905 /**
906 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
907 *
908 * @f: open file handle
909 * @buf: User buffer to write data from
910 * @size: Number of bytes to write
911 * @pos: Offset to seek to
912 *
913 * The lower bits are the BYTE offset of the register to write. This
914 * allows writing multiple registers in a single call and having
915 * the returned size reflect that.
916 */
amdgpu_debugfs_regs_smc_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)917 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
918 size_t size, loff_t *pos)
919 {
920 struct amdgpu_device *adev = file_inode(f)->i_private;
921 ssize_t result = 0;
922 int r;
923
924 if (!adev->reg.smc.wreg)
925 return -EOPNOTSUPP;
926
927 if (size & 0x3 || *pos & 0x3)
928 return -EINVAL;
929
930 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
931 if (r < 0) {
932 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
933 return r;
934 }
935
936 r = amdgpu_virt_enable_access_debugfs(adev);
937 if (r < 0) {
938 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
939 return r;
940 }
941
942 while (size) {
943 uint32_t value;
944
945 r = get_user(value, (uint32_t *)buf);
946 if (r)
947 goto out;
948
949 WREG32_SMC(*pos, value);
950
951 result += 4;
952 buf += 4;
953 *pos += 4;
954 size -= 4;
955 }
956
957 r = result;
958 out:
959 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
960 amdgpu_virt_disable_access_debugfs(adev);
961 return r;
962 }
963
964 /**
965 * amdgpu_debugfs_gca_config_read - Read from gfx config data
966 *
967 * @f: open file handle
968 * @buf: User buffer to store read data in
969 * @size: Number of bytes to read
970 * @pos: Offset to seek to
971 *
972 * This file is used to access configuration data in a somewhat
973 * stable fashion. The format is a series of DWORDs with the first
974 * indicating which revision it is. New content is appended to the
975 * end so that older software can still read the data.
976 */
977
amdgpu_debugfs_gca_config_read(struct file * f,char __user * buf,size_t size,loff_t * pos)978 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
979 size_t size, loff_t *pos)
980 {
981 struct amdgpu_device *adev = file_inode(f)->i_private;
982 ssize_t result = 0;
983 int r;
984 uint32_t *config, no_regs = 0;
985
986 if (size & 0x3 || *pos & 0x3)
987 return -EINVAL;
988
989 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
990 if (!config)
991 return -ENOMEM;
992
993 /* version, increment each time something is added */
994 config[no_regs++] = 5;
995 config[no_regs++] = adev->gfx.config.max_shader_engines;
996 config[no_regs++] = adev->gfx.config.max_tile_pipes;
997 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
998 config[no_regs++] = adev->gfx.config.max_sh_per_se;
999 config[no_regs++] = adev->gfx.config.max_backends_per_se;
1000 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
1001 config[no_regs++] = adev->gfx.config.max_gprs;
1002 config[no_regs++] = adev->gfx.config.max_gs_threads;
1003 config[no_regs++] = adev->gfx.config.max_hw_contexts;
1004 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
1005 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
1006 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
1007 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
1008 config[no_regs++] = adev->gfx.config.num_tile_pipes;
1009 config[no_regs++] = adev->gfx.config.backend_enable_mask;
1010 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
1011 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
1012 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
1013 config[no_regs++] = adev->gfx.config.num_gpus;
1014 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
1015 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
1016 config[no_regs++] = adev->gfx.config.gb_addr_config;
1017 config[no_regs++] = adev->gfx.config.num_rbs;
1018
1019 /* rev==1 */
1020 config[no_regs++] = adev->rev_id;
1021 config[no_regs++] = adev->pg_flags;
1022 config[no_regs++] = lower_32_bits(adev->cg_flags);
1023
1024 /* rev==2 */
1025 config[no_regs++] = adev->family;
1026 config[no_regs++] = adev->external_rev_id;
1027
1028 /* rev==3 */
1029 config[no_regs++] = adev->pdev->device;
1030 config[no_regs++] = adev->pdev->revision;
1031 config[no_regs++] = adev->pdev->subsystem_device;
1032 config[no_regs++] = adev->pdev->subsystem_vendor;
1033
1034 /* rev==4 APU flag */
1035 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
1036
1037 /* rev==5 PG/CG flag upper 32bit */
1038 config[no_regs++] = 0;
1039 config[no_regs++] = upper_32_bits(adev->cg_flags);
1040
1041 while (size && (*pos < no_regs * 4)) {
1042 uint32_t value;
1043
1044 value = config[*pos >> 2];
1045 r = put_user(value, (uint32_t *)buf);
1046 if (r) {
1047 kfree(config);
1048 return r;
1049 }
1050
1051 result += 4;
1052 buf += 4;
1053 *pos += 4;
1054 size -= 4;
1055 }
1056
1057 kfree(config);
1058 return result;
1059 }
1060
1061 /**
1062 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
1063 *
1064 * @f: open file handle
1065 * @buf: User buffer to store read data in
1066 * @size: Number of bytes to read
1067 * @pos: Offset to seek to
1068 *
1069 * The offset is treated as the BYTE address of one of the sensors
1070 * enumerated in amd/include/kgd_pp_interface.h under the
1071 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
1072 * you would use the offset 3 * 4 = 12.
1073 */
amdgpu_debugfs_sensor_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1074 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
1075 size_t size, loff_t *pos)
1076 {
1077 struct amdgpu_device *adev = file_inode(f)->i_private;
1078 int idx, x, outsize, r, valuesize;
1079 uint32_t values[16];
1080
1081 if (size & 3 || *pos & 0x3)
1082 return -EINVAL;
1083
1084 if (!adev->pm.dpm_enabled)
1085 return -EINVAL;
1086
1087 /* convert offset to sensor number */
1088 idx = *pos >> 2;
1089
1090 valuesize = sizeof(values);
1091
1092 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1093 if (r < 0) {
1094 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1095 return r;
1096 }
1097
1098 r = amdgpu_virt_enable_access_debugfs(adev);
1099 if (r < 0) {
1100 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1101 return r;
1102 }
1103
1104 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
1105
1106 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1107
1108 if (r) {
1109 amdgpu_virt_disable_access_debugfs(adev);
1110 return r;
1111 }
1112
1113 if (idx == AMDGPU_PP_SENSOR_GPU_AVG_POWER ||
1114 idx == AMDGPU_PP_SENSOR_GPU_INPUT_POWER)
1115 values[0] = AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(values[0]);
1116
1117 if (size > valuesize) {
1118 amdgpu_virt_disable_access_debugfs(adev);
1119 return -EINVAL;
1120 }
1121
1122 outsize = 0;
1123 x = 0;
1124 if (!r) {
1125 while (size) {
1126 r = put_user(values[x++], (int32_t *)buf);
1127 buf += 4;
1128 size -= 4;
1129 outsize += 4;
1130 }
1131 }
1132
1133 amdgpu_virt_disable_access_debugfs(adev);
1134 return !r ? outsize : r;
1135 }
1136
1137 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1138 *
1139 * @f: open file handle
1140 * @buf: User buffer to store read data in
1141 * @size: Number of bytes to read
1142 * @pos: Offset to seek to
1143 *
1144 * The offset being sought changes which wave that the status data
1145 * will be returned for. The bits are used as follows:
1146 *
1147 * Bits 0..6: Byte offset into data
1148 * Bits 7..14: SE selector
1149 * Bits 15..22: SH/SA selector
1150 * Bits 23..30: CU/{WGP+SIMD} selector
1151 * Bits 31..36: WAVE ID selector
1152 * Bits 37..44: SIMD ID selector
1153 *
1154 * The returned data begins with one DWORD of version information
1155 * Followed by WAVE STATUS registers relevant to the GFX IP version
1156 * being used. See gfx_v8_0_read_wave_data() for an example output.
1157 */
amdgpu_debugfs_wave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1158 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1159 size_t size, loff_t *pos)
1160 {
1161 struct amdgpu_device *adev = f->f_inode->i_private;
1162 int r, x;
1163 ssize_t result = 0;
1164 uint32_t offset, se, sh, cu, wave, simd, data[32];
1165
1166 if (size & 3 || *pos & 3)
1167 return -EINVAL;
1168
1169 /* decode offset */
1170 offset = (*pos & GENMASK_ULL(6, 0));
1171 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1172 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1173 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1174 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1175 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1176
1177 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1178 if (r < 0) {
1179 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1180 return r;
1181 }
1182
1183 r = amdgpu_virt_enable_access_debugfs(adev);
1184 if (r < 0) {
1185 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1186 return r;
1187 }
1188
1189 /* switch to the specific se/sh/cu */
1190 mutex_lock(&adev->grbm_idx_mutex);
1191 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1192
1193 x = 0;
1194 if (adev->gfx.funcs->read_wave_data)
1195 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1196
1197 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1198 mutex_unlock(&adev->grbm_idx_mutex);
1199
1200 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1201
1202 if (!x) {
1203 amdgpu_virt_disable_access_debugfs(adev);
1204 return -EINVAL;
1205 }
1206
1207 while (size && (offset < x * 4)) {
1208 uint32_t value;
1209
1210 value = data[offset >> 2];
1211 r = put_user(value, (uint32_t *)buf);
1212 if (r) {
1213 amdgpu_virt_disable_access_debugfs(adev);
1214 return r;
1215 }
1216
1217 result += 4;
1218 buf += 4;
1219 offset += 4;
1220 size -= 4;
1221 }
1222
1223 amdgpu_virt_disable_access_debugfs(adev);
1224 return result;
1225 }
1226
1227 /** amdgpu_debugfs_gpr_read - Read wave gprs
1228 *
1229 * @f: open file handle
1230 * @buf: User buffer to store read data in
1231 * @size: Number of bytes to read
1232 * @pos: Offset to seek to
1233 *
1234 * The offset being sought changes which wave that the status data
1235 * will be returned for. The bits are used as follows:
1236 *
1237 * Bits 0..11: Byte offset into data
1238 * Bits 12..19: SE selector
1239 * Bits 20..27: SH/SA selector
1240 * Bits 28..35: CU/{WGP+SIMD} selector
1241 * Bits 36..43: WAVE ID selector
1242 * Bits 37..44: SIMD ID selector
1243 * Bits 52..59: Thread selector
1244 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1245 *
1246 * The return data comes from the SGPR or VGPR register bank for
1247 * the selected operational unit.
1248 */
amdgpu_debugfs_gpr_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1249 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1250 size_t size, loff_t *pos)
1251 {
1252 struct amdgpu_device *adev = f->f_inode->i_private;
1253 int r;
1254 ssize_t result = 0;
1255 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1256
1257 if (size > 4096 || size & 3 || *pos & 3)
1258 return -EINVAL;
1259
1260 /* decode offset */
1261 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1262 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1263 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1264 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1265 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1266 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1267 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1268 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1269
1270 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1271 if (!data)
1272 return -ENOMEM;
1273
1274 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1275 if (r < 0)
1276 goto err;
1277
1278 r = amdgpu_virt_enable_access_debugfs(adev);
1279 if (r < 0)
1280 goto err;
1281
1282 /* switch to the specific se/sh/cu */
1283 mutex_lock(&adev->grbm_idx_mutex);
1284 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1285
1286 if (bank == 0) {
1287 if (adev->gfx.funcs->read_wave_vgprs)
1288 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1289 } else {
1290 if (adev->gfx.funcs->read_wave_sgprs)
1291 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1292 }
1293
1294 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1295 mutex_unlock(&adev->grbm_idx_mutex);
1296
1297 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1298
1299 while (size) {
1300 uint32_t value;
1301
1302 value = data[result >> 2];
1303 r = put_user(value, (uint32_t *)buf);
1304 if (r) {
1305 amdgpu_virt_disable_access_debugfs(adev);
1306 goto err;
1307 }
1308
1309 result += 4;
1310 buf += 4;
1311 size -= 4;
1312 }
1313
1314 kfree(data);
1315 amdgpu_virt_disable_access_debugfs(adev);
1316 return result;
1317
1318 err:
1319 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1320 kfree(data);
1321 return r;
1322 }
1323
1324 /**
1325 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1326 *
1327 * @f: open file handle
1328 * @buf: User buffer to store read data in
1329 * @size: Number of bytes to read
1330 * @pos: Offset to seek to
1331 *
1332 * Read a live GFXOFF residency sample from firmware. One needs to start logging
1333 * before getting the current value.
1334 */
amdgpu_debugfs_gfxoff_residency_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1335 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1336 size_t size, loff_t *pos)
1337 {
1338 struct amdgpu_device *adev = file_inode(f)->i_private;
1339 ssize_t result = 0;
1340 int r;
1341
1342 if (size & 0x3 || *pos & 0x3)
1343 return -EINVAL;
1344
1345 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1346 if (r < 0) {
1347 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1348 return r;
1349 }
1350
1351 while (size) {
1352 uint32_t value;
1353
1354 r = amdgpu_get_gfx_off_residency(adev, &value);
1355 if (r)
1356 goto out;
1357
1358 r = put_user(value, (uint32_t *)buf);
1359 if (r)
1360 goto out;
1361
1362 result += 4;
1363 buf += 4;
1364 *pos += 4;
1365 size -= 4;
1366 }
1367
1368 r = result;
1369 out:
1370 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1371
1372 return r;
1373 }
1374
1375 /**
1376 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1377 *
1378 * @f: open file handle
1379 * @buf: User buffer to write data from
1380 * @size: Number of bytes to write
1381 * @pos: Offset to seek to
1382 *
1383 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1384 */
amdgpu_debugfs_gfxoff_residency_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1385 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1386 size_t size, loff_t *pos)
1387 {
1388 struct amdgpu_device *adev = file_inode(f)->i_private;
1389 ssize_t result = 0;
1390 int r;
1391
1392 if (size & 0x3 || *pos & 0x3)
1393 return -EINVAL;
1394
1395 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1396 if (r < 0) {
1397 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1398 return r;
1399 }
1400
1401 while (size) {
1402 u32 value;
1403
1404 r = get_user(value, (uint32_t *)buf);
1405 if (r)
1406 goto out;
1407
1408 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1409
1410 result += 4;
1411 buf += 4;
1412 *pos += 4;
1413 size -= 4;
1414 }
1415
1416 r = result;
1417 out:
1418 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1419
1420 return r;
1421 }
1422
1423
1424 /**
1425 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1426 *
1427 * @f: open file handle
1428 * @buf: User buffer to store read data in
1429 * @size: Number of bytes to read
1430 * @pos: Offset to seek to
1431 */
amdgpu_debugfs_gfxoff_count_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1432 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1433 size_t size, loff_t *pos)
1434 {
1435 struct amdgpu_device *adev = file_inode(f)->i_private;
1436 ssize_t result = 0;
1437 int r;
1438
1439 if (size & 0x3 || *pos & 0x3)
1440 return -EINVAL;
1441
1442 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1443 if (r < 0) {
1444 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1445 return r;
1446 }
1447
1448 while (size) {
1449 u64 value = 0;
1450
1451 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1452 if (r)
1453 goto out;
1454
1455 r = put_user(value, (u64 *)buf);
1456 if (r)
1457 goto out;
1458
1459 result += 4;
1460 buf += 4;
1461 *pos += 4;
1462 size -= 4;
1463 }
1464
1465 r = result;
1466 out:
1467 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1468
1469 return r;
1470 }
1471
1472 /**
1473 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1474 *
1475 * @f: open file handle
1476 * @buf: User buffer to write data from
1477 * @size: Number of bytes to write
1478 * @pos: Offset to seek to
1479 *
1480 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1481 */
amdgpu_debugfs_gfxoff_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1482 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1483 size_t size, loff_t *pos)
1484 {
1485 struct amdgpu_device *adev = file_inode(f)->i_private;
1486 ssize_t result = 0;
1487 int r;
1488
1489 if (size & 0x3 || *pos & 0x3)
1490 return -EINVAL;
1491
1492 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1493 if (r < 0) {
1494 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1495 return r;
1496 }
1497
1498 while (size) {
1499 uint32_t value;
1500
1501 r = get_user(value, (uint32_t *)buf);
1502 if (r)
1503 goto out;
1504
1505 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1506
1507 result += 4;
1508 buf += 4;
1509 *pos += 4;
1510 size -= 4;
1511 }
1512
1513 r = result;
1514 out:
1515 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1516
1517 return r;
1518 }
1519
1520
1521 /**
1522 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1523 *
1524 * @f: open file handle
1525 * @buf: User buffer to store read data in
1526 * @size: Number of bytes to read
1527 * @pos: Offset to seek to
1528 */
amdgpu_debugfs_gfxoff_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1529 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1530 size_t size, loff_t *pos)
1531 {
1532 struct amdgpu_device *adev = file_inode(f)->i_private;
1533 ssize_t result = 0;
1534 int r;
1535
1536 if (size & 0x3 || *pos & 0x3)
1537 return -EINVAL;
1538
1539 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1540 if (r < 0) {
1541 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1542 return r;
1543 }
1544
1545 while (size) {
1546 u32 value = adev->gfx.gfx_off_state;
1547
1548 r = put_user(value, (u32 *)buf);
1549 if (r)
1550 goto out;
1551
1552 result += 4;
1553 buf += 4;
1554 *pos += 4;
1555 size -= 4;
1556 }
1557
1558 r = result;
1559 out:
1560 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1561
1562 return r;
1563 }
1564
amdgpu_debugfs_gfxoff_status_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1565 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1566 size_t size, loff_t *pos)
1567 {
1568 struct amdgpu_device *adev = file_inode(f)->i_private;
1569 ssize_t result = 0;
1570 int r;
1571
1572 if (size & 0x3 || *pos & 0x3)
1573 return -EINVAL;
1574
1575 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1576 if (r < 0) {
1577 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1578 return r;
1579 }
1580
1581 while (size) {
1582 u32 value;
1583
1584 r = amdgpu_get_gfx_off_status(adev, &value);
1585 if (r)
1586 goto out;
1587
1588 r = put_user(value, (u32 *)buf);
1589 if (r)
1590 goto out;
1591
1592 result += 4;
1593 buf += 4;
1594 *pos += 4;
1595 size -= 4;
1596 }
1597
1598 r = result;
1599 out:
1600 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1601
1602 return r;
1603 }
1604
1605 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1606 .owner = THIS_MODULE,
1607 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1608 .read = amdgpu_debugfs_regs2_read,
1609 .write = amdgpu_debugfs_regs2_write,
1610 .open = amdgpu_debugfs_regs2_open,
1611 .release = amdgpu_debugfs_regs2_release,
1612 .llseek = default_llseek
1613 };
1614
1615 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1616 .owner = THIS_MODULE,
1617 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1618 .read = amdgpu_debugfs_gprwave_read,
1619 .open = amdgpu_debugfs_gprwave_open,
1620 .release = amdgpu_debugfs_gprwave_release,
1621 .llseek = default_llseek
1622 };
1623
1624 static const struct file_operations amdgpu_debugfs_regs_fops = {
1625 .owner = THIS_MODULE,
1626 .read = amdgpu_debugfs_regs_read,
1627 .write = amdgpu_debugfs_regs_write,
1628 .llseek = default_llseek
1629 };
1630 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1631 .owner = THIS_MODULE,
1632 .read = amdgpu_debugfs_regs_didt_read,
1633 .write = amdgpu_debugfs_regs_didt_write,
1634 .llseek = default_llseek
1635 };
1636 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1637 .owner = THIS_MODULE,
1638 .read = amdgpu_debugfs_regs_pcie_read,
1639 .write = amdgpu_debugfs_regs_pcie_write,
1640 .llseek = default_llseek
1641 };
1642 static const struct file_operations amdgpu_debugfs_regs_pcie64_fops = {
1643 .owner = THIS_MODULE,
1644 .read = amdgpu_debugfs_regs_pcie64_read,
1645 .write = amdgpu_debugfs_regs_pcie64_write,
1646 .llseek = default_llseek
1647 };
1648 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1649 .owner = THIS_MODULE,
1650 .read = amdgpu_debugfs_regs_smc_read,
1651 .write = amdgpu_debugfs_regs_smc_write,
1652 .llseek = default_llseek
1653 };
1654
1655 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1656 .owner = THIS_MODULE,
1657 .read = amdgpu_debugfs_gca_config_read,
1658 .llseek = default_llseek
1659 };
1660
1661 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1662 .owner = THIS_MODULE,
1663 .read = amdgpu_debugfs_sensor_read,
1664 .llseek = default_llseek
1665 };
1666
1667 static const struct file_operations amdgpu_debugfs_wave_fops = {
1668 .owner = THIS_MODULE,
1669 .read = amdgpu_debugfs_wave_read,
1670 .llseek = default_llseek
1671 };
1672 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1673 .owner = THIS_MODULE,
1674 .read = amdgpu_debugfs_gpr_read,
1675 .llseek = default_llseek
1676 };
1677
1678 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1679 .owner = THIS_MODULE,
1680 .read = amdgpu_debugfs_gfxoff_read,
1681 .write = amdgpu_debugfs_gfxoff_write,
1682 .llseek = default_llseek
1683 };
1684
1685 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1686 .owner = THIS_MODULE,
1687 .read = amdgpu_debugfs_gfxoff_status_read,
1688 .llseek = default_llseek
1689 };
1690
1691 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1692 .owner = THIS_MODULE,
1693 .read = amdgpu_debugfs_gfxoff_count_read,
1694 .llseek = default_llseek
1695 };
1696
1697 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1698 .owner = THIS_MODULE,
1699 .read = amdgpu_debugfs_gfxoff_residency_read,
1700 .write = amdgpu_debugfs_gfxoff_residency_write,
1701 .llseek = default_llseek
1702 };
1703
1704 static const struct file_operations *debugfs_regs[] = {
1705 &amdgpu_debugfs_regs_fops,
1706 &amdgpu_debugfs_regs2_fops,
1707 &amdgpu_debugfs_gprwave_fops,
1708 &amdgpu_debugfs_regs_didt_fops,
1709 &amdgpu_debugfs_regs_pcie_fops,
1710 &amdgpu_debugfs_regs_pcie64_fops,
1711 &amdgpu_debugfs_regs_smc_fops,
1712 &amdgpu_debugfs_gca_config_fops,
1713 &amdgpu_debugfs_sensors_fops,
1714 &amdgpu_debugfs_wave_fops,
1715 &amdgpu_debugfs_gpr_fops,
1716 &amdgpu_debugfs_gfxoff_fops,
1717 &amdgpu_debugfs_gfxoff_status_fops,
1718 &amdgpu_debugfs_gfxoff_count_fops,
1719 &amdgpu_debugfs_gfxoff_residency_fops,
1720 };
1721
1722 static const char * const debugfs_regs_names[] = {
1723 "amdgpu_regs",
1724 "amdgpu_regs2",
1725 "amdgpu_gprwave",
1726 "amdgpu_regs_didt",
1727 "amdgpu_regs_pcie",
1728 "amdgpu_regs_pcie64",
1729 "amdgpu_regs_smc",
1730 "amdgpu_gca_config",
1731 "amdgpu_sensors",
1732 "amdgpu_wave",
1733 "amdgpu_gpr",
1734 "amdgpu_gfxoff",
1735 "amdgpu_gfxoff_status",
1736 "amdgpu_gfxoff_count",
1737 "amdgpu_gfxoff_residency",
1738 };
1739
1740 /**
1741 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1742 * register access.
1743 *
1744 * @adev: The device to attach the debugfs entries to
1745 */
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)1746 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1747 {
1748 struct drm_minor *minor = adev_to_drm(adev)->primary;
1749 struct dentry *ent, *root = minor->debugfs_root;
1750 unsigned int i;
1751
1752 if (security_locked_down(LOCKDOWN_PCI_ACCESS)) {
1753 drm_info(adev_to_drm(adev),
1754 "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n");
1755 return 0;
1756 }
1757
1758 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1759 ent = debugfs_create_file(debugfs_regs_names[i],
1760 S_IFREG | 0400, root,
1761 adev, debugfs_regs[i]);
1762 if (!i && !IS_ERR_OR_NULL(ent))
1763 i_size_write(ent->d_inode, adev->rmmio_size);
1764 }
1765
1766 return 0;
1767 }
1768
amdgpu_debugfs_test_ib_show(struct seq_file * m,void * unused)1769 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1770 {
1771 struct amdgpu_device *adev = m->private;
1772 struct drm_device *dev = adev_to_drm(adev);
1773 int r = 0, i;
1774
1775 r = pm_runtime_get_sync(dev->dev);
1776 if (r < 0) {
1777 pm_runtime_put_autosuspend(dev->dev);
1778 return r;
1779 }
1780
1781 /* Avoid accidently unparking the sched thread during GPU reset */
1782 r = down_write_killable(&adev->reset_domain->sem);
1783 if (r) {
1784 pm_runtime_put_autosuspend(dev->dev);
1785 return r;
1786 }
1787
1788 /* hold on the scheduler */
1789 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1790 struct amdgpu_ring *ring = adev->rings[i];
1791
1792 if (!amdgpu_ring_sched_ready(ring))
1793 continue;
1794 drm_sched_wqueue_stop(&ring->sched);
1795 }
1796
1797 seq_puts(m, "run ib test:\n");
1798 r = amdgpu_ib_ring_tests(adev);
1799 if (r)
1800 seq_printf(m, "ib ring tests failed (%d).\n", r);
1801 else
1802 seq_puts(m, "ib ring tests passed.\n");
1803
1804 /* go on the scheduler */
1805 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1806 struct amdgpu_ring *ring = adev->rings[i];
1807
1808 if (!amdgpu_ring_sched_ready(ring))
1809 continue;
1810 drm_sched_wqueue_start(&ring->sched);
1811 }
1812
1813 up_write(&adev->reset_domain->sem);
1814
1815 pm_runtime_put_autosuspend(dev->dev);
1816
1817 return 0;
1818 }
1819
amdgpu_debugfs_evict_vram(void * data,u64 * val)1820 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1821 {
1822 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1823 struct drm_device *dev = adev_to_drm(adev);
1824 int r;
1825
1826 r = pm_runtime_get_sync(dev->dev);
1827 if (r < 0) {
1828 pm_runtime_put_autosuspend(dev->dev);
1829 return r;
1830 }
1831
1832 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1833
1834 pm_runtime_put_autosuspend(dev->dev);
1835
1836 return 0;
1837 }
1838
1839
amdgpu_debugfs_evict_gtt(void * data,u64 * val)1840 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1841 {
1842 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1843 struct drm_device *dev = adev_to_drm(adev);
1844 int r;
1845
1846 r = pm_runtime_get_sync(dev->dev);
1847 if (r < 0) {
1848 pm_runtime_put_autosuspend(dev->dev);
1849 return r;
1850 }
1851
1852 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1853
1854 pm_runtime_put_autosuspend(dev->dev);
1855
1856 return 0;
1857 }
1858
amdgpu_debugfs_benchmark(void * data,u64 val)1859 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1860 {
1861 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1862 struct drm_device *dev = adev_to_drm(adev);
1863 int r;
1864
1865 r = pm_runtime_get_sync(dev->dev);
1866 if (r < 0) {
1867 pm_runtime_put_autosuspend(dev->dev);
1868 return r;
1869 }
1870
1871 r = amdgpu_benchmark(adev, val);
1872
1873 pm_runtime_put_autosuspend(dev->dev);
1874
1875 return r;
1876 }
1877
amdgpu_debugfs_vm_info_show(struct seq_file * m,void * unused)1878 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1879 {
1880 struct amdgpu_device *adev = m->private;
1881 struct drm_device *dev = adev_to_drm(adev);
1882 struct drm_file *file;
1883 int r;
1884
1885 r = mutex_lock_interruptible(&dev->filelist_mutex);
1886 if (r)
1887 return r;
1888
1889 list_for_each_entry(file, &dev->filelist, lhead) {
1890 struct amdgpu_fpriv *fpriv = file->driver_priv;
1891 struct amdgpu_vm *vm = &fpriv->vm;
1892 struct amdgpu_task_info *ti;
1893
1894 ti = amdgpu_vm_get_task_info_vm(vm);
1895 if (ti) {
1896 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->task.pid, ti->process_name);
1897 amdgpu_vm_put_task_info(ti);
1898 }
1899
1900 r = amdgpu_bo_reserve(vm->root.bo, true);
1901 if (r)
1902 break;
1903 amdgpu_debugfs_vm_bo_info(vm, m);
1904 amdgpu_bo_unreserve(vm->root.bo);
1905 }
1906
1907 mutex_unlock(&dev->filelist_mutex);
1908
1909 return r;
1910 }
1911
1912 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1913 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1914 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1915 NULL, "%lld\n");
1916 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1917 NULL, "%lld\n");
1918 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1919 "%lld\n");
1920
amdgpu_ib_preempt_fences_swap(struct amdgpu_ring * ring,struct dma_fence ** fences)1921 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1922 struct dma_fence **fences)
1923 {
1924 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1925 uint32_t sync_seq, last_seq;
1926
1927 last_seq = atomic_read(&ring->fence_drv.last_seq);
1928 sync_seq = ring->fence_drv.sync_seq;
1929
1930 last_seq &= drv->num_fences_mask;
1931 sync_seq &= drv->num_fences_mask;
1932
1933 do {
1934 struct dma_fence *fence, **ptr;
1935
1936 ++last_seq;
1937 last_seq &= drv->num_fences_mask;
1938 ptr = &drv->fences[last_seq];
1939
1940 fence = rcu_dereference_protected(*ptr, 1);
1941 RCU_INIT_POINTER(*ptr, NULL);
1942
1943 if (!fence)
1944 continue;
1945
1946 fences[last_seq] = fence;
1947
1948 } while (last_seq != sync_seq);
1949 }
1950
amdgpu_ib_preempt_signal_fences(struct dma_fence ** fences,int length)1951 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1952 int length)
1953 {
1954 int i;
1955 struct dma_fence *fence;
1956
1957 for (i = 0; i < length; i++) {
1958 fence = fences[i];
1959 if (!fence)
1960 continue;
1961 dma_fence_signal(fence);
1962 dma_fence_put(fence);
1963 }
1964 }
1965
amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler * sched)1966 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1967 {
1968 struct drm_sched_job *s_job;
1969 struct dma_fence *fence;
1970
1971 spin_lock(&sched->job_list_lock);
1972 list_for_each_entry(s_job, &sched->pending_list, list) {
1973 fence = sched->ops->run_job(s_job);
1974 dma_fence_put(fence);
1975 }
1976 spin_unlock(&sched->job_list_lock);
1977 }
1978
amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring * ring)1979 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1980 {
1981 struct amdgpu_job *job;
1982 struct drm_sched_job *s_job, *tmp;
1983 uint32_t preempt_seq;
1984 struct dma_fence *fence, **ptr;
1985 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1986 struct drm_gpu_scheduler *sched = &ring->sched;
1987 bool preempted = true;
1988
1989 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1990 return;
1991
1992 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1993 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1994 preempted = false;
1995 goto no_preempt;
1996 }
1997
1998 preempt_seq &= drv->num_fences_mask;
1999 ptr = &drv->fences[preempt_seq];
2000 fence = rcu_dereference_protected(*ptr, 1);
2001
2002 no_preempt:
2003 spin_lock(&sched->job_list_lock);
2004 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
2005 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
2006 /* remove job from ring_mirror_list */
2007 list_del_init(&s_job->list);
2008 sched->ops->free_job(s_job);
2009 continue;
2010 }
2011 job = to_amdgpu_job(s_job);
2012 if (preempted && (&job->hw_fence->base) == fence)
2013 /* mark the job as preempted */
2014 job->preemption_status |= AMDGPU_IB_PREEMPTED;
2015 }
2016 spin_unlock(&sched->job_list_lock);
2017 }
2018
amdgpu_debugfs_ib_preempt(void * data,u64 val)2019 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
2020 {
2021 int r, length;
2022 struct amdgpu_ring *ring;
2023 struct dma_fence **fences = NULL;
2024 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2025
2026 if (val >= AMDGPU_MAX_RINGS)
2027 return -EINVAL;
2028
2029 ring = adev->rings[val];
2030
2031 if (!amdgpu_ring_sched_ready(ring) ||
2032 !ring->funcs->preempt_ib)
2033 return -EINVAL;
2034
2035 /* the last preemption failed */
2036 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
2037 return -EBUSY;
2038
2039 length = ring->fence_drv.num_fences_mask + 1;
2040 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
2041 if (!fences)
2042 return -ENOMEM;
2043
2044 /* Avoid accidently unparking the sched thread during GPU reset */
2045 r = down_read_killable(&adev->reset_domain->sem);
2046 if (r)
2047 goto pro_end;
2048
2049 /* stop the scheduler */
2050 drm_sched_wqueue_stop(&ring->sched);
2051
2052 /* preempt the IB */
2053 r = amdgpu_ring_preempt_ib(ring);
2054 if (r) {
2055 drm_warn(adev_to_drm(adev), "failed to preempt ring %d\n", ring->idx);
2056 goto failure;
2057 }
2058
2059 amdgpu_fence_process(ring);
2060
2061 if (atomic_read(&ring->fence_drv.last_seq) !=
2062 ring->fence_drv.sync_seq) {
2063 drm_info(adev_to_drm(adev), "ring %d was preempted\n", ring->idx);
2064
2065 amdgpu_ib_preempt_mark_partial_job(ring);
2066
2067 /* swap out the old fences */
2068 amdgpu_ib_preempt_fences_swap(ring, fences);
2069
2070 amdgpu_fence_driver_force_completion(ring, NULL);
2071
2072 /* resubmit unfinished jobs */
2073 amdgpu_ib_preempt_job_recovery(&ring->sched);
2074
2075 /* wait for jobs finished */
2076 amdgpu_fence_wait_empty(ring);
2077
2078 /* signal the old fences */
2079 amdgpu_ib_preempt_signal_fences(fences, length);
2080 }
2081
2082 failure:
2083 /* restart the scheduler */
2084 drm_sched_wqueue_start(&ring->sched);
2085
2086 up_read(&adev->reset_domain->sem);
2087
2088 pro_end:
2089 kfree(fences);
2090
2091 return r;
2092 }
2093
amdgpu_debugfs_sclk_set(void * data,u64 val)2094 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
2095 {
2096 int ret = 0;
2097 uint32_t max_freq, min_freq;
2098 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2099
2100 if (amdgpu_sriov_multi_vf_mode(adev))
2101 return -EINVAL;
2102
2103 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
2104 if (ret < 0) {
2105 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2106 return ret;
2107 }
2108
2109 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
2110 if (ret == -EOPNOTSUPP) {
2111 ret = 0;
2112 goto out;
2113 }
2114 if (ret || val > max_freq || val < min_freq) {
2115 ret = -EINVAL;
2116 goto out;
2117 }
2118
2119 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2120 if (ret)
2121 ret = -EINVAL;
2122
2123 out:
2124 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2125
2126 return ret;
2127 }
2128
2129 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2130 amdgpu_debugfs_ib_preempt, "%llu\n");
2131
2132 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2133 amdgpu_debugfs_sclk_set, "%llu\n");
2134
amdgpu_debugfs_init(struct amdgpu_device * adev)2135 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2136 {
2137 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2138 struct dentry *ent;
2139 int r, i;
2140
2141 if (!debugfs_initialized())
2142 return 0;
2143
2144 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2145 &adev->pm.smu_debug_mask);
2146
2147 debugfs_create_x64("unique_id", 0444, root, &adev->unique_id);
2148 debugfs_create_x8("unitid", 0444, root, &adev->unitid);
2149
2150 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2151 &fops_ib_preempt);
2152 if (IS_ERR(ent)) {
2153 drm_err(adev_to_drm(adev),
2154 "unable to create amdgpu_preempt_ib debugsfs file\n");
2155 return PTR_ERR(ent);
2156 }
2157
2158 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2159 &fops_sclk_set);
2160 if (IS_ERR(ent)) {
2161 drm_err(adev_to_drm(adev),
2162 "unable to create amdgpu_set_sclk debugsfs file\n");
2163 return PTR_ERR(ent);
2164 }
2165
2166 /* Register debugfs entries for amdgpu_ttm */
2167 amdgpu_ttm_debugfs_init(adev);
2168 amdgpu_debugfs_pm_init(adev);
2169 amdgpu_debugfs_sa_init(adev);
2170 amdgpu_debugfs_fence_init(adev);
2171 amdgpu_debugfs_gem_init(adev);
2172
2173 r = amdgpu_debugfs_regs_init(adev);
2174 if (r)
2175 drm_err(adev_to_drm(adev), "registering register debugfs failed (%d).\n", r);
2176
2177 amdgpu_debugfs_firmware_init(adev);
2178 amdgpu_ta_if_debugfs_init(adev);
2179
2180 amdgpu_debugfs_mes_event_log_init(adev);
2181
2182 #if defined(CONFIG_DRM_AMD_DC)
2183 if (adev->dc_enabled)
2184 dtn_debugfs_init(adev);
2185 #endif
2186
2187 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2188 struct amdgpu_ring *ring = adev->rings[i];
2189
2190 if (!ring)
2191 continue;
2192 if (ring == &adev->cper.ring_buf && !adev->cper.enabled)
2193 continue;
2194
2195 amdgpu_debugfs_ring_init(adev, ring);
2196 }
2197
2198 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2199 if (!amdgpu_vcnfw_log)
2200 break;
2201
2202 if (adev->vcn.harvest_config & (1 << i))
2203 continue;
2204
2205 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2206 }
2207
2208 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog)
2209 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm);
2210
2211 amdgpu_debugfs_vcn_sched_mask_init(adev);
2212 amdgpu_debugfs_jpeg_sched_mask_init(adev);
2213 amdgpu_debugfs_gfx_sched_mask_init(adev);
2214 amdgpu_debugfs_compute_sched_mask_init(adev);
2215 amdgpu_debugfs_sdma_sched_mask_init(adev);
2216
2217 amdgpu_ras_debugfs_create_all(adev);
2218 amdgpu_rap_debugfs_init(adev);
2219 amdgpu_securedisplay_debugfs_init(adev);
2220 amdgpu_fw_attestation_debugfs_init(adev);
2221 amdgpu_psp_debugfs_init(adev);
2222
2223 debugfs_create_file("amdgpu_evict_vram", 0400, root, adev,
2224 &amdgpu_evict_vram_fops);
2225 debugfs_create_file("amdgpu_evict_gtt", 0400, root, adev,
2226 &amdgpu_evict_gtt_fops);
2227 debugfs_create_file("amdgpu_test_ib", 0400, root, adev,
2228 &amdgpu_debugfs_test_ib_fops);
2229 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2230 &amdgpu_debugfs_vm_info_fops);
2231 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2232 &amdgpu_benchmark_fops);
2233
2234 adev->debugfs_vbios_blob.data = adev->bios;
2235 adev->debugfs_vbios_blob.size = adev->bios_size;
2236 debugfs_create_blob("amdgpu_vbios", 0444, root,
2237 &adev->debugfs_vbios_blob);
2238
2239 if (adev->discovery.debugfs_blob.size)
2240 debugfs_create_blob("amdgpu_discovery", 0444, root,
2241 &adev->discovery.debugfs_blob);
2242
2243 return 0;
2244 }
2245
amdgpu_pt_info_read(struct seq_file * m,void * unused)2246 static int amdgpu_pt_info_read(struct seq_file *m, void *unused)
2247 {
2248 struct drm_file *file;
2249 struct amdgpu_fpriv *fpriv;
2250 struct amdgpu_bo *root_bo;
2251 struct amdgpu_device *adev;
2252 int r;
2253
2254 file = m->private;
2255 if (!file)
2256 return -EINVAL;
2257
2258 adev = drm_to_adev(file->minor->dev);
2259 fpriv = file->driver_priv;
2260 if (!fpriv || !fpriv->vm.root.bo)
2261 return -ENODEV;
2262
2263 root_bo = amdgpu_bo_ref(fpriv->vm.root.bo);
2264 r = amdgpu_bo_reserve(root_bo, true);
2265 if (r) {
2266 amdgpu_bo_unref(&root_bo);
2267 return -EINVAL;
2268 }
2269
2270 seq_printf(m, "pd_address: 0x%llx\n", amdgpu_gmc_pd_addr(fpriv->vm.root.bo));
2271 seq_printf(m, "max_pfn: 0x%llx\n", adev->vm_manager.max_pfn);
2272 seq_printf(m, "num_level: 0x%x\n", adev->vm_manager.num_level);
2273 seq_printf(m, "block_size: 0x%x\n", adev->vm_manager.block_size);
2274 seq_printf(m, "fragment_size: 0x%x\n", adev->vm_manager.fragment_size);
2275
2276 amdgpu_bo_unreserve(root_bo);
2277 amdgpu_bo_unref(&root_bo);
2278
2279 return 0;
2280 }
2281
amdgpu_pt_info_open(struct inode * inode,struct file * file)2282 static int amdgpu_pt_info_open(struct inode *inode, struct file *file)
2283 {
2284 return single_open(file, amdgpu_pt_info_read, inode->i_private);
2285 }
2286
2287 static const struct file_operations amdgpu_pt_info_fops = {
2288 .owner = THIS_MODULE,
2289 .open = amdgpu_pt_info_open,
2290 .read = seq_read,
2291 .llseek = seq_lseek,
2292 .release = single_release,
2293 };
2294
amdgpu_mqd_info_read(struct seq_file * m,void * unused)2295 static int amdgpu_mqd_info_read(struct seq_file *m, void *unused)
2296 {
2297 struct amdgpu_usermode_queue *queue = m->private;
2298 struct amdgpu_bo *bo;
2299 int r;
2300
2301 if (!queue || !queue->mqd.obj)
2302 return -EINVAL;
2303
2304 bo = amdgpu_bo_ref(queue->mqd.obj);
2305 r = amdgpu_bo_reserve(bo, true);
2306 if (r) {
2307 amdgpu_bo_unref(&bo);
2308 return -EINVAL;
2309 }
2310
2311 seq_printf(m, "queue_type: %d\n", queue->queue_type);
2312 seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));
2313
2314 amdgpu_bo_unreserve(bo);
2315 amdgpu_bo_unref(&bo);
2316
2317 return 0;
2318 }
2319
amdgpu_mqd_info_open(struct inode * inode,struct file * file)2320 static int amdgpu_mqd_info_open(struct inode *inode, struct file *file)
2321 {
2322 return single_open(file, amdgpu_mqd_info_read, inode->i_private);
2323 }
2324
2325 static const struct file_operations amdgpu_mqd_info_fops = {
2326 .owner = THIS_MODULE,
2327 .open = amdgpu_mqd_info_open,
2328 .read = seq_read,
2329 .llseek = seq_lseek,
2330 .release = single_release,
2331 };
2332
amdgpu_debugfs_userq_init(struct drm_file * file,struct amdgpu_usermode_queue * queue,int qid)2333 void amdgpu_debugfs_userq_init(struct drm_file *file, struct amdgpu_usermode_queue *queue, int qid)
2334 {
2335 char queue_name[32];
2336
2337 scnprintf(queue_name, sizeof(queue_name), "queue_%d", qid);
2338 queue->debugfs_queue = debugfs_create_dir(queue_name, file->debugfs_client);
2339 debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
2340 }
2341
amdgpu_debugfs_vm_init(struct drm_file * file)2342 void amdgpu_debugfs_vm_init(struct drm_file *file)
2343 {
2344 debugfs_create_file("vm_pagetable_info", 0444, file->debugfs_client, file,
2345 &amdgpu_pt_info_fops);
2346 }
2347
2348 #else
amdgpu_debugfs_init(struct amdgpu_device * adev)2349 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2350 {
2351 return 0;
2352 }
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)2353 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2354 {
2355 return 0;
2356 }
amdgpu_debugfs_vm_init(struct drm_file * file)2357 void amdgpu_debugfs_vm_init(struct drm_file *file)
2358 {
2359 }
amdgpu_debugfs_userq_init(struct drm_file * file,struct amdgpu_usermode_queue * queue,int qid)2360 void amdgpu_debugfs_userq_init(struct drm_file *file,
2361 struct amdgpu_usermode_queue *queue,
2362 int qid)
2363 {
2364 }
2365 #endif
2366