1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/kthread.h> 27 #include <linux/pci.h> 28 #include <linux/uaccess.h> 29 #include <linux/security.h> 30 #include <linux/pm_runtime.h> 31 32 #include "amdgpu.h" 33 #include "amdgpu_pm.h" 34 #include "amdgpu_dm_debugfs.h" 35 #include "amdgpu_ras.h" 36 #include "amdgpu_rap.h" 37 #include "amdgpu_securedisplay.h" 38 #include "amdgpu_fw_attestation.h" 39 #include "amdgpu_umr.h" 40 41 #include "amdgpu_reset.h" 42 #include "amdgpu_psp_ta.h" 43 #include "amdgpu_userq.h" 44 45 #if defined(CONFIG_DEBUG_FS) 46 47 /* Encode milliwatts in the raw Q24.8 sensor report format used by UMR. */ 48 #define AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(power_mw) \ 49 DIV_ROUND_CLOSEST_ULL((u64)(power_mw) * BIT(8), \ 50 MILLIWATT_PER_WATT) 51 52 /** 53 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes 54 * 55 * @read: True if reading 56 * @f: open file handle 57 * @buf: User buffer to write/read to 58 * @size: Number of bytes to write/read 59 * @pos: Offset to seek to 60 * 61 * This debugfs entry has special meaning on the offset being sought. 62 * Various bits have different meanings: 63 * 64 * Bit 62: Indicates a GRBM bank switch is needed 65 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is 66 * zero) 67 * Bits 24..33: The SE or ME selector if needed 68 * Bits 34..43: The SH (or SA) or PIPE selector if needed 69 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed 70 * 71 * Bit 23: Indicates that the PM power gating lock should be held 72 * This is necessary to read registers that might be 73 * unreliable during a power gating transistion. 74 * 75 * The lower bits are the BYTE offset of the register to read. This 76 * allows reading multiple registers in a single call and having 77 * the returned size reflect that. 78 */ 79 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, 80 char __user *buf, size_t size, loff_t *pos) 81 { 82 struct amdgpu_device *adev = file_inode(f)->i_private; 83 ssize_t result = 0; 84 int r; 85 bool pm_pg_lock, use_bank, use_ring; 86 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid; 87 88 pm_pg_lock = use_bank = use_ring = false; 89 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0; 90 91 if (size & 0x3 || *pos & 0x3 || 92 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61)))) 93 return -EINVAL; 94 95 /* are we reading registers for which a PG lock is necessary? */ 96 pm_pg_lock = (*pos >> 23) & 1; 97 98 if (*pos & (1ULL << 62)) { 99 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24; 100 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34; 101 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44; 102 103 if (se_bank == 0x3FF) 104 se_bank = 0xFFFFFFFF; 105 if (sh_bank == 0x3FF) 106 sh_bank = 0xFFFFFFFF; 107 if (instance_bank == 0x3FF) 108 instance_bank = 0xFFFFFFFF; 109 use_bank = true; 110 } else if (*pos & (1ULL << 61)) { 111 112 me = (*pos & GENMASK_ULL(33, 24)) >> 24; 113 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34; 114 queue = (*pos & GENMASK_ULL(53, 44)) >> 44; 115 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54; 116 117 use_ring = true; 118 } else { 119 use_bank = use_ring = false; 120 } 121 122 *pos &= (1UL << 22) - 1; 123 124 if (size > PAGE_SIZE) 125 return -EINVAL; 126 127 /* 128 * Access the user buffer only while none of the locks below are held: 129 * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest 130 * inside it. 131 */ 132 u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL); 133 if (!data) 134 return -ENOMEM; 135 if (!read && copy_from_user(data, buf, size)) 136 return -EFAULT; 137 138 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 139 if (r < 0) { 140 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 141 return r; 142 } 143 144 r = amdgpu_virt_enable_access_debugfs(adev); 145 if (r < 0) { 146 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 147 return r; 148 } 149 150 if (use_bank) { 151 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) || 152 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) { 153 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 154 amdgpu_virt_disable_access_debugfs(adev); 155 return -EINVAL; 156 } 157 mutex_lock(&adev->grbm_idx_mutex); 158 amdgpu_gfx_select_se_sh(adev, se_bank, 159 sh_bank, instance_bank, 0); 160 } else if (use_ring) { 161 mutex_lock(&adev->srbm_mutex); 162 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0); 163 } 164 165 if (pm_pg_lock) 166 mutex_lock(&adev->pm.mutex); 167 168 while (size) { 169 if (read) 170 data[result >> 2] = RREG32(*pos >> 2); 171 else 172 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, data[result >> 2], 0); 173 174 result += 4; 175 *pos += 4; 176 size -= 4; 177 } 178 179 if (use_bank) { 180 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 181 mutex_unlock(&adev->grbm_idx_mutex); 182 } else if (use_ring) { 183 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); 184 mutex_unlock(&adev->srbm_mutex); 185 } 186 187 if (pm_pg_lock) 188 mutex_unlock(&adev->pm.mutex); 189 190 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 191 192 amdgpu_virt_disable_access_debugfs(adev); 193 194 if (read && copy_to_user(buf, data, result)) 195 return -EFAULT; 196 197 return result; 198 } 199 200 /* 201 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers 202 */ 203 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, 204 size_t size, loff_t *pos) 205 { 206 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos); 207 } 208 209 /* 210 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers 211 */ 212 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf, 213 size_t size, loff_t *pos) 214 { 215 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos); 216 } 217 218 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file) 219 { 220 struct amdgpu_debugfs_regs2_data *rd; 221 222 rd = kzalloc_obj(*rd); 223 if (!rd) 224 return -ENOMEM; 225 rd->adev = file_inode(file)->i_private; 226 file->private_data = rd; 227 mutex_init(&rd->lock); 228 229 return 0; 230 } 231 232 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file) 233 { 234 struct amdgpu_debugfs_regs2_data *rd = file->private_data; 235 236 mutex_destroy(&rd->lock); 237 kfree(file->private_data); 238 return 0; 239 } 240 241 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en) 242 { 243 struct amdgpu_debugfs_regs2_data *rd = f->private_data; 244 struct amdgpu_device *adev = rd->adev; 245 ssize_t result = 0; 246 int r; 247 248 if (size & 0x3 || offset & 0x3) 249 return -EINVAL; 250 251 if (size > PAGE_SIZE) 252 return -EINVAL; 253 254 /* 255 * Access the user buffer only while none of the locks below are held: 256 * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest 257 * inside it. 258 */ 259 u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL); 260 if (!data) 261 return -ENOMEM; 262 if (write_en && copy_from_user(data, buf, size)) 263 return -EFAULT; 264 265 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 266 if (r < 0) { 267 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 268 return r; 269 } 270 271 r = amdgpu_virt_enable_access_debugfs(adev); 272 if (r < 0) { 273 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 274 return r; 275 } 276 277 mutex_lock(&rd->lock); 278 279 if (rd->id.use_grbm) { 280 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) || 281 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) { 282 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 283 amdgpu_virt_disable_access_debugfs(adev); 284 mutex_unlock(&rd->lock); 285 return -EINVAL; 286 } 287 mutex_lock(&adev->grbm_idx_mutex); 288 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se, 289 rd->id.grbm.sh, 290 rd->id.grbm.instance, rd->id.xcc_id); 291 } 292 293 if (rd->id.use_srbm) { 294 mutex_lock(&adev->srbm_mutex); 295 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe, 296 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id); 297 } 298 299 if (rd->id.pg_lock) 300 mutex_lock(&adev->pm.mutex); 301 302 while (size) { 303 if (!write_en) 304 data[result >> 2] = RREG32(offset >> 2); 305 else 306 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, data[result >> 2], 307 rd->id.xcc_id); 308 offset += 4; 309 size -= 4; 310 result += 4; 311 } 312 313 if (rd->id.use_grbm) { 314 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id); 315 mutex_unlock(&adev->grbm_idx_mutex); 316 } 317 318 if (rd->id.use_srbm) { 319 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id); 320 mutex_unlock(&adev->srbm_mutex); 321 } 322 323 if (rd->id.pg_lock) 324 mutex_unlock(&adev->pm.mutex); 325 326 mutex_unlock(&rd->lock); 327 328 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 329 330 amdgpu_virt_disable_access_debugfs(adev); 331 332 if (!write_en && copy_to_user(buf, data, result)) 333 return -EFAULT; 334 335 return result; 336 } 337 338 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data) 339 { 340 struct amdgpu_debugfs_regs2_data *rd = f->private_data; 341 struct amdgpu_debugfs_regs2_iocdata v1_data; 342 int r; 343 344 mutex_lock(&rd->lock); 345 346 switch (cmd) { 347 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2: 348 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data, 349 sizeof(rd->id)); 350 if (r) 351 r = -EINVAL; 352 goto done; 353 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE: 354 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data, 355 sizeof(v1_data)); 356 if (r) { 357 r = -EINVAL; 358 goto done; 359 } 360 goto v1_copy; 361 default: 362 r = -EINVAL; 363 goto done; 364 } 365 366 v1_copy: 367 rd->id.use_srbm = v1_data.use_srbm; 368 rd->id.use_grbm = v1_data.use_grbm; 369 rd->id.pg_lock = v1_data.pg_lock; 370 rd->id.grbm.se = v1_data.grbm.se; 371 rd->id.grbm.sh = v1_data.grbm.sh; 372 rd->id.grbm.instance = v1_data.grbm.instance; 373 rd->id.srbm.me = v1_data.srbm.me; 374 rd->id.srbm.pipe = v1_data.srbm.pipe; 375 rd->id.srbm.queue = v1_data.srbm.queue; 376 rd->id.xcc_id = 0; 377 done: 378 mutex_unlock(&rd->lock); 379 return r; 380 } 381 382 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos) 383 { 384 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0); 385 } 386 387 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) 388 { 389 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1); 390 } 391 392 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file) 393 { 394 struct amdgpu_debugfs_gprwave_data *rd; 395 396 rd = kzalloc_obj(*rd); 397 if (!rd) 398 return -ENOMEM; 399 rd->adev = file_inode(file)->i_private; 400 file->private_data = rd; 401 mutex_init(&rd->lock); 402 403 return 0; 404 } 405 406 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file) 407 { 408 struct amdgpu_debugfs_gprwave_data *rd = file->private_data; 409 410 mutex_destroy(&rd->lock); 411 kfree(file->private_data); 412 return 0; 413 } 414 415 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos) 416 { 417 struct amdgpu_debugfs_gprwave_data *rd = f->private_data; 418 struct amdgpu_device *adev = rd->adev; 419 ssize_t result = 0; 420 int r; 421 uint32_t *data, x; 422 423 if (size > 4096 || size & 0x3 || *pos & 0x3) 424 return -EINVAL; 425 426 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 427 if (r < 0) { 428 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 429 return r; 430 } 431 432 r = amdgpu_virt_enable_access_debugfs(adev); 433 if (r < 0) { 434 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 435 return r; 436 } 437 438 data = kcalloc(1024, sizeof(*data), GFP_KERNEL); 439 if (!data) { 440 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 441 amdgpu_virt_disable_access_debugfs(adev); 442 return -ENOMEM; 443 } 444 445 /* switch to the specific se/sh/cu */ 446 mutex_lock(&adev->grbm_idx_mutex); 447 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id); 448 449 if (!rd->id.gpr_or_wave) { 450 x = 0; 451 if (adev->gfx.funcs->read_wave_data) 452 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x); 453 } else { 454 x = size >> 2; 455 if (rd->id.gpr.vpgr_or_sgpr) { 456 if (adev->gfx.funcs->read_wave_vgprs) 457 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data); 458 } else { 459 if (adev->gfx.funcs->read_wave_sgprs) 460 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data); 461 } 462 } 463 464 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id); 465 mutex_unlock(&adev->grbm_idx_mutex); 466 467 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 468 469 if (!x) { 470 result = -EINVAL; 471 goto done; 472 } 473 474 while (size && (*pos < x * 4)) { 475 uint32_t value; 476 477 value = data[*pos >> 2]; 478 r = put_user(value, (uint32_t *)buf); 479 if (r) { 480 result = r; 481 goto done; 482 } 483 484 result += 4; 485 buf += 4; 486 *pos += 4; 487 size -= 4; 488 } 489 490 done: 491 amdgpu_virt_disable_access_debugfs(adev); 492 kfree(data); 493 return result; 494 } 495 496 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data) 497 { 498 struct amdgpu_debugfs_gprwave_data *rd = f->private_data; 499 int r = 0; 500 501 mutex_lock(&rd->lock); 502 503 switch (cmd) { 504 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE: 505 if (copy_from_user(&rd->id, 506 (struct amdgpu_debugfs_gprwave_iocdata *)data, 507 sizeof(rd->id))) 508 r = -EFAULT; 509 goto done; 510 default: 511 r = -EINVAL; 512 goto done; 513 } 514 515 done: 516 mutex_unlock(&rd->lock); 517 return r; 518 } 519 520 521 522 523 /** 524 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register 525 * 526 * @f: open file handle 527 * @buf: User buffer to store read data in 528 * @size: Number of bytes to read 529 * @pos: Offset to seek to 530 * 531 * The lower bits are the BYTE offset of the register to read. This 532 * allows reading multiple registers in a single call and having 533 * the returned size reflect that. 534 */ 535 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, 536 size_t size, loff_t *pos) 537 { 538 struct amdgpu_device *adev = file_inode(f)->i_private; 539 ssize_t result = 0; 540 int r; 541 542 if (size & 0x3 || *pos & 0x3) 543 return -EINVAL; 544 545 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 546 if (r < 0) { 547 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 548 return r; 549 } 550 551 r = amdgpu_virt_enable_access_debugfs(adev); 552 if (r < 0) { 553 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 554 return r; 555 } 556 557 while (size) { 558 uint32_t value; 559 560 if (upper_32_bits(*pos)) 561 value = RREG32_PCIE_EXT(*pos); 562 else 563 value = RREG32_PCIE(*pos); 564 565 r = put_user(value, (uint32_t *)buf); 566 if (r) 567 goto out; 568 569 result += 4; 570 buf += 4; 571 *pos += 4; 572 size -= 4; 573 } 574 575 r = result; 576 out: 577 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 578 amdgpu_virt_disable_access_debugfs(adev); 579 return r; 580 } 581 582 /** 583 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register 584 * 585 * @f: open file handle 586 * @buf: User buffer to write data from 587 * @size: Number of bytes to write 588 * @pos: Offset to seek to 589 * 590 * The lower bits are the BYTE offset of the register to write. This 591 * allows writing multiple registers in a single call and having 592 * the returned size reflect that. 593 */ 594 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf, 595 size_t size, loff_t *pos) 596 { 597 struct amdgpu_device *adev = file_inode(f)->i_private; 598 ssize_t result = 0; 599 int r; 600 601 if (size & 0x3 || *pos & 0x3) 602 return -EINVAL; 603 604 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 605 if (r < 0) { 606 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 607 return r; 608 } 609 610 r = amdgpu_virt_enable_access_debugfs(adev); 611 if (r < 0) { 612 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 613 return r; 614 } 615 616 while (size) { 617 uint32_t value; 618 619 r = get_user(value, (uint32_t *)buf); 620 if (r) 621 goto out; 622 623 if (upper_32_bits(*pos)) 624 WREG32_PCIE_EXT(*pos, value); 625 else 626 WREG32_PCIE(*pos, value); 627 628 result += 4; 629 buf += 4; 630 *pos += 4; 631 size -= 4; 632 } 633 634 r = result; 635 out: 636 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 637 amdgpu_virt_disable_access_debugfs(adev); 638 return r; 639 } 640 641 /** 642 * amdgpu_debugfs_regs_pcie64_read - Read from a 64-bit PCIE register 643 * 644 * @f: open file handle 645 * @buf: User buffer to store read data in 646 * @size: Number of bytes to read 647 * @pos: Offset to seek to 648 */ 649 static ssize_t amdgpu_debugfs_regs_pcie64_read(struct file *f, char __user *buf, 650 size_t size, loff_t *pos) 651 { 652 struct amdgpu_device *adev = file_inode(f)->i_private; 653 ssize_t result = 0; 654 int r; 655 656 if (size & 0x7 || *pos & 0x7) 657 return -EINVAL; 658 659 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 660 if (r < 0) { 661 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 662 return r; 663 } 664 665 r = amdgpu_virt_enable_access_debugfs(adev); 666 if (r < 0) { 667 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 668 return r; 669 } 670 671 while (size) { 672 uint64_t value; 673 674 value = RREG64_PCIE_EXT(*pos); 675 676 r = put_user(value, (uint64_t *)buf); 677 if (r) 678 goto out; 679 680 result += 8; 681 buf += 8; 682 *pos += 8; 683 size -= 8; 684 } 685 686 r = result; 687 out: 688 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 689 amdgpu_virt_disable_access_debugfs(adev); 690 return r; 691 } 692 693 /** 694 * amdgpu_debugfs_regs_pcie64_write - Write to a 64-bit PCIE register 695 * 696 * @f: open file handle 697 * @buf: User buffer to write data from 698 * @size: Number of bytes to write 699 * @pos: Offset to seek to 700 */ 701 static ssize_t amdgpu_debugfs_regs_pcie64_write(struct file *f, const char __user *buf, 702 size_t size, loff_t *pos) 703 { 704 struct amdgpu_device *adev = file_inode(f)->i_private; 705 ssize_t result = 0; 706 int r; 707 708 if (size & 0x7 || *pos & 0x7) 709 return -EINVAL; 710 711 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 712 if (r < 0) { 713 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 714 return r; 715 } 716 717 r = amdgpu_virt_enable_access_debugfs(adev); 718 if (r < 0) { 719 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 720 return r; 721 } 722 723 while (size) { 724 uint64_t value; 725 726 r = get_user(value, (uint64_t *)buf); 727 if (r) 728 goto out; 729 730 WREG64_PCIE_EXT(*pos, value); 731 732 result += 8; 733 buf += 8; 734 *pos += 8; 735 size -= 8; 736 } 737 738 r = result; 739 out: 740 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 741 amdgpu_virt_disable_access_debugfs(adev); 742 return r; 743 } 744 745 /** 746 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register 747 * 748 * @f: open file handle 749 * @buf: User buffer to store read data in 750 * @size: Number of bytes to read 751 * @pos: Offset to seek to 752 * 753 * The lower bits are the BYTE offset of the register to read. This 754 * allows reading multiple registers in a single call and having 755 * the returned size reflect that. 756 */ 757 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, 758 size_t size, loff_t *pos) 759 { 760 struct amdgpu_device *adev = file_inode(f)->i_private; 761 ssize_t result = 0; 762 int r; 763 764 if (size & 0x3 || *pos & 0x3) 765 return -EINVAL; 766 767 if (!adev->reg.didt.rreg) 768 return -EOPNOTSUPP; 769 770 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 771 if (r < 0) { 772 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 773 return r; 774 } 775 776 r = amdgpu_virt_enable_access_debugfs(adev); 777 if (r < 0) { 778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 779 return r; 780 } 781 782 while (size) { 783 uint32_t value; 784 785 value = RREG32_DIDT(*pos >> 2); 786 r = put_user(value, (uint32_t *)buf); 787 if (r) 788 goto out; 789 790 result += 4; 791 buf += 4; 792 *pos += 4; 793 size -= 4; 794 } 795 796 r = result; 797 out: 798 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 799 amdgpu_virt_disable_access_debugfs(adev); 800 return r; 801 } 802 803 /** 804 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register 805 * 806 * @f: open file handle 807 * @buf: User buffer to write data from 808 * @size: Number of bytes to write 809 * @pos: Offset to seek to 810 * 811 * The lower bits are the BYTE offset of the register to write. This 812 * allows writing multiple registers in a single call and having 813 * the returned size reflect that. 814 */ 815 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf, 816 size_t size, loff_t *pos) 817 { 818 struct amdgpu_device *adev = file_inode(f)->i_private; 819 ssize_t result = 0; 820 int r; 821 822 if (size & 0x3 || *pos & 0x3) 823 return -EINVAL; 824 825 if (!adev->reg.didt.wreg) 826 return -EOPNOTSUPP; 827 828 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 829 if (r < 0) { 830 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 831 return r; 832 } 833 834 r = amdgpu_virt_enable_access_debugfs(adev); 835 if (r < 0) { 836 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 837 return r; 838 } 839 840 while (size) { 841 uint32_t value; 842 843 r = get_user(value, (uint32_t *)buf); 844 if (r) 845 goto out; 846 847 WREG32_DIDT(*pos >> 2, value); 848 849 result += 4; 850 buf += 4; 851 *pos += 4; 852 size -= 4; 853 } 854 855 r = result; 856 out: 857 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 858 amdgpu_virt_disable_access_debugfs(adev); 859 return r; 860 } 861 862 /** 863 * amdgpu_debugfs_regs_smc_read - Read from a SMC register 864 * 865 * @f: open file handle 866 * @buf: User buffer to store read data in 867 * @size: Number of bytes to read 868 * @pos: Offset to seek to 869 * 870 * The lower bits are the BYTE offset of the register to read. This 871 * allows reading multiple registers in a single call and having 872 * the returned size reflect that. 873 */ 874 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, 875 size_t size, loff_t *pos) 876 { 877 struct amdgpu_device *adev = file_inode(f)->i_private; 878 ssize_t result = 0; 879 int r; 880 881 if (!adev->reg.smc.rreg) 882 return -EOPNOTSUPP; 883 884 if (size & 0x3 || *pos & 0x3) 885 return -EINVAL; 886 887 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 888 if (r < 0) { 889 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 890 return r; 891 } 892 893 r = amdgpu_virt_enable_access_debugfs(adev); 894 if (r < 0) { 895 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 896 return r; 897 } 898 899 while (size) { 900 uint32_t value; 901 902 value = RREG32_SMC(*pos); 903 r = put_user(value, (uint32_t *)buf); 904 if (r) 905 goto out; 906 907 result += 4; 908 buf += 4; 909 *pos += 4; 910 size -= 4; 911 } 912 913 r = result; 914 out: 915 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 916 amdgpu_virt_disable_access_debugfs(adev); 917 return r; 918 } 919 920 /** 921 * amdgpu_debugfs_regs_smc_write - Write to a SMC register 922 * 923 * @f: open file handle 924 * @buf: User buffer to write data from 925 * @size: Number of bytes to write 926 * @pos: Offset to seek to 927 * 928 * The lower bits are the BYTE offset of the register to write. This 929 * allows writing multiple registers in a single call and having 930 * the returned size reflect that. 931 */ 932 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf, 933 size_t size, loff_t *pos) 934 { 935 struct amdgpu_device *adev = file_inode(f)->i_private; 936 ssize_t result = 0; 937 int r; 938 939 if (!adev->reg.smc.wreg) 940 return -EOPNOTSUPP; 941 942 if (size & 0x3 || *pos & 0x3) 943 return -EINVAL; 944 945 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 946 if (r < 0) { 947 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 948 return r; 949 } 950 951 r = amdgpu_virt_enable_access_debugfs(adev); 952 if (r < 0) { 953 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 954 return r; 955 } 956 957 while (size) { 958 uint32_t value; 959 960 r = get_user(value, (uint32_t *)buf); 961 if (r) 962 goto out; 963 964 WREG32_SMC(*pos, value); 965 966 result += 4; 967 buf += 4; 968 *pos += 4; 969 size -= 4; 970 } 971 972 r = result; 973 out: 974 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 975 amdgpu_virt_disable_access_debugfs(adev); 976 return r; 977 } 978 979 /** 980 * amdgpu_debugfs_gca_config_read - Read from gfx config data 981 * 982 * @f: open file handle 983 * @buf: User buffer to store read data in 984 * @size: Number of bytes to read 985 * @pos: Offset to seek to 986 * 987 * This file is used to access configuration data in a somewhat 988 * stable fashion. The format is a series of DWORDs with the first 989 * indicating which revision it is. New content is appended to the 990 * end so that older software can still read the data. 991 */ 992 993 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf, 994 size_t size, loff_t *pos) 995 { 996 struct amdgpu_device *adev = file_inode(f)->i_private; 997 ssize_t result = 0; 998 int r; 999 uint32_t *config, no_regs = 0; 1000 1001 if (size & 0x3 || *pos & 0x3) 1002 return -EINVAL; 1003 1004 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL); 1005 if (!config) 1006 return -ENOMEM; 1007 1008 /* version, increment each time something is added */ 1009 config[no_regs++] = 5; 1010 config[no_regs++] = adev->gfx.config.max_shader_engines; 1011 config[no_regs++] = adev->gfx.config.max_tile_pipes; 1012 config[no_regs++] = adev->gfx.config.max_cu_per_sh; 1013 config[no_regs++] = adev->gfx.config.max_sh_per_se; 1014 config[no_regs++] = adev->gfx.config.max_backends_per_se; 1015 config[no_regs++] = adev->gfx.config.max_texture_channel_caches; 1016 config[no_regs++] = adev->gfx.config.max_gprs; 1017 config[no_regs++] = adev->gfx.config.max_gs_threads; 1018 config[no_regs++] = adev->gfx.config.max_hw_contexts; 1019 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend; 1020 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend; 1021 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size; 1022 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size; 1023 config[no_regs++] = adev->gfx.config.num_tile_pipes; 1024 config[no_regs++] = adev->gfx.config.backend_enable_mask; 1025 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes; 1026 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb; 1027 config[no_regs++] = adev->gfx.config.shader_engine_tile_size; 1028 config[no_regs++] = adev->gfx.config.num_gpus; 1029 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size; 1030 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg; 1031 config[no_regs++] = adev->gfx.config.gb_addr_config; 1032 config[no_regs++] = adev->gfx.config.num_rbs; 1033 1034 /* rev==1 */ 1035 config[no_regs++] = adev->rev_id; 1036 config[no_regs++] = adev->pg_flags; 1037 config[no_regs++] = lower_32_bits(adev->cg_flags); 1038 1039 /* rev==2 */ 1040 config[no_regs++] = adev->family; 1041 config[no_regs++] = adev->external_rev_id; 1042 1043 /* rev==3 */ 1044 config[no_regs++] = adev->pdev->device; 1045 config[no_regs++] = adev->pdev->revision; 1046 config[no_regs++] = adev->pdev->subsystem_device; 1047 config[no_regs++] = adev->pdev->subsystem_vendor; 1048 1049 /* rev==4 APU flag */ 1050 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0; 1051 1052 /* rev==5 PG/CG flag upper 32bit */ 1053 config[no_regs++] = 0; 1054 config[no_regs++] = upper_32_bits(adev->cg_flags); 1055 1056 while (size && (*pos < no_regs * 4)) { 1057 uint32_t value; 1058 1059 value = config[*pos >> 2]; 1060 r = put_user(value, (uint32_t *)buf); 1061 if (r) { 1062 kfree(config); 1063 return r; 1064 } 1065 1066 result += 4; 1067 buf += 4; 1068 *pos += 4; 1069 size -= 4; 1070 } 1071 1072 kfree(config); 1073 return result; 1074 } 1075 1076 /** 1077 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors 1078 * 1079 * @f: open file handle 1080 * @buf: User buffer to store read data in 1081 * @size: Number of bytes to read 1082 * @pos: Offset to seek to 1083 * 1084 * The offset is treated as the BYTE address of one of the sensors 1085 * enumerated in amd/include/kgd_pp_interface.h under the 1086 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK 1087 * you would use the offset 3 * 4 = 12. 1088 */ 1089 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf, 1090 size_t size, loff_t *pos) 1091 { 1092 struct amdgpu_device *adev = file_inode(f)->i_private; 1093 int idx, x, outsize, r, valuesize; 1094 uint32_t values[16]; 1095 1096 if (size & 3 || *pos & 0x3) 1097 return -EINVAL; 1098 1099 if (!adev->pm.dpm_enabled) 1100 return -EINVAL; 1101 1102 /* convert offset to sensor number */ 1103 idx = *pos >> 2; 1104 1105 valuesize = sizeof(values); 1106 1107 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1108 if (r < 0) { 1109 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1110 return r; 1111 } 1112 1113 r = amdgpu_virt_enable_access_debugfs(adev); 1114 if (r < 0) { 1115 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1116 return r; 1117 } 1118 1119 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize); 1120 1121 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1122 1123 if (r) { 1124 amdgpu_virt_disable_access_debugfs(adev); 1125 return r; 1126 } 1127 1128 if (idx == AMDGPU_PP_SENSOR_GPU_AVG_POWER || 1129 idx == AMDGPU_PP_SENSOR_GPU_INPUT_POWER) 1130 values[0] = AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(values[0]); 1131 1132 if (size > valuesize) { 1133 amdgpu_virt_disable_access_debugfs(adev); 1134 return -EINVAL; 1135 } 1136 1137 outsize = 0; 1138 x = 0; 1139 if (!r) { 1140 while (size) { 1141 r = put_user(values[x++], (int32_t *)buf); 1142 buf += 4; 1143 size -= 4; 1144 outsize += 4; 1145 } 1146 } 1147 1148 amdgpu_virt_disable_access_debugfs(adev); 1149 return !r ? outsize : r; 1150 } 1151 1152 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data 1153 * 1154 * @f: open file handle 1155 * @buf: User buffer to store read data in 1156 * @size: Number of bytes to read 1157 * @pos: Offset to seek to 1158 * 1159 * The offset being sought changes which wave that the status data 1160 * will be returned for. The bits are used as follows: 1161 * 1162 * Bits 0..6: Byte offset into data 1163 * Bits 7..14: SE selector 1164 * Bits 15..22: SH/SA selector 1165 * Bits 23..30: CU/{WGP+SIMD} selector 1166 * Bits 31..36: WAVE ID selector 1167 * Bits 37..44: SIMD ID selector 1168 * 1169 * The returned data begins with one DWORD of version information 1170 * Followed by WAVE STATUS registers relevant to the GFX IP version 1171 * being used. See gfx_v8_0_read_wave_data() for an example output. 1172 */ 1173 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, 1174 size_t size, loff_t *pos) 1175 { 1176 struct amdgpu_device *adev = f->f_inode->i_private; 1177 int r, x; 1178 ssize_t result = 0; 1179 uint32_t offset, se, sh, cu, wave, simd, data[32]; 1180 1181 if (size & 3 || *pos & 3) 1182 return -EINVAL; 1183 1184 /* decode offset */ 1185 offset = (*pos & GENMASK_ULL(6, 0)); 1186 se = (*pos & GENMASK_ULL(14, 7)) >> 7; 1187 sh = (*pos & GENMASK_ULL(22, 15)) >> 15; 1188 cu = (*pos & GENMASK_ULL(30, 23)) >> 23; 1189 wave = (*pos & GENMASK_ULL(36, 31)) >> 31; 1190 simd = (*pos & GENMASK_ULL(44, 37)) >> 37; 1191 1192 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1193 if (r < 0) { 1194 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1195 return r; 1196 } 1197 1198 r = amdgpu_virt_enable_access_debugfs(adev); 1199 if (r < 0) { 1200 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1201 return r; 1202 } 1203 1204 /* switch to the specific se/sh/cu */ 1205 mutex_lock(&adev->grbm_idx_mutex); 1206 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); 1207 1208 x = 0; 1209 if (adev->gfx.funcs->read_wave_data) 1210 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x); 1211 1212 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); 1213 mutex_unlock(&adev->grbm_idx_mutex); 1214 1215 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1216 1217 if (!x) { 1218 amdgpu_virt_disable_access_debugfs(adev); 1219 return -EINVAL; 1220 } 1221 1222 while (size && (offset < x * 4)) { 1223 uint32_t value; 1224 1225 value = data[offset >> 2]; 1226 r = put_user(value, (uint32_t *)buf); 1227 if (r) { 1228 amdgpu_virt_disable_access_debugfs(adev); 1229 return r; 1230 } 1231 1232 result += 4; 1233 buf += 4; 1234 offset += 4; 1235 size -= 4; 1236 } 1237 1238 amdgpu_virt_disable_access_debugfs(adev); 1239 return result; 1240 } 1241 1242 /** amdgpu_debugfs_gpr_read - Read wave gprs 1243 * 1244 * @f: open file handle 1245 * @buf: User buffer to store read data in 1246 * @size: Number of bytes to read 1247 * @pos: Offset to seek to 1248 * 1249 * The offset being sought changes which wave that the status data 1250 * will be returned for. The bits are used as follows: 1251 * 1252 * Bits 0..11: Byte offset into data 1253 * Bits 12..19: SE selector 1254 * Bits 20..27: SH/SA selector 1255 * Bits 28..35: CU/{WGP+SIMD} selector 1256 * Bits 36..43: WAVE ID selector 1257 * Bits 37..44: SIMD ID selector 1258 * Bits 52..59: Thread selector 1259 * Bits 60..61: Bank selector (VGPR=0,SGPR=1) 1260 * 1261 * The return data comes from the SGPR or VGPR register bank for 1262 * the selected operational unit. 1263 */ 1264 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, 1265 size_t size, loff_t *pos) 1266 { 1267 struct amdgpu_device *adev = f->f_inode->i_private; 1268 int r; 1269 ssize_t result = 0; 1270 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data; 1271 1272 if (size > 4096 || size & 3 || *pos & 3) 1273 return -EINVAL; 1274 1275 /* decode offset */ 1276 offset = (*pos & GENMASK_ULL(11, 0)) >> 2; 1277 se = (*pos & GENMASK_ULL(19, 12)) >> 12; 1278 sh = (*pos & GENMASK_ULL(27, 20)) >> 20; 1279 cu = (*pos & GENMASK_ULL(35, 28)) >> 28; 1280 wave = (*pos & GENMASK_ULL(43, 36)) >> 36; 1281 simd = (*pos & GENMASK_ULL(51, 44)) >> 44; 1282 thread = (*pos & GENMASK_ULL(59, 52)) >> 52; 1283 bank = (*pos & GENMASK_ULL(61, 60)) >> 60; 1284 1285 data = kcalloc(1024, sizeof(*data), GFP_KERNEL); 1286 if (!data) 1287 return -ENOMEM; 1288 1289 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1290 if (r < 0) 1291 goto err; 1292 1293 r = amdgpu_virt_enable_access_debugfs(adev); 1294 if (r < 0) 1295 goto err; 1296 1297 /* switch to the specific se/sh/cu */ 1298 mutex_lock(&adev->grbm_idx_mutex); 1299 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); 1300 1301 if (bank == 0) { 1302 if (adev->gfx.funcs->read_wave_vgprs) 1303 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data); 1304 } else { 1305 if (adev->gfx.funcs->read_wave_sgprs) 1306 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data); 1307 } 1308 1309 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); 1310 mutex_unlock(&adev->grbm_idx_mutex); 1311 1312 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1313 1314 while (size) { 1315 uint32_t value; 1316 1317 value = data[result >> 2]; 1318 r = put_user(value, (uint32_t *)buf); 1319 if (r) { 1320 amdgpu_virt_disable_access_debugfs(adev); 1321 goto err; 1322 } 1323 1324 result += 4; 1325 buf += 4; 1326 size -= 4; 1327 } 1328 1329 kfree(data); 1330 amdgpu_virt_disable_access_debugfs(adev); 1331 return result; 1332 1333 err: 1334 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1335 kfree(data); 1336 return r; 1337 } 1338 1339 /** 1340 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency 1341 * 1342 * @f: open file handle 1343 * @buf: User buffer to store read data in 1344 * @size: Number of bytes to read 1345 * @pos: Offset to seek to 1346 * 1347 * Read a live GFXOFF residency sample from firmware. One needs to start logging 1348 * before getting the current value. 1349 */ 1350 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf, 1351 size_t size, loff_t *pos) 1352 { 1353 struct amdgpu_device *adev = file_inode(f)->i_private; 1354 ssize_t result = 0; 1355 int r; 1356 1357 if (size & 0x3 || *pos & 0x3) 1358 return -EINVAL; 1359 1360 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1361 if (r < 0) { 1362 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1363 return r; 1364 } 1365 1366 while (size) { 1367 uint32_t value; 1368 1369 r = amdgpu_get_gfx_off_residency(adev, &value); 1370 if (r) 1371 goto out; 1372 1373 r = put_user(value, (uint32_t *)buf); 1374 if (r) 1375 goto out; 1376 1377 result += 4; 1378 buf += 4; 1379 *pos += 4; 1380 size -= 4; 1381 } 1382 1383 r = result; 1384 out: 1385 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1386 1387 return r; 1388 } 1389 1390 /** 1391 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency 1392 * 1393 * @f: open file handle 1394 * @buf: User buffer to write data from 1395 * @size: Number of bytes to write 1396 * @pos: Offset to seek to 1397 * 1398 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop 1399 */ 1400 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf, 1401 size_t size, loff_t *pos) 1402 { 1403 struct amdgpu_device *adev = file_inode(f)->i_private; 1404 ssize_t result = 0; 1405 int r; 1406 1407 if (size & 0x3 || *pos & 0x3) 1408 return -EINVAL; 1409 1410 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1411 if (r < 0) { 1412 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1413 return r; 1414 } 1415 1416 while (size) { 1417 u32 value; 1418 1419 r = get_user(value, (uint32_t *)buf); 1420 if (r) 1421 goto out; 1422 1423 amdgpu_set_gfx_off_residency(adev, value ? true : false); 1424 1425 result += 4; 1426 buf += 4; 1427 *pos += 4; 1428 size -= 4; 1429 } 1430 1431 r = result; 1432 out: 1433 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1434 1435 return r; 1436 } 1437 1438 1439 /** 1440 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count 1441 * 1442 * @f: open file handle 1443 * @buf: User buffer to store read data in 1444 * @size: Number of bytes to read 1445 * @pos: Offset to seek to 1446 */ 1447 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf, 1448 size_t size, loff_t *pos) 1449 { 1450 struct amdgpu_device *adev = file_inode(f)->i_private; 1451 ssize_t result = 0; 1452 int r; 1453 1454 if (size & 0x3 || *pos & 0x3) 1455 return -EINVAL; 1456 1457 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1458 if (r < 0) { 1459 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1460 return r; 1461 } 1462 1463 while (size) { 1464 u64 value = 0; 1465 1466 r = amdgpu_get_gfx_off_entrycount(adev, &value); 1467 if (r) 1468 goto out; 1469 1470 r = put_user(value, (u64 *)buf); 1471 if (r) 1472 goto out; 1473 1474 result += 4; 1475 buf += 4; 1476 *pos += 4; 1477 size -= 4; 1478 } 1479 1480 r = result; 1481 out: 1482 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1483 1484 return r; 1485 } 1486 1487 /** 1488 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF 1489 * 1490 * @f: open file handle 1491 * @buf: User buffer to write data from 1492 * @size: Number of bytes to write 1493 * @pos: Offset to seek to 1494 * 1495 * Write a 32-bit zero to disable or a 32-bit non-zero to enable 1496 */ 1497 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf, 1498 size_t size, loff_t *pos) 1499 { 1500 struct amdgpu_device *adev = file_inode(f)->i_private; 1501 ssize_t result = 0; 1502 int r; 1503 1504 if (size & 0x3 || *pos & 0x3) 1505 return -EINVAL; 1506 1507 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1508 if (r < 0) { 1509 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1510 return r; 1511 } 1512 1513 while (size) { 1514 uint32_t value; 1515 1516 r = get_user(value, (uint32_t *)buf); 1517 if (r) 1518 goto out; 1519 1520 amdgpu_gfx_off_ctrl(adev, value ? true : false); 1521 1522 result += 4; 1523 buf += 4; 1524 *pos += 4; 1525 size -= 4; 1526 } 1527 1528 r = result; 1529 out: 1530 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1531 1532 return r; 1533 } 1534 1535 1536 /** 1537 * amdgpu_debugfs_gfxoff_read - read gfxoff status 1538 * 1539 * @f: open file handle 1540 * @buf: User buffer to store read data in 1541 * @size: Number of bytes to read 1542 * @pos: Offset to seek to 1543 */ 1544 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, 1545 size_t size, loff_t *pos) 1546 { 1547 struct amdgpu_device *adev = file_inode(f)->i_private; 1548 ssize_t result = 0; 1549 int r; 1550 1551 if (size & 0x3 || *pos & 0x3) 1552 return -EINVAL; 1553 1554 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1555 if (r < 0) { 1556 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1557 return r; 1558 } 1559 1560 while (size) { 1561 u32 value = adev->gfx.gfx_off_state; 1562 1563 r = put_user(value, (u32 *)buf); 1564 if (r) 1565 goto out; 1566 1567 result += 4; 1568 buf += 4; 1569 *pos += 4; 1570 size -= 4; 1571 } 1572 1573 r = result; 1574 out: 1575 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1576 1577 return r; 1578 } 1579 1580 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf, 1581 size_t size, loff_t *pos) 1582 { 1583 struct amdgpu_device *adev = file_inode(f)->i_private; 1584 ssize_t result = 0; 1585 int r; 1586 1587 if (size & 0x3 || *pos & 0x3) 1588 return -EINVAL; 1589 1590 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1591 if (r < 0) { 1592 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1593 return r; 1594 } 1595 1596 while (size) { 1597 u32 value; 1598 1599 r = amdgpu_get_gfx_off_status(adev, &value); 1600 if (r) 1601 goto out; 1602 1603 r = put_user(value, (u32 *)buf); 1604 if (r) 1605 goto out; 1606 1607 result += 4; 1608 buf += 4; 1609 *pos += 4; 1610 size -= 4; 1611 } 1612 1613 r = result; 1614 out: 1615 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1616 1617 return r; 1618 } 1619 1620 static const struct file_operations amdgpu_debugfs_regs2_fops = { 1621 .owner = THIS_MODULE, 1622 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl, 1623 .read = amdgpu_debugfs_regs2_read, 1624 .write = amdgpu_debugfs_regs2_write, 1625 .open = amdgpu_debugfs_regs2_open, 1626 .release = amdgpu_debugfs_regs2_release, 1627 .llseek = default_llseek 1628 }; 1629 1630 static const struct file_operations amdgpu_debugfs_gprwave_fops = { 1631 .owner = THIS_MODULE, 1632 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl, 1633 .read = amdgpu_debugfs_gprwave_read, 1634 .open = amdgpu_debugfs_gprwave_open, 1635 .release = amdgpu_debugfs_gprwave_release, 1636 .llseek = default_llseek 1637 }; 1638 1639 static const struct file_operations amdgpu_debugfs_regs_fops = { 1640 .owner = THIS_MODULE, 1641 .read = amdgpu_debugfs_regs_read, 1642 .write = amdgpu_debugfs_regs_write, 1643 .llseek = default_llseek 1644 }; 1645 static const struct file_operations amdgpu_debugfs_regs_didt_fops = { 1646 .owner = THIS_MODULE, 1647 .read = amdgpu_debugfs_regs_didt_read, 1648 .write = amdgpu_debugfs_regs_didt_write, 1649 .llseek = default_llseek 1650 }; 1651 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = { 1652 .owner = THIS_MODULE, 1653 .read = amdgpu_debugfs_regs_pcie_read, 1654 .write = amdgpu_debugfs_regs_pcie_write, 1655 .llseek = default_llseek 1656 }; 1657 static const struct file_operations amdgpu_debugfs_regs_pcie64_fops = { 1658 .owner = THIS_MODULE, 1659 .read = amdgpu_debugfs_regs_pcie64_read, 1660 .write = amdgpu_debugfs_regs_pcie64_write, 1661 .llseek = default_llseek 1662 }; 1663 static const struct file_operations amdgpu_debugfs_regs_smc_fops = { 1664 .owner = THIS_MODULE, 1665 .read = amdgpu_debugfs_regs_smc_read, 1666 .write = amdgpu_debugfs_regs_smc_write, 1667 .llseek = default_llseek 1668 }; 1669 1670 static const struct file_operations amdgpu_debugfs_gca_config_fops = { 1671 .owner = THIS_MODULE, 1672 .read = amdgpu_debugfs_gca_config_read, 1673 .llseek = default_llseek 1674 }; 1675 1676 static const struct file_operations amdgpu_debugfs_sensors_fops = { 1677 .owner = THIS_MODULE, 1678 .read = amdgpu_debugfs_sensor_read, 1679 .llseek = default_llseek 1680 }; 1681 1682 static const struct file_operations amdgpu_debugfs_wave_fops = { 1683 .owner = THIS_MODULE, 1684 .read = amdgpu_debugfs_wave_read, 1685 .llseek = default_llseek 1686 }; 1687 static const struct file_operations amdgpu_debugfs_gpr_fops = { 1688 .owner = THIS_MODULE, 1689 .read = amdgpu_debugfs_gpr_read, 1690 .llseek = default_llseek 1691 }; 1692 1693 static const struct file_operations amdgpu_debugfs_gfxoff_fops = { 1694 .owner = THIS_MODULE, 1695 .read = amdgpu_debugfs_gfxoff_read, 1696 .write = amdgpu_debugfs_gfxoff_write, 1697 .llseek = default_llseek 1698 }; 1699 1700 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = { 1701 .owner = THIS_MODULE, 1702 .read = amdgpu_debugfs_gfxoff_status_read, 1703 .llseek = default_llseek 1704 }; 1705 1706 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = { 1707 .owner = THIS_MODULE, 1708 .read = amdgpu_debugfs_gfxoff_count_read, 1709 .llseek = default_llseek 1710 }; 1711 1712 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = { 1713 .owner = THIS_MODULE, 1714 .read = amdgpu_debugfs_gfxoff_residency_read, 1715 .write = amdgpu_debugfs_gfxoff_residency_write, 1716 .llseek = default_llseek 1717 }; 1718 1719 static const struct file_operations *debugfs_regs[] = { 1720 &amdgpu_debugfs_regs_fops, 1721 &amdgpu_debugfs_regs2_fops, 1722 &amdgpu_debugfs_gprwave_fops, 1723 &amdgpu_debugfs_regs_didt_fops, 1724 &amdgpu_debugfs_regs_pcie_fops, 1725 &amdgpu_debugfs_regs_pcie64_fops, 1726 &amdgpu_debugfs_regs_smc_fops, 1727 &amdgpu_debugfs_gca_config_fops, 1728 &amdgpu_debugfs_sensors_fops, 1729 &amdgpu_debugfs_wave_fops, 1730 &amdgpu_debugfs_gpr_fops, 1731 &amdgpu_debugfs_gfxoff_fops, 1732 &amdgpu_debugfs_gfxoff_status_fops, 1733 &amdgpu_debugfs_gfxoff_count_fops, 1734 &amdgpu_debugfs_gfxoff_residency_fops, 1735 }; 1736 1737 static const char * const debugfs_regs_names[] = { 1738 "amdgpu_regs", 1739 "amdgpu_regs2", 1740 "amdgpu_gprwave", 1741 "amdgpu_regs_didt", 1742 "amdgpu_regs_pcie", 1743 "amdgpu_regs_pcie64", 1744 "amdgpu_regs_smc", 1745 "amdgpu_gca_config", 1746 "amdgpu_sensors", 1747 "amdgpu_wave", 1748 "amdgpu_gpr", 1749 "amdgpu_gfxoff", 1750 "amdgpu_gfxoff_status", 1751 "amdgpu_gfxoff_count", 1752 "amdgpu_gfxoff_residency", 1753 }; 1754 1755 /** 1756 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide 1757 * register access. 1758 * 1759 * @adev: The device to attach the debugfs entries to 1760 */ 1761 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) 1762 { 1763 struct drm_minor *minor = adev_to_drm(adev)->primary; 1764 struct dentry *ent, *root = minor->debugfs_root; 1765 unsigned int i; 1766 1767 if (security_locked_down(LOCKDOWN_PCI_ACCESS)) { 1768 drm_info(adev_to_drm(adev), 1769 "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n"); 1770 return 0; 1771 } 1772 1773 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { 1774 ent = debugfs_create_file(debugfs_regs_names[i], 1775 S_IFREG | 0400, root, 1776 adev, debugfs_regs[i]); 1777 if (!i && !IS_ERR_OR_NULL(ent)) 1778 i_size_write(ent->d_inode, adev->rmmio_size); 1779 } 1780 1781 return 0; 1782 } 1783 1784 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) 1785 { 1786 struct amdgpu_device *adev = m->private; 1787 struct drm_device *dev = adev_to_drm(adev); 1788 int r = 0, i; 1789 1790 r = pm_runtime_get_sync(dev->dev); 1791 if (r < 0) { 1792 pm_runtime_put_autosuspend(dev->dev); 1793 return r; 1794 } 1795 1796 /* Avoid accidently unparking the sched thread during GPU reset */ 1797 r = down_write_killable(&adev->reset_domain->sem); 1798 if (r) { 1799 pm_runtime_put_autosuspend(dev->dev); 1800 return r; 1801 } 1802 1803 /* hold on the scheduler */ 1804 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 1805 struct amdgpu_ring *ring = adev->rings[i]; 1806 1807 if (!amdgpu_ring_sched_ready(ring)) 1808 continue; 1809 drm_sched_wqueue_stop(&ring->sched); 1810 } 1811 1812 seq_puts(m, "run ib test:\n"); 1813 r = amdgpu_ib_ring_tests(adev); 1814 if (r) 1815 seq_printf(m, "ib ring tests failed (%d).\n", r); 1816 else 1817 seq_puts(m, "ib ring tests passed.\n"); 1818 1819 /* go on the scheduler */ 1820 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 1821 struct amdgpu_ring *ring = adev->rings[i]; 1822 1823 if (!amdgpu_ring_sched_ready(ring)) 1824 continue; 1825 drm_sched_wqueue_start(&ring->sched); 1826 } 1827 1828 up_write(&adev->reset_domain->sem); 1829 1830 pm_runtime_put_autosuspend(dev->dev); 1831 1832 return 0; 1833 } 1834 1835 static int amdgpu_debugfs_evict_vram(void *data, u64 *val) 1836 { 1837 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1838 struct drm_device *dev = adev_to_drm(adev); 1839 int r; 1840 1841 r = pm_runtime_get_sync(dev->dev); 1842 if (r < 0) { 1843 pm_runtime_put_autosuspend(dev->dev); 1844 return r; 1845 } 1846 1847 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 1848 1849 pm_runtime_put_autosuspend(dev->dev); 1850 1851 return 0; 1852 } 1853 1854 1855 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val) 1856 { 1857 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1858 struct drm_device *dev = adev_to_drm(adev); 1859 int r; 1860 1861 r = pm_runtime_get_sync(dev->dev); 1862 if (r < 0) { 1863 pm_runtime_put_autosuspend(dev->dev); 1864 return r; 1865 } 1866 1867 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT); 1868 1869 pm_runtime_put_autosuspend(dev->dev); 1870 1871 return 0; 1872 } 1873 1874 static int amdgpu_debugfs_benchmark(void *data, u64 val) 1875 { 1876 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1877 struct drm_device *dev = adev_to_drm(adev); 1878 int r; 1879 1880 r = pm_runtime_get_sync(dev->dev); 1881 if (r < 0) { 1882 pm_runtime_put_autosuspend(dev->dev); 1883 return r; 1884 } 1885 1886 r = amdgpu_benchmark(adev, val); 1887 1888 pm_runtime_put_autosuspend(dev->dev); 1889 1890 return r; 1891 } 1892 1893 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused) 1894 { 1895 struct amdgpu_device *adev = m->private; 1896 struct drm_device *dev = adev_to_drm(adev); 1897 struct drm_file *file; 1898 int r; 1899 1900 r = mutex_lock_interruptible(&dev->filelist_mutex); 1901 if (r) 1902 return r; 1903 1904 list_for_each_entry(file, &dev->filelist, lhead) { 1905 struct amdgpu_fpriv *fpriv = file->driver_priv; 1906 struct amdgpu_vm *vm = &fpriv->vm; 1907 struct amdgpu_task_info *ti; 1908 1909 ti = amdgpu_vm_get_task_info_vm(vm); 1910 if (ti) { 1911 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->task.pid, ti->process_name); 1912 amdgpu_vm_put_task_info(ti); 1913 } 1914 1915 r = amdgpu_bo_reserve(vm->root.bo, true); 1916 if (r) 1917 break; 1918 amdgpu_debugfs_vm_bo_info(vm, m); 1919 amdgpu_bo_unreserve(vm->root.bo); 1920 } 1921 1922 mutex_unlock(&dev->filelist_mutex); 1923 1924 return r; 1925 } 1926 1927 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib); 1928 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info); 1929 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram, 1930 NULL, "%lld\n"); 1931 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt, 1932 NULL, "%lld\n"); 1933 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark, 1934 "%lld\n"); 1935 1936 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring, 1937 struct dma_fence **fences) 1938 { 1939 struct amdgpu_fence_driver *drv = &ring->fence_drv; 1940 uint32_t sync_seq, last_seq; 1941 1942 last_seq = atomic_read(&ring->fence_drv.last_seq); 1943 sync_seq = ring->fence_drv.sync_seq; 1944 1945 last_seq &= drv->num_fences_mask; 1946 sync_seq &= drv->num_fences_mask; 1947 1948 do { 1949 struct dma_fence *fence, **ptr; 1950 1951 ++last_seq; 1952 last_seq &= drv->num_fences_mask; 1953 ptr = &drv->fences[last_seq]; 1954 1955 fence = rcu_dereference_protected(*ptr, 1); 1956 RCU_INIT_POINTER(*ptr, NULL); 1957 1958 if (!fence) 1959 continue; 1960 1961 fences[last_seq] = fence; 1962 1963 } while (last_seq != sync_seq); 1964 } 1965 1966 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences, 1967 int length) 1968 { 1969 int i; 1970 struct dma_fence *fence; 1971 1972 for (i = 0; i < length; i++) { 1973 fence = fences[i]; 1974 if (!fence) 1975 continue; 1976 dma_fence_signal(fence); 1977 dma_fence_put(fence); 1978 } 1979 } 1980 1981 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) 1982 { 1983 struct drm_sched_job *s_job; 1984 struct dma_fence *fence; 1985 1986 spin_lock(&sched->job_list_lock); 1987 list_for_each_entry(s_job, &sched->pending_list, list) { 1988 fence = sched->ops->run_job(s_job); 1989 dma_fence_put(fence); 1990 } 1991 spin_unlock(&sched->job_list_lock); 1992 } 1993 1994 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) 1995 { 1996 struct amdgpu_job *job; 1997 struct drm_sched_job *s_job, *tmp; 1998 uint32_t preempt_seq; 1999 struct dma_fence *fence, **ptr; 2000 struct amdgpu_fence_driver *drv = &ring->fence_drv; 2001 struct drm_gpu_scheduler *sched = &ring->sched; 2002 bool preempted = true; 2003 2004 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) 2005 return; 2006 2007 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2)); 2008 if (preempt_seq <= atomic_read(&drv->last_seq)) { 2009 preempted = false; 2010 goto no_preempt; 2011 } 2012 2013 preempt_seq &= drv->num_fences_mask; 2014 ptr = &drv->fences[preempt_seq]; 2015 fence = rcu_dereference_protected(*ptr, 1); 2016 2017 no_preempt: 2018 spin_lock(&sched->job_list_lock); 2019 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { 2020 if (dma_fence_is_signaled(&s_job->s_fence->finished)) { 2021 /* remove job from ring_mirror_list */ 2022 list_del_init(&s_job->list); 2023 sched->ops->free_job(s_job); 2024 continue; 2025 } 2026 job = to_amdgpu_job(s_job); 2027 if (preempted && (&job->hw_fence->base) == fence) 2028 /* mark the job as preempted */ 2029 job->preemption_status |= AMDGPU_IB_PREEMPTED; 2030 } 2031 spin_unlock(&sched->job_list_lock); 2032 } 2033 2034 static int amdgpu_debugfs_ib_preempt(void *data, u64 val) 2035 { 2036 int r, length; 2037 struct amdgpu_ring *ring; 2038 struct dma_fence **fences = NULL; 2039 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2040 2041 if (val >= AMDGPU_MAX_RINGS) 2042 return -EINVAL; 2043 2044 ring = adev->rings[val]; 2045 2046 if (!amdgpu_ring_sched_ready(ring) || 2047 !ring->funcs->preempt_ib) 2048 return -EINVAL; 2049 2050 /* the last preemption failed */ 2051 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr)) 2052 return -EBUSY; 2053 2054 length = ring->fence_drv.num_fences_mask + 1; 2055 fences = kcalloc(length, sizeof(void *), GFP_KERNEL); 2056 if (!fences) 2057 return -ENOMEM; 2058 2059 /* Avoid accidently unparking the sched thread during GPU reset */ 2060 r = down_read_killable(&adev->reset_domain->sem); 2061 if (r) 2062 goto pro_end; 2063 2064 /* stop the scheduler */ 2065 drm_sched_wqueue_stop(&ring->sched); 2066 2067 /* preempt the IB */ 2068 r = amdgpu_ring_preempt_ib(ring); 2069 if (r) { 2070 drm_warn(adev_to_drm(adev), "failed to preempt ring %d\n", ring->idx); 2071 goto failure; 2072 } 2073 2074 amdgpu_fence_process(ring); 2075 2076 if (atomic_read(&ring->fence_drv.last_seq) != 2077 ring->fence_drv.sync_seq) { 2078 drm_info(adev_to_drm(adev), "ring %d was preempted\n", ring->idx); 2079 2080 amdgpu_ib_preempt_mark_partial_job(ring); 2081 2082 /* swap out the old fences */ 2083 amdgpu_ib_preempt_fences_swap(ring, fences); 2084 2085 amdgpu_fence_driver_force_completion(ring, NULL); 2086 2087 /* resubmit unfinished jobs */ 2088 amdgpu_ib_preempt_job_recovery(&ring->sched); 2089 2090 /* wait for jobs finished */ 2091 amdgpu_fence_wait_empty(ring); 2092 2093 /* signal the old fences */ 2094 amdgpu_ib_preempt_signal_fences(fences, length); 2095 } 2096 2097 failure: 2098 /* restart the scheduler */ 2099 drm_sched_wqueue_start(&ring->sched); 2100 2101 up_read(&adev->reset_domain->sem); 2102 2103 pro_end: 2104 kfree(fences); 2105 2106 return r; 2107 } 2108 2109 static int amdgpu_debugfs_sclk_set(void *data, u64 val) 2110 { 2111 int ret = 0; 2112 uint32_t max_freq, min_freq; 2113 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2114 2115 if (amdgpu_sriov_multi_vf_mode(adev)) 2116 return -EINVAL; 2117 2118 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); 2119 if (ret < 0) { 2120 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 2121 return ret; 2122 } 2123 2124 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq); 2125 if (ret == -EOPNOTSUPP) { 2126 ret = 0; 2127 goto out; 2128 } 2129 if (ret || val > max_freq || val < min_freq) { 2130 ret = -EINVAL; 2131 goto out; 2132 } 2133 2134 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val); 2135 if (ret) 2136 ret = -EINVAL; 2137 2138 out: 2139 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 2140 2141 return ret; 2142 } 2143 2144 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL, 2145 amdgpu_debugfs_ib_preempt, "%llu\n"); 2146 2147 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL, 2148 amdgpu_debugfs_sclk_set, "%llu\n"); 2149 2150 int amdgpu_debugfs_init(struct amdgpu_device *adev) 2151 { 2152 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root; 2153 struct dentry *ent; 2154 int r, i; 2155 2156 if (!debugfs_initialized()) 2157 return 0; 2158 2159 debugfs_create_x32("amdgpu_smu_debug", 0600, root, 2160 &adev->pm.smu_debug_mask); 2161 2162 debugfs_create_x64("unique_id", 0444, root, &adev->unique_id); 2163 debugfs_create_x8("unitid", 0444, root, &adev->unitid); 2164 2165 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev, 2166 &fops_ib_preempt); 2167 if (IS_ERR(ent)) { 2168 drm_err(adev_to_drm(adev), 2169 "unable to create amdgpu_preempt_ib debugsfs file\n"); 2170 return PTR_ERR(ent); 2171 } 2172 2173 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev, 2174 &fops_sclk_set); 2175 if (IS_ERR(ent)) { 2176 drm_err(adev_to_drm(adev), 2177 "unable to create amdgpu_set_sclk debugsfs file\n"); 2178 return PTR_ERR(ent); 2179 } 2180 2181 /* Register debugfs entries for amdgpu_ttm */ 2182 amdgpu_ttm_debugfs_init(adev); 2183 amdgpu_debugfs_pm_init(adev); 2184 amdgpu_debugfs_sa_init(adev); 2185 amdgpu_debugfs_fence_init(adev); 2186 amdgpu_debugfs_gem_init(adev); 2187 2188 r = amdgpu_debugfs_regs_init(adev); 2189 if (r) 2190 drm_err(adev_to_drm(adev), "registering register debugfs failed (%d).\n", r); 2191 2192 amdgpu_debugfs_firmware_init(adev); 2193 amdgpu_ta_if_debugfs_init(adev); 2194 2195 amdgpu_debugfs_mes_event_log_init(adev); 2196 2197 #if defined(CONFIG_DRM_AMD_DC) 2198 if (adev->dc_enabled) 2199 dtn_debugfs_init(adev); 2200 #endif 2201 2202 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2203 struct amdgpu_ring *ring = adev->rings[i]; 2204 2205 if (!ring) 2206 continue; 2207 if (ring == &adev->cper.ring_buf && !adev->cper.enabled) 2208 continue; 2209 2210 amdgpu_debugfs_ring_init(adev, ring); 2211 } 2212 2213 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 2214 if (!amdgpu_vcnfw_log) 2215 break; 2216 2217 if (adev->vcn.harvest_config & (1 << i)) 2218 continue; 2219 2220 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]); 2221 } 2222 2223 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog) 2224 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm); 2225 2226 amdgpu_debugfs_vcn_sched_mask_init(adev); 2227 amdgpu_debugfs_jpeg_sched_mask_init(adev); 2228 amdgpu_debugfs_gfx_sched_mask_init(adev); 2229 amdgpu_debugfs_compute_sched_mask_init(adev); 2230 amdgpu_debugfs_sdma_sched_mask_init(adev); 2231 2232 amdgpu_ras_debugfs_create_all(adev); 2233 amdgpu_rap_debugfs_init(adev); 2234 amdgpu_securedisplay_debugfs_init(adev); 2235 amdgpu_fw_attestation_debugfs_init(adev); 2236 amdgpu_psp_debugfs_init(adev); 2237 2238 debugfs_create_file("amdgpu_evict_vram", 0400, root, adev, 2239 &amdgpu_evict_vram_fops); 2240 debugfs_create_file("amdgpu_evict_gtt", 0400, root, adev, 2241 &amdgpu_evict_gtt_fops); 2242 debugfs_create_file("amdgpu_test_ib", 0400, root, adev, 2243 &amdgpu_debugfs_test_ib_fops); 2244 debugfs_create_file("amdgpu_vm_info", 0444, root, adev, 2245 &amdgpu_debugfs_vm_info_fops); 2246 debugfs_create_file("amdgpu_benchmark", 0200, root, adev, 2247 &amdgpu_benchmark_fops); 2248 2249 adev->debugfs_vbios_blob.data = adev->bios; 2250 adev->debugfs_vbios_blob.size = adev->bios_size; 2251 debugfs_create_blob("amdgpu_vbios", 0444, root, 2252 &adev->debugfs_vbios_blob); 2253 2254 if (adev->discovery.debugfs_blob.size) 2255 debugfs_create_blob("amdgpu_discovery", 0444, root, 2256 &adev->discovery.debugfs_blob); 2257 2258 return 0; 2259 } 2260 2261 static int amdgpu_pt_info_read(struct seq_file *m, void *unused) 2262 { 2263 struct drm_file *file; 2264 struct amdgpu_fpriv *fpriv; 2265 struct amdgpu_bo *root_bo; 2266 struct amdgpu_device *adev; 2267 int r; 2268 2269 file = m->private; 2270 if (!file) 2271 return -EINVAL; 2272 2273 adev = drm_to_adev(file->minor->dev); 2274 fpriv = file->driver_priv; 2275 if (!fpriv || !fpriv->vm.root.bo) 2276 return -ENODEV; 2277 2278 root_bo = amdgpu_bo_ref(fpriv->vm.root.bo); 2279 r = amdgpu_bo_reserve(root_bo, true); 2280 if (r) { 2281 amdgpu_bo_unref(&root_bo); 2282 return -EINVAL; 2283 } 2284 2285 seq_printf(m, "pd_address: 0x%llx\n", amdgpu_gmc_pd_addr(fpriv->vm.root.bo)); 2286 seq_printf(m, "max_pfn: 0x%llx\n", adev->vm_manager.max_pfn); 2287 seq_printf(m, "num_level: 0x%x\n", adev->vm_manager.num_level); 2288 seq_printf(m, "block_size: 0x%x\n", adev->vm_manager.block_size); 2289 seq_printf(m, "fragment_size: 0x%x\n", adev->vm_manager.fragment_size); 2290 2291 amdgpu_bo_unreserve(root_bo); 2292 amdgpu_bo_unref(&root_bo); 2293 2294 return 0; 2295 } 2296 2297 static int amdgpu_pt_info_open(struct inode *inode, struct file *file) 2298 { 2299 return single_open(file, amdgpu_pt_info_read, inode->i_private); 2300 } 2301 2302 static const struct file_operations amdgpu_pt_info_fops = { 2303 .owner = THIS_MODULE, 2304 .open = amdgpu_pt_info_open, 2305 .read = seq_read, 2306 .llseek = seq_lseek, 2307 .release = single_release, 2308 }; 2309 2310 static int amdgpu_mqd_info_read(struct seq_file *m, void *unused) 2311 { 2312 struct amdgpu_usermode_queue *queue = m->private; 2313 struct amdgpu_bo *bo; 2314 int r; 2315 2316 if (!queue || !queue->mqd.obj) 2317 return -EINVAL; 2318 2319 bo = amdgpu_bo_ref(queue->mqd.obj); 2320 r = amdgpu_bo_reserve(bo, true); 2321 if (r) { 2322 amdgpu_bo_unref(&bo); 2323 return -EINVAL; 2324 } 2325 2326 seq_printf(m, "queue_type: %d\n", queue->queue_type); 2327 seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj)); 2328 2329 amdgpu_bo_unreserve(bo); 2330 amdgpu_bo_unref(&bo); 2331 2332 return 0; 2333 } 2334 2335 static int amdgpu_mqd_info_open(struct inode *inode, struct file *file) 2336 { 2337 return single_open(file, amdgpu_mqd_info_read, inode->i_private); 2338 } 2339 2340 static const struct file_operations amdgpu_mqd_info_fops = { 2341 .owner = THIS_MODULE, 2342 .open = amdgpu_mqd_info_open, 2343 .read = seq_read, 2344 .llseek = seq_lseek, 2345 .release = single_release, 2346 }; 2347 2348 void amdgpu_debugfs_userq_init(struct drm_file *file, struct amdgpu_usermode_queue *queue, int qid) 2349 { 2350 char queue_name[32]; 2351 2352 scnprintf(queue_name, sizeof(queue_name), "queue_%d", qid); 2353 queue->debugfs_queue = debugfs_create_dir(queue_name, file->debugfs_client); 2354 debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops); 2355 } 2356 2357 void amdgpu_debugfs_vm_init(struct drm_file *file) 2358 { 2359 debugfs_create_file("vm_pagetable_info", 0444, file->debugfs_client, file, 2360 &amdgpu_pt_info_fops); 2361 } 2362 2363 #else 2364 int amdgpu_debugfs_init(struct amdgpu_device *adev) 2365 { 2366 return 0; 2367 } 2368 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) 2369 { 2370 return 0; 2371 } 2372 void amdgpu_debugfs_vm_init(struct drm_file *file) 2373 { 2374 } 2375 void amdgpu_debugfs_userq_init(struct drm_file *file, 2376 struct amdgpu_usermode_queue *queue, 2377 int qid) 2378 { 2379 } 2380 #endif 2381