1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/kthread.h> 27 #include <linux/pci.h> 28 #include <linux/uaccess.h> 29 #include <linux/security.h> 30 #include <linux/pm_runtime.h> 31 32 #include "amdgpu.h" 33 #include "amdgpu_pm.h" 34 #include "amdgpu_dm_debugfs.h" 35 #include "amdgpu_ras.h" 36 #include "amdgpu_rap.h" 37 #include "amdgpu_securedisplay.h" 38 #include "amdgpu_fw_attestation.h" 39 #include "amdgpu_umr.h" 40 41 #include "amdgpu_reset.h" 42 #include "amdgpu_psp_ta.h" 43 #include "amdgpu_userq.h" 44 45 #if defined(CONFIG_DEBUG_FS) 46 47 /** 48 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes 49 * 50 * @read: True if reading 51 * @f: open file handle 52 * @buf: User buffer to write/read to 53 * @size: Number of bytes to write/read 54 * @pos: Offset to seek to 55 * 56 * This debugfs entry has special meaning on the offset being sought. 57 * Various bits have different meanings: 58 * 59 * Bit 62: Indicates a GRBM bank switch is needed 60 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is 61 * zero) 62 * Bits 24..33: The SE or ME selector if needed 63 * Bits 34..43: The SH (or SA) or PIPE selector if needed 64 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed 65 * 66 * Bit 23: Indicates that the PM power gating lock should be held 67 * This is necessary to read registers that might be 68 * unreliable during a power gating transistion. 69 * 70 * The lower bits are the BYTE offset of the register to read. This 71 * allows reading multiple registers in a single call and having 72 * the returned size reflect that. 73 */ 74 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, 75 char __user *buf, size_t size, loff_t *pos) 76 { 77 struct amdgpu_device *adev = file_inode(f)->i_private; 78 ssize_t result = 0; 79 int r; 80 bool pm_pg_lock, use_bank, use_ring; 81 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid; 82 83 pm_pg_lock = use_bank = use_ring = false; 84 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0; 85 86 if (size & 0x3 || *pos & 0x3 || 87 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61)))) 88 return -EINVAL; 89 90 /* are we reading registers for which a PG lock is necessary? */ 91 pm_pg_lock = (*pos >> 23) & 1; 92 93 if (*pos & (1ULL << 62)) { 94 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24; 95 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34; 96 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44; 97 98 if (se_bank == 0x3FF) 99 se_bank = 0xFFFFFFFF; 100 if (sh_bank == 0x3FF) 101 sh_bank = 0xFFFFFFFF; 102 if (instance_bank == 0x3FF) 103 instance_bank = 0xFFFFFFFF; 104 use_bank = true; 105 } else if (*pos & (1ULL << 61)) { 106 107 me = (*pos & GENMASK_ULL(33, 24)) >> 24; 108 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34; 109 queue = (*pos & GENMASK_ULL(53, 44)) >> 44; 110 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54; 111 112 use_ring = true; 113 } else { 114 use_bank = use_ring = false; 115 } 116 117 *pos &= (1UL << 22) - 1; 118 119 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 120 if (r < 0) { 121 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 122 return r; 123 } 124 125 r = amdgpu_virt_enable_access_debugfs(adev); 126 if (r < 0) { 127 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 128 return r; 129 } 130 131 if (use_bank) { 132 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) || 133 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) { 134 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 135 amdgpu_virt_disable_access_debugfs(adev); 136 return -EINVAL; 137 } 138 mutex_lock(&adev->grbm_idx_mutex); 139 amdgpu_gfx_select_se_sh(adev, se_bank, 140 sh_bank, instance_bank, 0); 141 } else if (use_ring) { 142 mutex_lock(&adev->srbm_mutex); 143 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0); 144 } 145 146 if (pm_pg_lock) 147 mutex_lock(&adev->pm.mutex); 148 149 while (size) { 150 uint32_t value; 151 152 if (read) { 153 value = RREG32(*pos >> 2); 154 r = put_user(value, (uint32_t *)buf); 155 } else { 156 r = get_user(value, (uint32_t *)buf); 157 if (!r) 158 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0); 159 } 160 if (r) { 161 result = r; 162 goto end; 163 } 164 165 result += 4; 166 buf += 4; 167 *pos += 4; 168 size -= 4; 169 } 170 171 end: 172 if (use_bank) { 173 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 174 mutex_unlock(&adev->grbm_idx_mutex); 175 } else if (use_ring) { 176 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0); 177 mutex_unlock(&adev->srbm_mutex); 178 } 179 180 if (pm_pg_lock) 181 mutex_unlock(&adev->pm.mutex); 182 183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 184 185 amdgpu_virt_disable_access_debugfs(adev); 186 return result; 187 } 188 189 /* 190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers 191 */ 192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, 193 size_t size, loff_t *pos) 194 { 195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos); 196 } 197 198 /* 199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers 200 */ 201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf, 202 size_t size, loff_t *pos) 203 { 204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos); 205 } 206 207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file) 208 { 209 struct amdgpu_debugfs_regs2_data *rd; 210 211 rd = kzalloc_obj(*rd); 212 if (!rd) 213 return -ENOMEM; 214 rd->adev = file_inode(file)->i_private; 215 file->private_data = rd; 216 mutex_init(&rd->lock); 217 218 return 0; 219 } 220 221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file) 222 { 223 struct amdgpu_debugfs_regs2_data *rd = file->private_data; 224 225 mutex_destroy(&rd->lock); 226 kfree(file->private_data); 227 return 0; 228 } 229 230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en) 231 { 232 struct amdgpu_debugfs_regs2_data *rd = f->private_data; 233 struct amdgpu_device *adev = rd->adev; 234 ssize_t result = 0; 235 int r; 236 uint32_t value; 237 238 if (size & 0x3 || offset & 0x3) 239 return -EINVAL; 240 241 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 242 if (r < 0) { 243 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 244 return r; 245 } 246 247 r = amdgpu_virt_enable_access_debugfs(adev); 248 if (r < 0) { 249 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 250 return r; 251 } 252 253 mutex_lock(&rd->lock); 254 255 if (rd->id.use_grbm) { 256 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) || 257 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) { 258 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 259 amdgpu_virt_disable_access_debugfs(adev); 260 mutex_unlock(&rd->lock); 261 return -EINVAL; 262 } 263 mutex_lock(&adev->grbm_idx_mutex); 264 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se, 265 rd->id.grbm.sh, 266 rd->id.grbm.instance, rd->id.xcc_id); 267 } 268 269 if (rd->id.use_srbm) { 270 mutex_lock(&adev->srbm_mutex); 271 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe, 272 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id); 273 } 274 275 if (rd->id.pg_lock) 276 mutex_lock(&adev->pm.mutex); 277 278 while (size) { 279 if (!write_en) { 280 value = RREG32(offset >> 2); 281 r = put_user(value, (uint32_t *)buf); 282 } else { 283 r = get_user(value, (uint32_t *)buf); 284 if (!r) 285 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id); 286 } 287 if (r) { 288 result = r; 289 goto end; 290 } 291 offset += 4; 292 size -= 4; 293 result += 4; 294 buf += 4; 295 } 296 end: 297 if (rd->id.use_grbm) { 298 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id); 299 mutex_unlock(&adev->grbm_idx_mutex); 300 } 301 302 if (rd->id.use_srbm) { 303 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id); 304 mutex_unlock(&adev->srbm_mutex); 305 } 306 307 if (rd->id.pg_lock) 308 mutex_unlock(&adev->pm.mutex); 309 310 mutex_unlock(&rd->lock); 311 312 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 313 314 amdgpu_virt_disable_access_debugfs(adev); 315 return result; 316 } 317 318 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data) 319 { 320 struct amdgpu_debugfs_regs2_data *rd = f->private_data; 321 struct amdgpu_debugfs_regs2_iocdata v1_data; 322 int r; 323 324 mutex_lock(&rd->lock); 325 326 switch (cmd) { 327 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2: 328 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data, 329 sizeof(rd->id)); 330 if (r) 331 r = -EINVAL; 332 goto done; 333 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE: 334 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data, 335 sizeof(v1_data)); 336 if (r) { 337 r = -EINVAL; 338 goto done; 339 } 340 goto v1_copy; 341 default: 342 r = -EINVAL; 343 goto done; 344 } 345 346 v1_copy: 347 rd->id.use_srbm = v1_data.use_srbm; 348 rd->id.use_grbm = v1_data.use_grbm; 349 rd->id.pg_lock = v1_data.pg_lock; 350 rd->id.grbm.se = v1_data.grbm.se; 351 rd->id.grbm.sh = v1_data.grbm.sh; 352 rd->id.grbm.instance = v1_data.grbm.instance; 353 rd->id.srbm.me = v1_data.srbm.me; 354 rd->id.srbm.pipe = v1_data.srbm.pipe; 355 rd->id.srbm.queue = v1_data.srbm.queue; 356 rd->id.xcc_id = 0; 357 done: 358 mutex_unlock(&rd->lock); 359 return r; 360 } 361 362 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos) 363 { 364 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0); 365 } 366 367 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos) 368 { 369 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1); 370 } 371 372 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file) 373 { 374 struct amdgpu_debugfs_gprwave_data *rd; 375 376 rd = kzalloc_obj(*rd); 377 if (!rd) 378 return -ENOMEM; 379 rd->adev = file_inode(file)->i_private; 380 file->private_data = rd; 381 mutex_init(&rd->lock); 382 383 return 0; 384 } 385 386 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file) 387 { 388 struct amdgpu_debugfs_gprwave_data *rd = file->private_data; 389 390 mutex_destroy(&rd->lock); 391 kfree(file->private_data); 392 return 0; 393 } 394 395 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos) 396 { 397 struct amdgpu_debugfs_gprwave_data *rd = f->private_data; 398 struct amdgpu_device *adev = rd->adev; 399 ssize_t result = 0; 400 int r; 401 uint32_t *data, x; 402 403 if (size > 4096 || size & 0x3 || *pos & 0x3) 404 return -EINVAL; 405 406 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 407 if (r < 0) { 408 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 409 return r; 410 } 411 412 r = amdgpu_virt_enable_access_debugfs(adev); 413 if (r < 0) { 414 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 415 return r; 416 } 417 418 data = kcalloc(1024, sizeof(*data), GFP_KERNEL); 419 if (!data) { 420 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 421 amdgpu_virt_disable_access_debugfs(adev); 422 return -ENOMEM; 423 } 424 425 /* switch to the specific se/sh/cu */ 426 mutex_lock(&adev->grbm_idx_mutex); 427 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id); 428 429 if (!rd->id.gpr_or_wave) { 430 x = 0; 431 if (adev->gfx.funcs->read_wave_data) 432 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x); 433 } else { 434 x = size >> 2; 435 if (rd->id.gpr.vpgr_or_sgpr) { 436 if (adev->gfx.funcs->read_wave_vgprs) 437 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data); 438 } else { 439 if (adev->gfx.funcs->read_wave_sgprs) 440 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data); 441 } 442 } 443 444 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id); 445 mutex_unlock(&adev->grbm_idx_mutex); 446 447 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 448 449 if (!x) { 450 result = -EINVAL; 451 goto done; 452 } 453 454 while (size && (*pos < x * 4)) { 455 uint32_t value; 456 457 value = data[*pos >> 2]; 458 r = put_user(value, (uint32_t *)buf); 459 if (r) { 460 result = r; 461 goto done; 462 } 463 464 result += 4; 465 buf += 4; 466 *pos += 4; 467 size -= 4; 468 } 469 470 done: 471 amdgpu_virt_disable_access_debugfs(adev); 472 kfree(data); 473 return result; 474 } 475 476 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data) 477 { 478 struct amdgpu_debugfs_gprwave_data *rd = f->private_data; 479 int r = 0; 480 481 mutex_lock(&rd->lock); 482 483 switch (cmd) { 484 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE: 485 if (copy_from_user(&rd->id, 486 (struct amdgpu_debugfs_gprwave_iocdata *)data, 487 sizeof(rd->id))) 488 r = -EFAULT; 489 goto done; 490 default: 491 r = -EINVAL; 492 goto done; 493 } 494 495 done: 496 mutex_unlock(&rd->lock); 497 return r; 498 } 499 500 501 502 503 /** 504 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register 505 * 506 * @f: open file handle 507 * @buf: User buffer to store read data in 508 * @size: Number of bytes to read 509 * @pos: Offset to seek to 510 * 511 * The lower bits are the BYTE offset of the register to read. This 512 * allows reading multiple registers in a single call and having 513 * the returned size reflect that. 514 */ 515 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf, 516 size_t size, loff_t *pos) 517 { 518 struct amdgpu_device *adev = file_inode(f)->i_private; 519 ssize_t result = 0; 520 int r; 521 522 if (size & 0x3 || *pos & 0x3) 523 return -EINVAL; 524 525 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 526 if (r < 0) { 527 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 528 return r; 529 } 530 531 r = amdgpu_virt_enable_access_debugfs(adev); 532 if (r < 0) { 533 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 534 return r; 535 } 536 537 while (size) { 538 uint32_t value; 539 540 if (upper_32_bits(*pos)) 541 value = RREG32_PCIE_EXT(*pos); 542 else 543 value = RREG32_PCIE(*pos); 544 545 r = put_user(value, (uint32_t *)buf); 546 if (r) 547 goto out; 548 549 result += 4; 550 buf += 4; 551 *pos += 4; 552 size -= 4; 553 } 554 555 r = result; 556 out: 557 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 558 amdgpu_virt_disable_access_debugfs(adev); 559 return r; 560 } 561 562 /** 563 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register 564 * 565 * @f: open file handle 566 * @buf: User buffer to write data from 567 * @size: Number of bytes to write 568 * @pos: Offset to seek to 569 * 570 * The lower bits are the BYTE offset of the register to write. This 571 * allows writing multiple registers in a single call and having 572 * the returned size reflect that. 573 */ 574 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf, 575 size_t size, loff_t *pos) 576 { 577 struct amdgpu_device *adev = file_inode(f)->i_private; 578 ssize_t result = 0; 579 int r; 580 581 if (size & 0x3 || *pos & 0x3) 582 return -EINVAL; 583 584 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 585 if (r < 0) { 586 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 587 return r; 588 } 589 590 r = amdgpu_virt_enable_access_debugfs(adev); 591 if (r < 0) { 592 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 593 return r; 594 } 595 596 while (size) { 597 uint32_t value; 598 599 r = get_user(value, (uint32_t *)buf); 600 if (r) 601 goto out; 602 603 if (upper_32_bits(*pos)) 604 WREG32_PCIE_EXT(*pos, value); 605 else 606 WREG32_PCIE(*pos, value); 607 608 result += 4; 609 buf += 4; 610 *pos += 4; 611 size -= 4; 612 } 613 614 r = result; 615 out: 616 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 617 amdgpu_virt_disable_access_debugfs(adev); 618 return r; 619 } 620 621 /** 622 * amdgpu_debugfs_regs_pcie64_read - Read from a 64-bit PCIE register 623 * 624 * @f: open file handle 625 * @buf: User buffer to store read data in 626 * @size: Number of bytes to read 627 * @pos: Offset to seek to 628 */ 629 static ssize_t amdgpu_debugfs_regs_pcie64_read(struct file *f, char __user *buf, 630 size_t size, loff_t *pos) 631 { 632 struct amdgpu_device *adev = file_inode(f)->i_private; 633 ssize_t result = 0; 634 int r; 635 636 if (size & 0x7 || *pos & 0x7) 637 return -EINVAL; 638 639 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 640 if (r < 0) { 641 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 642 return r; 643 } 644 645 r = amdgpu_virt_enable_access_debugfs(adev); 646 if (r < 0) { 647 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 648 return r; 649 } 650 651 while (size) { 652 uint64_t value; 653 654 value = RREG64_PCIE_EXT(*pos); 655 656 r = put_user(value, (uint64_t *)buf); 657 if (r) 658 goto out; 659 660 result += 8; 661 buf += 8; 662 *pos += 8; 663 size -= 8; 664 } 665 666 r = result; 667 out: 668 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 669 amdgpu_virt_disable_access_debugfs(adev); 670 return r; 671 } 672 673 /** 674 * amdgpu_debugfs_regs_pcie64_write - Write to a 64-bit PCIE register 675 * 676 * @f: open file handle 677 * @buf: User buffer to write data from 678 * @size: Number of bytes to write 679 * @pos: Offset to seek to 680 */ 681 static ssize_t amdgpu_debugfs_regs_pcie64_write(struct file *f, const char __user *buf, 682 size_t size, loff_t *pos) 683 { 684 struct amdgpu_device *adev = file_inode(f)->i_private; 685 ssize_t result = 0; 686 int r; 687 688 if (size & 0x7 || *pos & 0x7) 689 return -EINVAL; 690 691 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 692 if (r < 0) { 693 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 694 return r; 695 } 696 697 r = amdgpu_virt_enable_access_debugfs(adev); 698 if (r < 0) { 699 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 700 return r; 701 } 702 703 while (size) { 704 uint64_t value; 705 706 r = get_user(value, (uint64_t *)buf); 707 if (r) 708 goto out; 709 710 WREG64_PCIE_EXT(*pos, value); 711 712 result += 8; 713 buf += 8; 714 *pos += 8; 715 size -= 8; 716 } 717 718 r = result; 719 out: 720 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 721 amdgpu_virt_disable_access_debugfs(adev); 722 return r; 723 } 724 725 /** 726 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register 727 * 728 * @f: open file handle 729 * @buf: User buffer to store read data in 730 * @size: Number of bytes to read 731 * @pos: Offset to seek to 732 * 733 * The lower bits are the BYTE offset of the register to read. This 734 * allows reading multiple registers in a single call and having 735 * the returned size reflect that. 736 */ 737 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf, 738 size_t size, loff_t *pos) 739 { 740 struct amdgpu_device *adev = file_inode(f)->i_private; 741 ssize_t result = 0; 742 int r; 743 744 if (size & 0x3 || *pos & 0x3) 745 return -EINVAL; 746 747 if (!adev->reg.didt.rreg) 748 return -EOPNOTSUPP; 749 750 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 751 if (r < 0) { 752 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 753 return r; 754 } 755 756 r = amdgpu_virt_enable_access_debugfs(adev); 757 if (r < 0) { 758 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 759 return r; 760 } 761 762 while (size) { 763 uint32_t value; 764 765 value = RREG32_DIDT(*pos >> 2); 766 r = put_user(value, (uint32_t *)buf); 767 if (r) 768 goto out; 769 770 result += 4; 771 buf += 4; 772 *pos += 4; 773 size -= 4; 774 } 775 776 r = result; 777 out: 778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 779 amdgpu_virt_disable_access_debugfs(adev); 780 return r; 781 } 782 783 /** 784 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register 785 * 786 * @f: open file handle 787 * @buf: User buffer to write data from 788 * @size: Number of bytes to write 789 * @pos: Offset to seek to 790 * 791 * The lower bits are the BYTE offset of the register to write. This 792 * allows writing multiple registers in a single call and having 793 * the returned size reflect that. 794 */ 795 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf, 796 size_t size, loff_t *pos) 797 { 798 struct amdgpu_device *adev = file_inode(f)->i_private; 799 ssize_t result = 0; 800 int r; 801 802 if (size & 0x3 || *pos & 0x3) 803 return -EINVAL; 804 805 if (!adev->reg.didt.wreg) 806 return -EOPNOTSUPP; 807 808 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 809 if (r < 0) { 810 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 811 return r; 812 } 813 814 r = amdgpu_virt_enable_access_debugfs(adev); 815 if (r < 0) { 816 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 817 return r; 818 } 819 820 while (size) { 821 uint32_t value; 822 823 r = get_user(value, (uint32_t *)buf); 824 if (r) 825 goto out; 826 827 WREG32_DIDT(*pos >> 2, value); 828 829 result += 4; 830 buf += 4; 831 *pos += 4; 832 size -= 4; 833 } 834 835 r = result; 836 out: 837 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 838 amdgpu_virt_disable_access_debugfs(adev); 839 return r; 840 } 841 842 /** 843 * amdgpu_debugfs_regs_smc_read - Read from a SMC register 844 * 845 * @f: open file handle 846 * @buf: User buffer to store read data in 847 * @size: Number of bytes to read 848 * @pos: Offset to seek to 849 * 850 * The lower bits are the BYTE offset of the register to read. This 851 * allows reading multiple registers in a single call and having 852 * the returned size reflect that. 853 */ 854 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf, 855 size_t size, loff_t *pos) 856 { 857 struct amdgpu_device *adev = file_inode(f)->i_private; 858 ssize_t result = 0; 859 int r; 860 861 if (!adev->reg.smc.rreg) 862 return -EOPNOTSUPP; 863 864 if (size & 0x3 || *pos & 0x3) 865 return -EINVAL; 866 867 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 868 if (r < 0) { 869 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 870 return r; 871 } 872 873 r = amdgpu_virt_enable_access_debugfs(adev); 874 if (r < 0) { 875 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 876 return r; 877 } 878 879 while (size) { 880 uint32_t value; 881 882 value = RREG32_SMC(*pos); 883 r = put_user(value, (uint32_t *)buf); 884 if (r) 885 goto out; 886 887 result += 4; 888 buf += 4; 889 *pos += 4; 890 size -= 4; 891 } 892 893 r = result; 894 out: 895 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 896 amdgpu_virt_disable_access_debugfs(adev); 897 return r; 898 } 899 900 /** 901 * amdgpu_debugfs_regs_smc_write - Write to a SMC register 902 * 903 * @f: open file handle 904 * @buf: User buffer to write data from 905 * @size: Number of bytes to write 906 * @pos: Offset to seek to 907 * 908 * The lower bits are the BYTE offset of the register to write. This 909 * allows writing multiple registers in a single call and having 910 * the returned size reflect that. 911 */ 912 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf, 913 size_t size, loff_t *pos) 914 { 915 struct amdgpu_device *adev = file_inode(f)->i_private; 916 ssize_t result = 0; 917 int r; 918 919 if (!adev->reg.smc.wreg) 920 return -EOPNOTSUPP; 921 922 if (size & 0x3 || *pos & 0x3) 923 return -EINVAL; 924 925 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 926 if (r < 0) { 927 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 928 return r; 929 } 930 931 r = amdgpu_virt_enable_access_debugfs(adev); 932 if (r < 0) { 933 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 934 return r; 935 } 936 937 while (size) { 938 uint32_t value; 939 940 r = get_user(value, (uint32_t *)buf); 941 if (r) 942 goto out; 943 944 WREG32_SMC(*pos, value); 945 946 result += 4; 947 buf += 4; 948 *pos += 4; 949 size -= 4; 950 } 951 952 r = result; 953 out: 954 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 955 amdgpu_virt_disable_access_debugfs(adev); 956 return r; 957 } 958 959 /** 960 * amdgpu_debugfs_gca_config_read - Read from gfx config data 961 * 962 * @f: open file handle 963 * @buf: User buffer to store read data in 964 * @size: Number of bytes to read 965 * @pos: Offset to seek to 966 * 967 * This file is used to access configuration data in a somewhat 968 * stable fashion. The format is a series of DWORDs with the first 969 * indicating which revision it is. New content is appended to the 970 * end so that older software can still read the data. 971 */ 972 973 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf, 974 size_t size, loff_t *pos) 975 { 976 struct amdgpu_device *adev = file_inode(f)->i_private; 977 ssize_t result = 0; 978 int r; 979 uint32_t *config, no_regs = 0; 980 981 if (size & 0x3 || *pos & 0x3) 982 return -EINVAL; 983 984 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL); 985 if (!config) 986 return -ENOMEM; 987 988 /* version, increment each time something is added */ 989 config[no_regs++] = 5; 990 config[no_regs++] = adev->gfx.config.max_shader_engines; 991 config[no_regs++] = adev->gfx.config.max_tile_pipes; 992 config[no_regs++] = adev->gfx.config.max_cu_per_sh; 993 config[no_regs++] = adev->gfx.config.max_sh_per_se; 994 config[no_regs++] = adev->gfx.config.max_backends_per_se; 995 config[no_regs++] = adev->gfx.config.max_texture_channel_caches; 996 config[no_regs++] = adev->gfx.config.max_gprs; 997 config[no_regs++] = adev->gfx.config.max_gs_threads; 998 config[no_regs++] = adev->gfx.config.max_hw_contexts; 999 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend; 1000 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend; 1001 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size; 1002 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size; 1003 config[no_regs++] = adev->gfx.config.num_tile_pipes; 1004 config[no_regs++] = adev->gfx.config.backend_enable_mask; 1005 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes; 1006 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb; 1007 config[no_regs++] = adev->gfx.config.shader_engine_tile_size; 1008 config[no_regs++] = adev->gfx.config.num_gpus; 1009 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size; 1010 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg; 1011 config[no_regs++] = adev->gfx.config.gb_addr_config; 1012 config[no_regs++] = adev->gfx.config.num_rbs; 1013 1014 /* rev==1 */ 1015 config[no_regs++] = adev->rev_id; 1016 config[no_regs++] = adev->pg_flags; 1017 config[no_regs++] = lower_32_bits(adev->cg_flags); 1018 1019 /* rev==2 */ 1020 config[no_regs++] = adev->family; 1021 config[no_regs++] = adev->external_rev_id; 1022 1023 /* rev==3 */ 1024 config[no_regs++] = adev->pdev->device; 1025 config[no_regs++] = adev->pdev->revision; 1026 config[no_regs++] = adev->pdev->subsystem_device; 1027 config[no_regs++] = adev->pdev->subsystem_vendor; 1028 1029 /* rev==4 APU flag */ 1030 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0; 1031 1032 /* rev==5 PG/CG flag upper 32bit */ 1033 config[no_regs++] = 0; 1034 config[no_regs++] = upper_32_bits(adev->cg_flags); 1035 1036 while (size && (*pos < no_regs * 4)) { 1037 uint32_t value; 1038 1039 value = config[*pos >> 2]; 1040 r = put_user(value, (uint32_t *)buf); 1041 if (r) { 1042 kfree(config); 1043 return r; 1044 } 1045 1046 result += 4; 1047 buf += 4; 1048 *pos += 4; 1049 size -= 4; 1050 } 1051 1052 kfree(config); 1053 return result; 1054 } 1055 1056 /** 1057 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors 1058 * 1059 * @f: open file handle 1060 * @buf: User buffer to store read data in 1061 * @size: Number of bytes to read 1062 * @pos: Offset to seek to 1063 * 1064 * The offset is treated as the BYTE address of one of the sensors 1065 * enumerated in amd/include/kgd_pp_interface.h under the 1066 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK 1067 * you would use the offset 3 * 4 = 12. 1068 */ 1069 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf, 1070 size_t size, loff_t *pos) 1071 { 1072 struct amdgpu_device *adev = file_inode(f)->i_private; 1073 int idx, x, outsize, r, valuesize; 1074 uint32_t values[16]; 1075 1076 if (size & 3 || *pos & 0x3) 1077 return -EINVAL; 1078 1079 if (!adev->pm.dpm_enabled) 1080 return -EINVAL; 1081 1082 /* convert offset to sensor number */ 1083 idx = *pos >> 2; 1084 1085 valuesize = sizeof(values); 1086 1087 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1088 if (r < 0) { 1089 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1090 return r; 1091 } 1092 1093 r = amdgpu_virt_enable_access_debugfs(adev); 1094 if (r < 0) { 1095 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1096 return r; 1097 } 1098 1099 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize); 1100 1101 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1102 1103 if (r) { 1104 amdgpu_virt_disable_access_debugfs(adev); 1105 return r; 1106 } 1107 1108 if (size > valuesize) { 1109 amdgpu_virt_disable_access_debugfs(adev); 1110 return -EINVAL; 1111 } 1112 1113 outsize = 0; 1114 x = 0; 1115 if (!r) { 1116 while (size) { 1117 r = put_user(values[x++], (int32_t *)buf); 1118 buf += 4; 1119 size -= 4; 1120 outsize += 4; 1121 } 1122 } 1123 1124 amdgpu_virt_disable_access_debugfs(adev); 1125 return !r ? outsize : r; 1126 } 1127 1128 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data 1129 * 1130 * @f: open file handle 1131 * @buf: User buffer to store read data in 1132 * @size: Number of bytes to read 1133 * @pos: Offset to seek to 1134 * 1135 * The offset being sought changes which wave that the status data 1136 * will be returned for. The bits are used as follows: 1137 * 1138 * Bits 0..6: Byte offset into data 1139 * Bits 7..14: SE selector 1140 * Bits 15..22: SH/SA selector 1141 * Bits 23..30: CU/{WGP+SIMD} selector 1142 * Bits 31..36: WAVE ID selector 1143 * Bits 37..44: SIMD ID selector 1144 * 1145 * The returned data begins with one DWORD of version information 1146 * Followed by WAVE STATUS registers relevant to the GFX IP version 1147 * being used. See gfx_v8_0_read_wave_data() for an example output. 1148 */ 1149 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf, 1150 size_t size, loff_t *pos) 1151 { 1152 struct amdgpu_device *adev = f->f_inode->i_private; 1153 int r, x; 1154 ssize_t result = 0; 1155 uint32_t offset, se, sh, cu, wave, simd, data[32]; 1156 1157 if (size & 3 || *pos & 3) 1158 return -EINVAL; 1159 1160 /* decode offset */ 1161 offset = (*pos & GENMASK_ULL(6, 0)); 1162 se = (*pos & GENMASK_ULL(14, 7)) >> 7; 1163 sh = (*pos & GENMASK_ULL(22, 15)) >> 15; 1164 cu = (*pos & GENMASK_ULL(30, 23)) >> 23; 1165 wave = (*pos & GENMASK_ULL(36, 31)) >> 31; 1166 simd = (*pos & GENMASK_ULL(44, 37)) >> 37; 1167 1168 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1169 if (r < 0) { 1170 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1171 return r; 1172 } 1173 1174 r = amdgpu_virt_enable_access_debugfs(adev); 1175 if (r < 0) { 1176 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1177 return r; 1178 } 1179 1180 /* switch to the specific se/sh/cu */ 1181 mutex_lock(&adev->grbm_idx_mutex); 1182 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); 1183 1184 x = 0; 1185 if (adev->gfx.funcs->read_wave_data) 1186 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x); 1187 1188 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); 1189 mutex_unlock(&adev->grbm_idx_mutex); 1190 1191 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1192 1193 if (!x) { 1194 amdgpu_virt_disable_access_debugfs(adev); 1195 return -EINVAL; 1196 } 1197 1198 while (size && (offset < x * 4)) { 1199 uint32_t value; 1200 1201 value = data[offset >> 2]; 1202 r = put_user(value, (uint32_t *)buf); 1203 if (r) { 1204 amdgpu_virt_disable_access_debugfs(adev); 1205 return r; 1206 } 1207 1208 result += 4; 1209 buf += 4; 1210 offset += 4; 1211 size -= 4; 1212 } 1213 1214 amdgpu_virt_disable_access_debugfs(adev); 1215 return result; 1216 } 1217 1218 /** amdgpu_debugfs_gpr_read - Read wave gprs 1219 * 1220 * @f: open file handle 1221 * @buf: User buffer to store read data in 1222 * @size: Number of bytes to read 1223 * @pos: Offset to seek to 1224 * 1225 * The offset being sought changes which wave that the status data 1226 * will be returned for. The bits are used as follows: 1227 * 1228 * Bits 0..11: Byte offset into data 1229 * Bits 12..19: SE selector 1230 * Bits 20..27: SH/SA selector 1231 * Bits 28..35: CU/{WGP+SIMD} selector 1232 * Bits 36..43: WAVE ID selector 1233 * Bits 37..44: SIMD ID selector 1234 * Bits 52..59: Thread selector 1235 * Bits 60..61: Bank selector (VGPR=0,SGPR=1) 1236 * 1237 * The return data comes from the SGPR or VGPR register bank for 1238 * the selected operational unit. 1239 */ 1240 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf, 1241 size_t size, loff_t *pos) 1242 { 1243 struct amdgpu_device *adev = f->f_inode->i_private; 1244 int r; 1245 ssize_t result = 0; 1246 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data; 1247 1248 if (size > 4096 || size & 3 || *pos & 3) 1249 return -EINVAL; 1250 1251 /* decode offset */ 1252 offset = (*pos & GENMASK_ULL(11, 0)) >> 2; 1253 se = (*pos & GENMASK_ULL(19, 12)) >> 12; 1254 sh = (*pos & GENMASK_ULL(27, 20)) >> 20; 1255 cu = (*pos & GENMASK_ULL(35, 28)) >> 28; 1256 wave = (*pos & GENMASK_ULL(43, 36)) >> 36; 1257 simd = (*pos & GENMASK_ULL(51, 44)) >> 44; 1258 thread = (*pos & GENMASK_ULL(59, 52)) >> 52; 1259 bank = (*pos & GENMASK_ULL(61, 60)) >> 60; 1260 1261 data = kcalloc(1024, sizeof(*data), GFP_KERNEL); 1262 if (!data) 1263 return -ENOMEM; 1264 1265 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1266 if (r < 0) 1267 goto err; 1268 1269 r = amdgpu_virt_enable_access_debugfs(adev); 1270 if (r < 0) 1271 goto err; 1272 1273 /* switch to the specific se/sh/cu */ 1274 mutex_lock(&adev->grbm_idx_mutex); 1275 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0); 1276 1277 if (bank == 0) { 1278 if (adev->gfx.funcs->read_wave_vgprs) 1279 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data); 1280 } else { 1281 if (adev->gfx.funcs->read_wave_sgprs) 1282 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data); 1283 } 1284 1285 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0); 1286 mutex_unlock(&adev->grbm_idx_mutex); 1287 1288 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1289 1290 while (size) { 1291 uint32_t value; 1292 1293 value = data[result >> 2]; 1294 r = put_user(value, (uint32_t *)buf); 1295 if (r) { 1296 amdgpu_virt_disable_access_debugfs(adev); 1297 goto err; 1298 } 1299 1300 result += 4; 1301 buf += 4; 1302 size -= 4; 1303 } 1304 1305 kfree(data); 1306 amdgpu_virt_disable_access_debugfs(adev); 1307 return result; 1308 1309 err: 1310 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1311 kfree(data); 1312 return r; 1313 } 1314 1315 /** 1316 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency 1317 * 1318 * @f: open file handle 1319 * @buf: User buffer to store read data in 1320 * @size: Number of bytes to read 1321 * @pos: Offset to seek to 1322 * 1323 * Read a live GFXOFF residency sample from firmware. One needs to start logging 1324 * before getting the current value. 1325 */ 1326 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf, 1327 size_t size, loff_t *pos) 1328 { 1329 struct amdgpu_device *adev = file_inode(f)->i_private; 1330 ssize_t result = 0; 1331 int r; 1332 1333 if (size & 0x3 || *pos & 0x3) 1334 return -EINVAL; 1335 1336 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1337 if (r < 0) { 1338 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1339 return r; 1340 } 1341 1342 while (size) { 1343 uint32_t value; 1344 1345 r = amdgpu_get_gfx_off_residency(adev, &value); 1346 if (r) 1347 goto out; 1348 1349 r = put_user(value, (uint32_t *)buf); 1350 if (r) 1351 goto out; 1352 1353 result += 4; 1354 buf += 4; 1355 *pos += 4; 1356 size -= 4; 1357 } 1358 1359 r = result; 1360 out: 1361 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1362 1363 return r; 1364 } 1365 1366 /** 1367 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency 1368 * 1369 * @f: open file handle 1370 * @buf: User buffer to write data from 1371 * @size: Number of bytes to write 1372 * @pos: Offset to seek to 1373 * 1374 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop 1375 */ 1376 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf, 1377 size_t size, loff_t *pos) 1378 { 1379 struct amdgpu_device *adev = file_inode(f)->i_private; 1380 ssize_t result = 0; 1381 int r; 1382 1383 if (size & 0x3 || *pos & 0x3) 1384 return -EINVAL; 1385 1386 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1387 if (r < 0) { 1388 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1389 return r; 1390 } 1391 1392 while (size) { 1393 u32 value; 1394 1395 r = get_user(value, (uint32_t *)buf); 1396 if (r) 1397 goto out; 1398 1399 amdgpu_set_gfx_off_residency(adev, value ? true : false); 1400 1401 result += 4; 1402 buf += 4; 1403 *pos += 4; 1404 size -= 4; 1405 } 1406 1407 r = result; 1408 out: 1409 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1410 1411 return r; 1412 } 1413 1414 1415 /** 1416 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count 1417 * 1418 * @f: open file handle 1419 * @buf: User buffer to store read data in 1420 * @size: Number of bytes to read 1421 * @pos: Offset to seek to 1422 */ 1423 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf, 1424 size_t size, loff_t *pos) 1425 { 1426 struct amdgpu_device *adev = file_inode(f)->i_private; 1427 ssize_t result = 0; 1428 int r; 1429 1430 if (size & 0x3 || *pos & 0x3) 1431 return -EINVAL; 1432 1433 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1434 if (r < 0) { 1435 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1436 return r; 1437 } 1438 1439 while (size) { 1440 u64 value = 0; 1441 1442 r = amdgpu_get_gfx_off_entrycount(adev, &value); 1443 if (r) 1444 goto out; 1445 1446 r = put_user(value, (u64 *)buf); 1447 if (r) 1448 goto out; 1449 1450 result += 4; 1451 buf += 4; 1452 *pos += 4; 1453 size -= 4; 1454 } 1455 1456 r = result; 1457 out: 1458 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1459 1460 return r; 1461 } 1462 1463 /** 1464 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF 1465 * 1466 * @f: open file handle 1467 * @buf: User buffer to write data from 1468 * @size: Number of bytes to write 1469 * @pos: Offset to seek to 1470 * 1471 * Write a 32-bit zero to disable or a 32-bit non-zero to enable 1472 */ 1473 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf, 1474 size_t size, loff_t *pos) 1475 { 1476 struct amdgpu_device *adev = file_inode(f)->i_private; 1477 ssize_t result = 0; 1478 int r; 1479 1480 if (size & 0x3 || *pos & 0x3) 1481 return -EINVAL; 1482 1483 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1484 if (r < 0) { 1485 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1486 return r; 1487 } 1488 1489 while (size) { 1490 uint32_t value; 1491 1492 r = get_user(value, (uint32_t *)buf); 1493 if (r) 1494 goto out; 1495 1496 amdgpu_gfx_off_ctrl(adev, value ? true : false); 1497 1498 result += 4; 1499 buf += 4; 1500 *pos += 4; 1501 size -= 4; 1502 } 1503 1504 r = result; 1505 out: 1506 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1507 1508 return r; 1509 } 1510 1511 1512 /** 1513 * amdgpu_debugfs_gfxoff_read - read gfxoff status 1514 * 1515 * @f: open file handle 1516 * @buf: User buffer to store read data in 1517 * @size: Number of bytes to read 1518 * @pos: Offset to seek to 1519 */ 1520 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf, 1521 size_t size, loff_t *pos) 1522 { 1523 struct amdgpu_device *adev = file_inode(f)->i_private; 1524 ssize_t result = 0; 1525 int r; 1526 1527 if (size & 0x3 || *pos & 0x3) 1528 return -EINVAL; 1529 1530 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1531 if (r < 0) { 1532 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1533 return r; 1534 } 1535 1536 while (size) { 1537 u32 value = adev->gfx.gfx_off_state; 1538 1539 r = put_user(value, (u32 *)buf); 1540 if (r) 1541 goto out; 1542 1543 result += 4; 1544 buf += 4; 1545 *pos += 4; 1546 size -= 4; 1547 } 1548 1549 r = result; 1550 out: 1551 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1552 1553 return r; 1554 } 1555 1556 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf, 1557 size_t size, loff_t *pos) 1558 { 1559 struct amdgpu_device *adev = file_inode(f)->i_private; 1560 ssize_t result = 0; 1561 int r; 1562 1563 if (size & 0x3 || *pos & 0x3) 1564 return -EINVAL; 1565 1566 r = pm_runtime_get_sync(adev_to_drm(adev)->dev); 1567 if (r < 0) { 1568 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1569 return r; 1570 } 1571 1572 while (size) { 1573 u32 value; 1574 1575 r = amdgpu_get_gfx_off_status(adev, &value); 1576 if (r) 1577 goto out; 1578 1579 r = put_user(value, (u32 *)buf); 1580 if (r) 1581 goto out; 1582 1583 result += 4; 1584 buf += 4; 1585 *pos += 4; 1586 size -= 4; 1587 } 1588 1589 r = result; 1590 out: 1591 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 1592 1593 return r; 1594 } 1595 1596 static const struct file_operations amdgpu_debugfs_regs2_fops = { 1597 .owner = THIS_MODULE, 1598 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl, 1599 .read = amdgpu_debugfs_regs2_read, 1600 .write = amdgpu_debugfs_regs2_write, 1601 .open = amdgpu_debugfs_regs2_open, 1602 .release = amdgpu_debugfs_regs2_release, 1603 .llseek = default_llseek 1604 }; 1605 1606 static const struct file_operations amdgpu_debugfs_gprwave_fops = { 1607 .owner = THIS_MODULE, 1608 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl, 1609 .read = amdgpu_debugfs_gprwave_read, 1610 .open = amdgpu_debugfs_gprwave_open, 1611 .release = amdgpu_debugfs_gprwave_release, 1612 .llseek = default_llseek 1613 }; 1614 1615 static const struct file_operations amdgpu_debugfs_regs_fops = { 1616 .owner = THIS_MODULE, 1617 .read = amdgpu_debugfs_regs_read, 1618 .write = amdgpu_debugfs_regs_write, 1619 .llseek = default_llseek 1620 }; 1621 static const struct file_operations amdgpu_debugfs_regs_didt_fops = { 1622 .owner = THIS_MODULE, 1623 .read = amdgpu_debugfs_regs_didt_read, 1624 .write = amdgpu_debugfs_regs_didt_write, 1625 .llseek = default_llseek 1626 }; 1627 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = { 1628 .owner = THIS_MODULE, 1629 .read = amdgpu_debugfs_regs_pcie_read, 1630 .write = amdgpu_debugfs_regs_pcie_write, 1631 .llseek = default_llseek 1632 }; 1633 static const struct file_operations amdgpu_debugfs_regs_pcie64_fops = { 1634 .owner = THIS_MODULE, 1635 .read = amdgpu_debugfs_regs_pcie64_read, 1636 .write = amdgpu_debugfs_regs_pcie64_write, 1637 .llseek = default_llseek 1638 }; 1639 static const struct file_operations amdgpu_debugfs_regs_smc_fops = { 1640 .owner = THIS_MODULE, 1641 .read = amdgpu_debugfs_regs_smc_read, 1642 .write = amdgpu_debugfs_regs_smc_write, 1643 .llseek = default_llseek 1644 }; 1645 1646 static const struct file_operations amdgpu_debugfs_gca_config_fops = { 1647 .owner = THIS_MODULE, 1648 .read = amdgpu_debugfs_gca_config_read, 1649 .llseek = default_llseek 1650 }; 1651 1652 static const struct file_operations amdgpu_debugfs_sensors_fops = { 1653 .owner = THIS_MODULE, 1654 .read = amdgpu_debugfs_sensor_read, 1655 .llseek = default_llseek 1656 }; 1657 1658 static const struct file_operations amdgpu_debugfs_wave_fops = { 1659 .owner = THIS_MODULE, 1660 .read = amdgpu_debugfs_wave_read, 1661 .llseek = default_llseek 1662 }; 1663 static const struct file_operations amdgpu_debugfs_gpr_fops = { 1664 .owner = THIS_MODULE, 1665 .read = amdgpu_debugfs_gpr_read, 1666 .llseek = default_llseek 1667 }; 1668 1669 static const struct file_operations amdgpu_debugfs_gfxoff_fops = { 1670 .owner = THIS_MODULE, 1671 .read = amdgpu_debugfs_gfxoff_read, 1672 .write = amdgpu_debugfs_gfxoff_write, 1673 .llseek = default_llseek 1674 }; 1675 1676 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = { 1677 .owner = THIS_MODULE, 1678 .read = amdgpu_debugfs_gfxoff_status_read, 1679 .llseek = default_llseek 1680 }; 1681 1682 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = { 1683 .owner = THIS_MODULE, 1684 .read = amdgpu_debugfs_gfxoff_count_read, 1685 .llseek = default_llseek 1686 }; 1687 1688 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = { 1689 .owner = THIS_MODULE, 1690 .read = amdgpu_debugfs_gfxoff_residency_read, 1691 .write = amdgpu_debugfs_gfxoff_residency_write, 1692 .llseek = default_llseek 1693 }; 1694 1695 static const struct file_operations *debugfs_regs[] = { 1696 &amdgpu_debugfs_regs_fops, 1697 &amdgpu_debugfs_regs2_fops, 1698 &amdgpu_debugfs_gprwave_fops, 1699 &amdgpu_debugfs_regs_didt_fops, 1700 &amdgpu_debugfs_regs_pcie_fops, 1701 &amdgpu_debugfs_regs_pcie64_fops, 1702 &amdgpu_debugfs_regs_smc_fops, 1703 &amdgpu_debugfs_gca_config_fops, 1704 &amdgpu_debugfs_sensors_fops, 1705 &amdgpu_debugfs_wave_fops, 1706 &amdgpu_debugfs_gpr_fops, 1707 &amdgpu_debugfs_gfxoff_fops, 1708 &amdgpu_debugfs_gfxoff_status_fops, 1709 &amdgpu_debugfs_gfxoff_count_fops, 1710 &amdgpu_debugfs_gfxoff_residency_fops, 1711 }; 1712 1713 static const char * const debugfs_regs_names[] = { 1714 "amdgpu_regs", 1715 "amdgpu_regs2", 1716 "amdgpu_gprwave", 1717 "amdgpu_regs_didt", 1718 "amdgpu_regs_pcie", 1719 "amdgpu_regs_pcie64", 1720 "amdgpu_regs_smc", 1721 "amdgpu_gca_config", 1722 "amdgpu_sensors", 1723 "amdgpu_wave", 1724 "amdgpu_gpr", 1725 "amdgpu_gfxoff", 1726 "amdgpu_gfxoff_status", 1727 "amdgpu_gfxoff_count", 1728 "amdgpu_gfxoff_residency", 1729 }; 1730 1731 /** 1732 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide 1733 * register access. 1734 * 1735 * @adev: The device to attach the debugfs entries to 1736 */ 1737 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) 1738 { 1739 struct drm_minor *minor = adev_to_drm(adev)->primary; 1740 struct dentry *ent, *root = minor->debugfs_root; 1741 unsigned int i; 1742 1743 if (security_locked_down(LOCKDOWN_PCI_ACCESS)) { 1744 drm_info(adev_to_drm(adev), 1745 "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n"); 1746 return 0; 1747 } 1748 1749 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { 1750 ent = debugfs_create_file(debugfs_regs_names[i], 1751 S_IFREG | 0400, root, 1752 adev, debugfs_regs[i]); 1753 if (!i && !IS_ERR_OR_NULL(ent)) 1754 i_size_write(ent->d_inode, adev->rmmio_size); 1755 } 1756 1757 return 0; 1758 } 1759 1760 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) 1761 { 1762 struct amdgpu_device *adev = m->private; 1763 struct drm_device *dev = adev_to_drm(adev); 1764 int r = 0, i; 1765 1766 r = pm_runtime_get_sync(dev->dev); 1767 if (r < 0) { 1768 pm_runtime_put_autosuspend(dev->dev); 1769 return r; 1770 } 1771 1772 /* Avoid accidently unparking the sched thread during GPU reset */ 1773 r = down_write_killable(&adev->reset_domain->sem); 1774 if (r) 1775 return r; 1776 1777 /* hold on the scheduler */ 1778 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 1779 struct amdgpu_ring *ring = adev->rings[i]; 1780 1781 if (!amdgpu_ring_sched_ready(ring)) 1782 continue; 1783 drm_sched_wqueue_stop(&ring->sched); 1784 } 1785 1786 seq_puts(m, "run ib test:\n"); 1787 r = amdgpu_ib_ring_tests(adev); 1788 if (r) 1789 seq_printf(m, "ib ring tests failed (%d).\n", r); 1790 else 1791 seq_puts(m, "ib ring tests passed.\n"); 1792 1793 /* go on the scheduler */ 1794 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 1795 struct amdgpu_ring *ring = adev->rings[i]; 1796 1797 if (!amdgpu_ring_sched_ready(ring)) 1798 continue; 1799 drm_sched_wqueue_start(&ring->sched); 1800 } 1801 1802 up_write(&adev->reset_domain->sem); 1803 1804 pm_runtime_put_autosuspend(dev->dev); 1805 1806 return 0; 1807 } 1808 1809 static int amdgpu_debugfs_evict_vram(void *data, u64 *val) 1810 { 1811 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1812 struct drm_device *dev = adev_to_drm(adev); 1813 int r; 1814 1815 r = pm_runtime_get_sync(dev->dev); 1816 if (r < 0) { 1817 pm_runtime_put_autosuspend(dev->dev); 1818 return r; 1819 } 1820 1821 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 1822 1823 pm_runtime_put_autosuspend(dev->dev); 1824 1825 return 0; 1826 } 1827 1828 1829 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val) 1830 { 1831 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1832 struct drm_device *dev = adev_to_drm(adev); 1833 int r; 1834 1835 r = pm_runtime_get_sync(dev->dev); 1836 if (r < 0) { 1837 pm_runtime_put_autosuspend(dev->dev); 1838 return r; 1839 } 1840 1841 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT); 1842 1843 pm_runtime_put_autosuspend(dev->dev); 1844 1845 return 0; 1846 } 1847 1848 static int amdgpu_debugfs_benchmark(void *data, u64 val) 1849 { 1850 struct amdgpu_device *adev = (struct amdgpu_device *)data; 1851 struct drm_device *dev = adev_to_drm(adev); 1852 int r; 1853 1854 r = pm_runtime_get_sync(dev->dev); 1855 if (r < 0) { 1856 pm_runtime_put_autosuspend(dev->dev); 1857 return r; 1858 } 1859 1860 r = amdgpu_benchmark(adev, val); 1861 1862 pm_runtime_put_autosuspend(dev->dev); 1863 1864 return r; 1865 } 1866 1867 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused) 1868 { 1869 struct amdgpu_device *adev = m->private; 1870 struct drm_device *dev = adev_to_drm(adev); 1871 struct drm_file *file; 1872 int r; 1873 1874 r = mutex_lock_interruptible(&dev->filelist_mutex); 1875 if (r) 1876 return r; 1877 1878 list_for_each_entry(file, &dev->filelist, lhead) { 1879 struct amdgpu_fpriv *fpriv = file->driver_priv; 1880 struct amdgpu_vm *vm = &fpriv->vm; 1881 struct amdgpu_task_info *ti; 1882 1883 ti = amdgpu_vm_get_task_info_vm(vm); 1884 if (ti) { 1885 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->task.pid, ti->process_name); 1886 amdgpu_vm_put_task_info(ti); 1887 } 1888 1889 r = amdgpu_bo_reserve(vm->root.bo, true); 1890 if (r) 1891 break; 1892 amdgpu_debugfs_vm_bo_info(vm, m); 1893 amdgpu_bo_unreserve(vm->root.bo); 1894 } 1895 1896 mutex_unlock(&dev->filelist_mutex); 1897 1898 return r; 1899 } 1900 1901 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib); 1902 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info); 1903 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram, 1904 NULL, "%lld\n"); 1905 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt, 1906 NULL, "%lld\n"); 1907 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark, 1908 "%lld\n"); 1909 1910 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring, 1911 struct dma_fence **fences) 1912 { 1913 struct amdgpu_fence_driver *drv = &ring->fence_drv; 1914 uint32_t sync_seq, last_seq; 1915 1916 last_seq = atomic_read(&ring->fence_drv.last_seq); 1917 sync_seq = ring->fence_drv.sync_seq; 1918 1919 last_seq &= drv->num_fences_mask; 1920 sync_seq &= drv->num_fences_mask; 1921 1922 do { 1923 struct dma_fence *fence, **ptr; 1924 1925 ++last_seq; 1926 last_seq &= drv->num_fences_mask; 1927 ptr = &drv->fences[last_seq]; 1928 1929 fence = rcu_dereference_protected(*ptr, 1); 1930 RCU_INIT_POINTER(*ptr, NULL); 1931 1932 if (!fence) 1933 continue; 1934 1935 fences[last_seq] = fence; 1936 1937 } while (last_seq != sync_seq); 1938 } 1939 1940 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences, 1941 int length) 1942 { 1943 int i; 1944 struct dma_fence *fence; 1945 1946 for (i = 0; i < length; i++) { 1947 fence = fences[i]; 1948 if (!fence) 1949 continue; 1950 dma_fence_signal(fence); 1951 dma_fence_put(fence); 1952 } 1953 } 1954 1955 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) 1956 { 1957 struct drm_sched_job *s_job; 1958 struct dma_fence *fence; 1959 1960 spin_lock(&sched->job_list_lock); 1961 list_for_each_entry(s_job, &sched->pending_list, list) { 1962 fence = sched->ops->run_job(s_job); 1963 dma_fence_put(fence); 1964 } 1965 spin_unlock(&sched->job_list_lock); 1966 } 1967 1968 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) 1969 { 1970 struct amdgpu_job *job; 1971 struct drm_sched_job *s_job, *tmp; 1972 uint32_t preempt_seq; 1973 struct dma_fence *fence, **ptr; 1974 struct amdgpu_fence_driver *drv = &ring->fence_drv; 1975 struct drm_gpu_scheduler *sched = &ring->sched; 1976 bool preempted = true; 1977 1978 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX) 1979 return; 1980 1981 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2)); 1982 if (preempt_seq <= atomic_read(&drv->last_seq)) { 1983 preempted = false; 1984 goto no_preempt; 1985 } 1986 1987 preempt_seq &= drv->num_fences_mask; 1988 ptr = &drv->fences[preempt_seq]; 1989 fence = rcu_dereference_protected(*ptr, 1); 1990 1991 no_preempt: 1992 spin_lock(&sched->job_list_lock); 1993 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { 1994 if (dma_fence_is_signaled(&s_job->s_fence->finished)) { 1995 /* remove job from ring_mirror_list */ 1996 list_del_init(&s_job->list); 1997 sched->ops->free_job(s_job); 1998 continue; 1999 } 2000 job = to_amdgpu_job(s_job); 2001 if (preempted && (&job->hw_fence->base) == fence) 2002 /* mark the job as preempted */ 2003 job->preemption_status |= AMDGPU_IB_PREEMPTED; 2004 } 2005 spin_unlock(&sched->job_list_lock); 2006 } 2007 2008 static int amdgpu_debugfs_ib_preempt(void *data, u64 val) 2009 { 2010 int r, length; 2011 struct amdgpu_ring *ring; 2012 struct dma_fence **fences = NULL; 2013 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2014 2015 if (val >= AMDGPU_MAX_RINGS) 2016 return -EINVAL; 2017 2018 ring = adev->rings[val]; 2019 2020 if (!amdgpu_ring_sched_ready(ring) || 2021 !ring->funcs->preempt_ib) 2022 return -EINVAL; 2023 2024 /* the last preemption failed */ 2025 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr)) 2026 return -EBUSY; 2027 2028 length = ring->fence_drv.num_fences_mask + 1; 2029 fences = kcalloc(length, sizeof(void *), GFP_KERNEL); 2030 if (!fences) 2031 return -ENOMEM; 2032 2033 /* Avoid accidently unparking the sched thread during GPU reset */ 2034 r = down_read_killable(&adev->reset_domain->sem); 2035 if (r) 2036 goto pro_end; 2037 2038 /* stop the scheduler */ 2039 drm_sched_wqueue_stop(&ring->sched); 2040 2041 /* preempt the IB */ 2042 r = amdgpu_ring_preempt_ib(ring); 2043 if (r) { 2044 drm_warn(adev_to_drm(adev), "failed to preempt ring %d\n", ring->idx); 2045 goto failure; 2046 } 2047 2048 amdgpu_fence_process(ring); 2049 2050 if (atomic_read(&ring->fence_drv.last_seq) != 2051 ring->fence_drv.sync_seq) { 2052 drm_info(adev_to_drm(adev), "ring %d was preempted\n", ring->idx); 2053 2054 amdgpu_ib_preempt_mark_partial_job(ring); 2055 2056 /* swap out the old fences */ 2057 amdgpu_ib_preempt_fences_swap(ring, fences); 2058 2059 amdgpu_fence_driver_force_completion(ring, NULL); 2060 2061 /* resubmit unfinished jobs */ 2062 amdgpu_ib_preempt_job_recovery(&ring->sched); 2063 2064 /* wait for jobs finished */ 2065 amdgpu_fence_wait_empty(ring); 2066 2067 /* signal the old fences */ 2068 amdgpu_ib_preempt_signal_fences(fences, length); 2069 } 2070 2071 failure: 2072 /* restart the scheduler */ 2073 drm_sched_wqueue_start(&ring->sched); 2074 2075 up_read(&adev->reset_domain->sem); 2076 2077 pro_end: 2078 kfree(fences); 2079 2080 return r; 2081 } 2082 2083 static int amdgpu_debugfs_sclk_set(void *data, u64 val) 2084 { 2085 int ret = 0; 2086 uint32_t max_freq, min_freq; 2087 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2088 2089 if (amdgpu_sriov_multi_vf_mode(adev)) 2090 return -EINVAL; 2091 2092 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev); 2093 if (ret < 0) { 2094 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 2095 return ret; 2096 } 2097 2098 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq); 2099 if (ret == -EOPNOTSUPP) { 2100 ret = 0; 2101 goto out; 2102 } 2103 if (ret || val > max_freq || val < min_freq) { 2104 ret = -EINVAL; 2105 goto out; 2106 } 2107 2108 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val); 2109 if (ret) 2110 ret = -EINVAL; 2111 2112 out: 2113 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); 2114 2115 return ret; 2116 } 2117 2118 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL, 2119 amdgpu_debugfs_ib_preempt, "%llu\n"); 2120 2121 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL, 2122 amdgpu_debugfs_sclk_set, "%llu\n"); 2123 2124 int amdgpu_debugfs_init(struct amdgpu_device *adev) 2125 { 2126 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root; 2127 struct dentry *ent; 2128 int r, i; 2129 2130 if (!debugfs_initialized()) 2131 return 0; 2132 2133 debugfs_create_x32("amdgpu_smu_debug", 0600, root, 2134 &adev->pm.smu_debug_mask); 2135 2136 debugfs_create_x64("unique_id", 0444, root, &adev->unique_id); 2137 debugfs_create_x8("unitid", 0444, root, &adev->unitid); 2138 2139 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev, 2140 &fops_ib_preempt); 2141 if (IS_ERR(ent)) { 2142 drm_err(adev_to_drm(adev), 2143 "unable to create amdgpu_preempt_ib debugsfs file\n"); 2144 return PTR_ERR(ent); 2145 } 2146 2147 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev, 2148 &fops_sclk_set); 2149 if (IS_ERR(ent)) { 2150 drm_err(adev_to_drm(adev), 2151 "unable to create amdgpu_set_sclk debugsfs file\n"); 2152 return PTR_ERR(ent); 2153 } 2154 2155 /* Register debugfs entries for amdgpu_ttm */ 2156 amdgpu_ttm_debugfs_init(adev); 2157 amdgpu_debugfs_pm_init(adev); 2158 amdgpu_debugfs_sa_init(adev); 2159 amdgpu_debugfs_fence_init(adev); 2160 amdgpu_debugfs_gem_init(adev); 2161 2162 r = amdgpu_debugfs_regs_init(adev); 2163 if (r) 2164 drm_err(adev_to_drm(adev), "registering register debugfs failed (%d).\n", r); 2165 2166 amdgpu_debugfs_firmware_init(adev); 2167 amdgpu_ta_if_debugfs_init(adev); 2168 2169 amdgpu_debugfs_mes_event_log_init(adev); 2170 2171 #if defined(CONFIG_DRM_AMD_DC) 2172 if (adev->dc_enabled) 2173 dtn_debugfs_init(adev); 2174 #endif 2175 2176 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2177 struct amdgpu_ring *ring = adev->rings[i]; 2178 2179 if (!ring) 2180 continue; 2181 if (ring == &adev->cper.ring_buf && !adev->cper.enabled) 2182 continue; 2183 2184 amdgpu_debugfs_ring_init(adev, ring); 2185 } 2186 2187 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 2188 if (!amdgpu_vcnfw_log) 2189 break; 2190 2191 if (adev->vcn.harvest_config & (1 << i)) 2192 continue; 2193 2194 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]); 2195 } 2196 2197 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog) 2198 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm); 2199 2200 amdgpu_debugfs_vcn_sched_mask_init(adev); 2201 amdgpu_debugfs_jpeg_sched_mask_init(adev); 2202 amdgpu_debugfs_gfx_sched_mask_init(adev); 2203 amdgpu_debugfs_compute_sched_mask_init(adev); 2204 amdgpu_debugfs_sdma_sched_mask_init(adev); 2205 2206 amdgpu_ras_debugfs_create_all(adev); 2207 amdgpu_rap_debugfs_init(adev); 2208 amdgpu_securedisplay_debugfs_init(adev); 2209 amdgpu_fw_attestation_debugfs_init(adev); 2210 amdgpu_psp_debugfs_init(adev); 2211 2212 debugfs_create_file("amdgpu_evict_vram", 0400, root, adev, 2213 &amdgpu_evict_vram_fops); 2214 debugfs_create_file("amdgpu_evict_gtt", 0400, root, adev, 2215 &amdgpu_evict_gtt_fops); 2216 debugfs_create_file("amdgpu_test_ib", 0400, root, adev, 2217 &amdgpu_debugfs_test_ib_fops); 2218 debugfs_create_file("amdgpu_vm_info", 0444, root, adev, 2219 &amdgpu_debugfs_vm_info_fops); 2220 debugfs_create_file("amdgpu_benchmark", 0200, root, adev, 2221 &amdgpu_benchmark_fops); 2222 2223 adev->debugfs_vbios_blob.data = adev->bios; 2224 adev->debugfs_vbios_blob.size = adev->bios_size; 2225 debugfs_create_blob("amdgpu_vbios", 0444, root, 2226 &adev->debugfs_vbios_blob); 2227 2228 if (adev->discovery.debugfs_blob.size) 2229 debugfs_create_blob("amdgpu_discovery", 0444, root, 2230 &adev->discovery.debugfs_blob); 2231 2232 return 0; 2233 } 2234 2235 static int amdgpu_pt_info_read(struct seq_file *m, void *unused) 2236 { 2237 struct drm_file *file; 2238 struct amdgpu_fpriv *fpriv; 2239 struct amdgpu_bo *root_bo; 2240 struct amdgpu_device *adev; 2241 int r; 2242 2243 file = m->private; 2244 if (!file) 2245 return -EINVAL; 2246 2247 adev = drm_to_adev(file->minor->dev); 2248 fpriv = file->driver_priv; 2249 if (!fpriv || !fpriv->vm.root.bo) 2250 return -ENODEV; 2251 2252 root_bo = amdgpu_bo_ref(fpriv->vm.root.bo); 2253 r = amdgpu_bo_reserve(root_bo, true); 2254 if (r) { 2255 amdgpu_bo_unref(&root_bo); 2256 return -EINVAL; 2257 } 2258 2259 seq_printf(m, "pd_address: 0x%llx\n", amdgpu_gmc_pd_addr(fpriv->vm.root.bo)); 2260 seq_printf(m, "max_pfn: 0x%llx\n", adev->vm_manager.max_pfn); 2261 seq_printf(m, "num_level: 0x%x\n", adev->vm_manager.num_level); 2262 seq_printf(m, "block_size: 0x%x\n", adev->vm_manager.block_size); 2263 seq_printf(m, "fragment_size: 0x%x\n", adev->vm_manager.fragment_size); 2264 2265 amdgpu_bo_unreserve(root_bo); 2266 amdgpu_bo_unref(&root_bo); 2267 2268 return 0; 2269 } 2270 2271 static int amdgpu_pt_info_open(struct inode *inode, struct file *file) 2272 { 2273 return single_open(file, amdgpu_pt_info_read, inode->i_private); 2274 } 2275 2276 static const struct file_operations amdgpu_pt_info_fops = { 2277 .owner = THIS_MODULE, 2278 .open = amdgpu_pt_info_open, 2279 .read = seq_read, 2280 .llseek = seq_lseek, 2281 .release = single_release, 2282 }; 2283 2284 static int amdgpu_mqd_info_read(struct seq_file *m, void *unused) 2285 { 2286 struct amdgpu_usermode_queue *queue = m->private; 2287 struct amdgpu_bo *bo; 2288 int r; 2289 2290 if (!queue || !queue->mqd.obj) 2291 return -EINVAL; 2292 2293 bo = amdgpu_bo_ref(queue->mqd.obj); 2294 r = amdgpu_bo_reserve(bo, true); 2295 if (r) { 2296 amdgpu_bo_unref(&bo); 2297 return -EINVAL; 2298 } 2299 2300 seq_printf(m, "queue_type: %d\n", queue->queue_type); 2301 seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj)); 2302 2303 amdgpu_bo_unreserve(bo); 2304 amdgpu_bo_unref(&bo); 2305 2306 return 0; 2307 } 2308 2309 static int amdgpu_mqd_info_open(struct inode *inode, struct file *file) 2310 { 2311 return single_open(file, amdgpu_mqd_info_read, inode->i_private); 2312 } 2313 2314 static const struct file_operations amdgpu_mqd_info_fops = { 2315 .owner = THIS_MODULE, 2316 .open = amdgpu_mqd_info_open, 2317 .read = seq_read, 2318 .llseek = seq_lseek, 2319 .release = single_release, 2320 }; 2321 2322 void amdgpu_debugfs_userq_init(struct drm_file *file, struct amdgpu_usermode_queue *queue, int qid) 2323 { 2324 char queue_name[32]; 2325 2326 scnprintf(queue_name, sizeof(queue_name), "queue_%d", qid); 2327 queue->debugfs_queue = debugfs_create_dir(queue_name, file->debugfs_client); 2328 debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops); 2329 } 2330 2331 void amdgpu_debugfs_vm_init(struct drm_file *file) 2332 { 2333 debugfs_create_file("vm_pagetable_info", 0444, file->debugfs_client, file, 2334 &amdgpu_pt_info_fops); 2335 } 2336 2337 #else 2338 int amdgpu_debugfs_init(struct amdgpu_device *adev) 2339 { 2340 return 0; 2341 } 2342 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) 2343 { 2344 return 0; 2345 } 2346 void amdgpu_debugfs_vm_init(struct drm_file *file) 2347 { 2348 } 2349 void amdgpu_debugfs_userq_init(struct drm_file *file, 2350 struct amdgpu_usermode_queue *queue, 2351 int qid) 2352 { 2353 } 2354 #endif 2355