1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/firmware.h> 27 #include <linux/pm_runtime.h> 28 29 #include "amdgpu.h" 30 #include "amdgpu_gfx.h" 31 #include "amdgpu_rlc.h" 32 #include "amdgpu_ras.h" 33 #include "amdgpu_reset.h" 34 #include "amdgpu_xcp.h" 35 #include "amdgpu_xgmi.h" 36 37 /* delay 0.1 second to enable gfx off feature */ 38 #define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100) 39 40 #define GFX_OFF_NO_DELAY 0 41 42 /* 43 * GPU GFX IP block helpers function. 44 */ 45 46 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, 47 int pipe, int queue) 48 { 49 int bit = 0; 50 51 bit += mec * adev->gfx.mec.num_pipe_per_mec 52 * adev->gfx.mec.num_queue_per_pipe; 53 bit += pipe * adev->gfx.mec.num_queue_per_pipe; 54 bit += queue; 55 56 return bit; 57 } 58 59 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, 60 int *mec, int *pipe, int *queue) 61 { 62 *queue = bit % adev->gfx.mec.num_queue_per_pipe; 63 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) 64 % adev->gfx.mec.num_pipe_per_mec; 65 *mec = (bit / adev->gfx.mec.num_queue_per_pipe) 66 / adev->gfx.mec.num_pipe_per_mec; 67 68 } 69 70 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, 71 int xcc_id, int mec, int pipe, int queue) 72 { 73 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), 74 adev->gfx.mec_bitmap[xcc_id].queue_bitmap); 75 } 76 77 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, 78 int me, int pipe, int queue) 79 { 80 int bit = 0; 81 82 bit += me * adev->gfx.me.num_pipe_per_me 83 * adev->gfx.me.num_queue_per_pipe; 84 bit += pipe * adev->gfx.me.num_queue_per_pipe; 85 bit += queue; 86 87 return bit; 88 } 89 90 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, 91 int me, int pipe, int queue) 92 { 93 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), 94 adev->gfx.me.queue_bitmap); 95 } 96 97 /** 98 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter 99 * 100 * @mask: array in which the per-shader array disable masks will be stored 101 * @max_se: number of SEs 102 * @max_sh: number of SHs 103 * 104 * The bitmask of CUs to be disabled in the shader array determined by se and 105 * sh is stored in mask[se * max_sh + sh]. 106 */ 107 void amdgpu_gfx_parse_disable_cu(unsigned int *mask, unsigned int max_se, unsigned int max_sh) 108 { 109 unsigned int se, sh, cu; 110 const char *p; 111 112 memset(mask, 0, sizeof(*mask) * max_se * max_sh); 113 114 if (!amdgpu_disable_cu || !*amdgpu_disable_cu) 115 return; 116 117 p = amdgpu_disable_cu; 118 for (;;) { 119 char *next; 120 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); 121 122 if (ret < 3) { 123 DRM_ERROR("amdgpu: could not parse disable_cu\n"); 124 return; 125 } 126 127 if (se < max_se && sh < max_sh && cu < 16) { 128 DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); 129 mask[se * max_sh + sh] |= 1u << cu; 130 } else { 131 DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", 132 se, sh, cu); 133 } 134 135 next = strchr(p, ','); 136 if (!next) 137 break; 138 p = next + 1; 139 } 140 } 141 142 static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev) 143 { 144 return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1; 145 } 146 147 static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev) 148 { 149 if (amdgpu_compute_multipipe != -1) { 150 DRM_INFO("amdgpu: forcing compute pipe policy %d\n", 151 amdgpu_compute_multipipe); 152 return amdgpu_compute_multipipe == 1; 153 } 154 155 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 156 return true; 157 158 /* FIXME: spreading the queues across pipes causes perf regressions 159 * on POLARIS11 compute workloads */ 160 if (adev->asic_type == CHIP_POLARIS11) 161 return false; 162 163 return adev->gfx.mec.num_mec > 1; 164 } 165 166 bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev, 167 struct amdgpu_ring *ring) 168 { 169 int queue = ring->queue; 170 int pipe = ring->pipe; 171 172 /* Policy: use pipe1 queue0 as high priority graphics queue if we 173 * have more than one gfx pipe. 174 */ 175 if (amdgpu_gfx_is_graphics_multipipe_capable(adev) && 176 adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) { 177 int me = ring->me; 178 int bit; 179 180 bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue); 181 if (ring == &adev->gfx.gfx_ring[bit]) 182 return true; 183 } 184 185 return false; 186 } 187 188 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, 189 struct amdgpu_ring *ring) 190 { 191 /* Policy: use 1st queue as high priority compute queue if we 192 * have more than one compute queue. 193 */ 194 if (adev->gfx.num_compute_rings > 1 && 195 ring == &adev->gfx.compute_ring[0]) 196 return true; 197 198 return false; 199 } 200 201 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) 202 { 203 int i, j, queue, pipe; 204 bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev); 205 int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec * 206 adev->gfx.mec.num_queue_per_pipe, 207 adev->gfx.num_compute_rings); 208 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 209 210 if (multipipe_policy) { 211 /* policy: make queues evenly cross all pipes on MEC1 only 212 * for multiple xcc, just use the original policy for simplicity */ 213 for (j = 0; j < num_xcc; j++) { 214 for (i = 0; i < max_queues_per_mec; i++) { 215 pipe = i % adev->gfx.mec.num_pipe_per_mec; 216 queue = (i / adev->gfx.mec.num_pipe_per_mec) % 217 adev->gfx.mec.num_queue_per_pipe; 218 219 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue, 220 adev->gfx.mec_bitmap[j].queue_bitmap); 221 } 222 } 223 } else { 224 /* policy: amdgpu owns all queues in the given pipe */ 225 for (j = 0; j < num_xcc; j++) { 226 for (i = 0; i < max_queues_per_mec; ++i) 227 set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap); 228 } 229 } 230 231 for (j = 0; j < num_xcc; j++) { 232 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", 233 bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); 234 } 235 } 236 237 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) 238 { 239 int i, queue, pipe; 240 bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev); 241 int max_queues_per_me = adev->gfx.me.num_pipe_per_me * 242 adev->gfx.me.num_queue_per_pipe; 243 244 if (multipipe_policy) { 245 /* policy: amdgpu owns the first queue per pipe at this stage 246 * will extend to mulitple queues per pipe later */ 247 for (i = 0; i < max_queues_per_me; i++) { 248 pipe = i % adev->gfx.me.num_pipe_per_me; 249 queue = (i / adev->gfx.me.num_pipe_per_me) % 250 adev->gfx.me.num_queue_per_pipe; 251 252 set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue, 253 adev->gfx.me.queue_bitmap); 254 } 255 } else { 256 for (i = 0; i < max_queues_per_me; ++i) 257 set_bit(i, adev->gfx.me.queue_bitmap); 258 } 259 260 /* update the number of active graphics rings */ 261 adev->gfx.num_gfx_rings = 262 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 263 } 264 265 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, 266 struct amdgpu_ring *ring, int xcc_id) 267 { 268 int queue_bit; 269 int mec, pipe, queue; 270 271 queue_bit = adev->gfx.mec.num_mec 272 * adev->gfx.mec.num_pipe_per_mec 273 * adev->gfx.mec.num_queue_per_pipe; 274 275 while (--queue_bit >= 0) { 276 if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) 277 continue; 278 279 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 280 281 /* 282 * 1. Using pipes 2/3 from MEC 2 seems cause problems. 283 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN 284 * only can be issued on queue 0. 285 */ 286 if ((mec == 1 && pipe > 1) || queue != 0) 287 continue; 288 289 ring->me = mec + 1; 290 ring->pipe = pipe; 291 ring->queue = queue; 292 293 return 0; 294 } 295 296 dev_err(adev->dev, "Failed to find a queue for KIQ\n"); 297 return -EINVAL; 298 } 299 300 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id) 301 { 302 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 303 struct amdgpu_irq_src *irq = &kiq->irq; 304 struct amdgpu_ring *ring = &kiq->ring; 305 int r = 0; 306 307 spin_lock_init(&kiq->ring_lock); 308 309 ring->adev = NULL; 310 ring->ring_obj = NULL; 311 ring->use_doorbell = true; 312 ring->xcc_id = xcc_id; 313 ring->vm_hub = AMDGPU_GFXHUB(xcc_id); 314 ring->doorbell_index = 315 (adev->doorbell_index.kiq + 316 xcc_id * adev->doorbell_index.xcc_doorbell_range) 317 << 1; 318 319 r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id); 320 if (r) 321 return r; 322 323 ring->eop_gpu_addr = kiq->eop_gpu_addr; 324 ring->no_scheduler = true; 325 snprintf(ring->name, sizeof(ring->name), "kiq_%hhu.%hhu.%hhu.%hhu", 326 (unsigned char)xcc_id, (unsigned char)ring->me, 327 (unsigned char)ring->pipe, (unsigned char)ring->queue); 328 r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0, 329 AMDGPU_RING_PRIO_DEFAULT, NULL); 330 if (r) 331 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); 332 333 return r; 334 } 335 336 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring) 337 { 338 amdgpu_ring_fini(ring); 339 } 340 341 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id) 342 { 343 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 344 345 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL); 346 } 347 348 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, 349 unsigned int hpd_size, int xcc_id) 350 { 351 int r; 352 u32 *hpd; 353 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 354 355 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, 356 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj, 357 &kiq->eop_gpu_addr, (void **)&hpd); 358 if (r) { 359 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); 360 return r; 361 } 362 363 memset(hpd, 0, hpd_size); 364 365 r = amdgpu_bo_reserve(kiq->eop_obj, true); 366 if (unlikely(r != 0)) 367 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 368 amdgpu_bo_kunmap(kiq->eop_obj); 369 amdgpu_bo_unreserve(kiq->eop_obj); 370 371 return 0; 372 } 373 374 /* create MQD for each compute/gfx queue */ 375 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, 376 unsigned int mqd_size, int xcc_id) 377 { 378 int r, i, j; 379 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 380 struct amdgpu_ring *ring = &kiq->ring; 381 u32 domain = AMDGPU_GEM_DOMAIN_GTT; 382 383 #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) 384 /* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */ 385 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0)) 386 domain |= AMDGPU_GEM_DOMAIN_VRAM; 387 #endif 388 389 /* create MQD for KIQ */ 390 if (!adev->enable_mes_kiq && !ring->mqd_obj) { 391 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must 392 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD 393 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for 394 * KIQ MQD no matter SRIOV or Bare-metal 395 */ 396 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 397 AMDGPU_GEM_DOMAIN_VRAM | 398 AMDGPU_GEM_DOMAIN_GTT, 399 &ring->mqd_obj, 400 &ring->mqd_gpu_addr, 401 &ring->mqd_ptr); 402 if (r) { 403 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); 404 return r; 405 } 406 407 /* prepare MQD backup */ 408 kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL); 409 if (!kiq->mqd_backup) { 410 dev_warn(adev->dev, 411 "no memory to create MQD backup for ring %s\n", ring->name); 412 return -ENOMEM; 413 } 414 } 415 416 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 417 /* create MQD for each KGQ */ 418 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 419 ring = &adev->gfx.gfx_ring[i]; 420 if (!ring->mqd_obj) { 421 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 422 domain, &ring->mqd_obj, 423 &ring->mqd_gpu_addr, &ring->mqd_ptr); 424 if (r) { 425 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 426 return r; 427 } 428 429 ring->mqd_size = mqd_size; 430 /* prepare MQD backup */ 431 adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL); 432 if (!adev->gfx.me.mqd_backup[i]) { 433 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 434 return -ENOMEM; 435 } 436 } 437 } 438 } 439 440 /* create MQD for each KCQ */ 441 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 442 j = i + xcc_id * adev->gfx.num_compute_rings; 443 ring = &adev->gfx.compute_ring[j]; 444 if (!ring->mqd_obj) { 445 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 446 domain, &ring->mqd_obj, 447 &ring->mqd_gpu_addr, &ring->mqd_ptr); 448 if (r) { 449 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 450 return r; 451 } 452 453 ring->mqd_size = mqd_size; 454 /* prepare MQD backup */ 455 adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL); 456 if (!adev->gfx.mec.mqd_backup[j]) { 457 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 458 return -ENOMEM; 459 } 460 } 461 } 462 463 return 0; 464 } 465 466 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id) 467 { 468 struct amdgpu_ring *ring = NULL; 469 int i, j; 470 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 471 472 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 473 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 474 ring = &adev->gfx.gfx_ring[i]; 475 kfree(adev->gfx.me.mqd_backup[i]); 476 amdgpu_bo_free_kernel(&ring->mqd_obj, 477 &ring->mqd_gpu_addr, 478 &ring->mqd_ptr); 479 } 480 } 481 482 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 483 j = i + xcc_id * adev->gfx.num_compute_rings; 484 ring = &adev->gfx.compute_ring[j]; 485 kfree(adev->gfx.mec.mqd_backup[j]); 486 amdgpu_bo_free_kernel(&ring->mqd_obj, 487 &ring->mqd_gpu_addr, 488 &ring->mqd_ptr); 489 } 490 491 ring = &kiq->ring; 492 kfree(kiq->mqd_backup); 493 amdgpu_bo_free_kernel(&ring->mqd_obj, 494 &ring->mqd_gpu_addr, 495 &ring->mqd_ptr); 496 } 497 498 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id) 499 { 500 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 501 struct amdgpu_ring *kiq_ring = &kiq->ring; 502 int i, r = 0; 503 int j; 504 505 if (adev->enable_mes) { 506 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 507 j = i + xcc_id * adev->gfx.num_compute_rings; 508 amdgpu_mes_unmap_legacy_queue(adev, 509 &adev->gfx.compute_ring[j], 510 RESET_QUEUES, 0, 0); 511 } 512 return 0; 513 } 514 515 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 516 return -EINVAL; 517 518 if (!kiq_ring->sched.ready || amdgpu_in_reset(adev)) 519 return 0; 520 521 spin_lock(&kiq->ring_lock); 522 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 523 adev->gfx.num_compute_rings)) { 524 spin_unlock(&kiq->ring_lock); 525 return -ENOMEM; 526 } 527 528 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 529 j = i + xcc_id * adev->gfx.num_compute_rings; 530 kiq->pmf->kiq_unmap_queues(kiq_ring, 531 &adev->gfx.compute_ring[j], 532 RESET_QUEUES, 0, 0); 533 } 534 /* Submit unmap queue packet */ 535 amdgpu_ring_commit(kiq_ring); 536 /* 537 * Ring test will do a basic scratch register change check. Just run 538 * this to ensure that unmap queues that is submitted before got 539 * processed successfully before returning. 540 */ 541 r = amdgpu_ring_test_helper(kiq_ring); 542 543 spin_unlock(&kiq->ring_lock); 544 545 return r; 546 } 547 548 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id) 549 { 550 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 551 struct amdgpu_ring *kiq_ring = &kiq->ring; 552 int i, r = 0; 553 int j; 554 555 if (adev->enable_mes) { 556 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 557 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 558 j = i + xcc_id * adev->gfx.num_gfx_rings; 559 amdgpu_mes_unmap_legacy_queue(adev, 560 &adev->gfx.gfx_ring[j], 561 PREEMPT_QUEUES, 0, 0); 562 } 563 } 564 return 0; 565 } 566 567 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 568 return -EINVAL; 569 570 if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev)) 571 return 0; 572 573 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 574 spin_lock(&kiq->ring_lock); 575 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 576 adev->gfx.num_gfx_rings)) { 577 spin_unlock(&kiq->ring_lock); 578 return -ENOMEM; 579 } 580 581 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 582 j = i + xcc_id * adev->gfx.num_gfx_rings; 583 kiq->pmf->kiq_unmap_queues(kiq_ring, 584 &adev->gfx.gfx_ring[j], 585 PREEMPT_QUEUES, 0, 0); 586 } 587 /* Submit unmap queue packet */ 588 amdgpu_ring_commit(kiq_ring); 589 590 /* 591 * Ring test will do a basic scratch register change check. 592 * Just run this to ensure that unmap queues that is submitted 593 * before got processed successfully before returning. 594 */ 595 r = amdgpu_ring_test_helper(kiq_ring); 596 spin_unlock(&kiq->ring_lock); 597 } 598 599 return r; 600 } 601 602 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, 603 int queue_bit) 604 { 605 int mec, pipe, queue; 606 int set_resource_bit = 0; 607 608 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 609 610 set_resource_bit = mec * 4 * 8 + pipe * 8 + queue; 611 612 return set_resource_bit; 613 } 614 615 static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id) 616 { 617 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 618 struct amdgpu_ring *kiq_ring = &kiq->ring; 619 uint64_t queue_mask = ~0ULL; 620 int r, i, j; 621 622 amdgpu_device_flush_hdp(adev, NULL); 623 624 if (!adev->enable_uni_mes) { 625 spin_lock(&kiq->ring_lock); 626 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->set_resources_size); 627 if (r) { 628 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r); 629 spin_unlock(&kiq->ring_lock); 630 return r; 631 } 632 633 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 634 r = amdgpu_ring_test_helper(kiq_ring); 635 spin_unlock(&kiq->ring_lock); 636 if (r) 637 dev_err(adev->dev, "KIQ failed to set resources\n"); 638 } 639 640 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 641 j = i + xcc_id * adev->gfx.num_compute_rings; 642 r = amdgpu_mes_map_legacy_queue(adev, 643 &adev->gfx.compute_ring[j]); 644 if (r) { 645 dev_err(adev->dev, "failed to map compute queue\n"); 646 return r; 647 } 648 } 649 650 return 0; 651 } 652 653 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id) 654 { 655 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 656 struct amdgpu_ring *kiq_ring = &kiq->ring; 657 uint64_t queue_mask = 0; 658 int r, i, j; 659 660 if (adev->mes.enable_legacy_queue_map) 661 return amdgpu_gfx_mes_enable_kcq(adev, xcc_id); 662 663 if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources) 664 return -EINVAL; 665 666 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 667 if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) 668 continue; 669 670 /* This situation may be hit in the future if a new HW 671 * generation exposes more than 64 queues. If so, the 672 * definition of queue_mask needs updating */ 673 if (WARN_ON(i > (sizeof(queue_mask)*8))) { 674 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 675 break; 676 } 677 678 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i)); 679 } 680 681 amdgpu_device_flush_hdp(adev, NULL); 682 683 DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe, 684 kiq_ring->queue); 685 686 spin_lock(&kiq->ring_lock); 687 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 688 adev->gfx.num_compute_rings + 689 kiq->pmf->set_resources_size); 690 if (r) { 691 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 692 spin_unlock(&kiq->ring_lock); 693 return r; 694 } 695 696 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 697 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 698 j = i + xcc_id * adev->gfx.num_compute_rings; 699 kiq->pmf->kiq_map_queues(kiq_ring, 700 &adev->gfx.compute_ring[j]); 701 } 702 /* Submit map queue packet */ 703 amdgpu_ring_commit(kiq_ring); 704 /* 705 * Ring test will do a basic scratch register change check. Just run 706 * this to ensure that map queues that is submitted before got 707 * processed successfully before returning. 708 */ 709 r = amdgpu_ring_test_helper(kiq_ring); 710 spin_unlock(&kiq->ring_lock); 711 if (r) 712 DRM_ERROR("KCQ enable failed\n"); 713 714 return r; 715 } 716 717 int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id) 718 { 719 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 720 struct amdgpu_ring *kiq_ring = &kiq->ring; 721 int r, i, j; 722 723 if (!kiq->pmf || !kiq->pmf->kiq_map_queues) 724 return -EINVAL; 725 726 amdgpu_device_flush_hdp(adev, NULL); 727 728 if (adev->mes.enable_legacy_queue_map) { 729 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 730 j = i + xcc_id * adev->gfx.num_gfx_rings; 731 r = amdgpu_mes_map_legacy_queue(adev, 732 &adev->gfx.gfx_ring[j]); 733 if (r) { 734 DRM_ERROR("failed to map gfx queue\n"); 735 return r; 736 } 737 } 738 739 return 0; 740 } 741 742 spin_lock(&kiq->ring_lock); 743 /* No need to map kcq on the slave */ 744 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 745 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 746 adev->gfx.num_gfx_rings); 747 if (r) { 748 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 749 spin_unlock(&kiq->ring_lock); 750 return r; 751 } 752 753 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 754 j = i + xcc_id * adev->gfx.num_gfx_rings; 755 kiq->pmf->kiq_map_queues(kiq_ring, 756 &adev->gfx.gfx_ring[j]); 757 } 758 } 759 /* Submit map queue packet */ 760 amdgpu_ring_commit(kiq_ring); 761 /* 762 * Ring test will do a basic scratch register change check. Just run 763 * this to ensure that map queues that is submitted before got 764 * processed successfully before returning. 765 */ 766 r = amdgpu_ring_test_helper(kiq_ring); 767 spin_unlock(&kiq->ring_lock); 768 if (r) 769 DRM_ERROR("KGQ enable failed\n"); 770 771 return r; 772 } 773 774 static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable, 775 bool no_delay) 776 { 777 unsigned long delay = GFX_OFF_DELAY_ENABLE; 778 779 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) 780 return; 781 782 mutex_lock(&adev->gfx.gfx_off_mutex); 783 784 if (enable) { 785 /* If the count is already 0, it means there's an imbalance bug somewhere. 786 * Note that the bug may be in a different caller than the one which triggers the 787 * WARN_ON_ONCE. 788 */ 789 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0)) 790 goto unlock; 791 792 adev->gfx.gfx_off_req_count--; 793 794 if (adev->gfx.gfx_off_req_count == 0 && 795 !adev->gfx.gfx_off_state) { 796 /* If going to s2idle, no need to wait */ 797 if (no_delay) { 798 if (!amdgpu_dpm_set_powergating_by_smu(adev, 799 AMD_IP_BLOCK_TYPE_GFX, true, 0)) 800 adev->gfx.gfx_off_state = true; 801 } else { 802 schedule_delayed_work(&adev->gfx.gfx_off_delay_work, 803 delay); 804 } 805 } 806 } else { 807 if (adev->gfx.gfx_off_req_count == 0) { 808 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 809 810 if (adev->gfx.gfx_off_state && 811 !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false, 0)) { 812 adev->gfx.gfx_off_state = false; 813 814 if (adev->gfx.funcs->init_spm_golden) { 815 dev_dbg(adev->dev, 816 "GFXOFF is disabled, re-init SPM golden settings\n"); 817 amdgpu_gfx_init_spm_golden(adev); 818 } 819 } 820 } 821 822 adev->gfx.gfx_off_req_count++; 823 } 824 825 unlock: 826 mutex_unlock(&adev->gfx.gfx_off_mutex); 827 } 828 829 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable 830 * 831 * @adev: amdgpu_device pointer 832 * @bool enable true: enable gfx off feature, false: disable gfx off feature 833 * 834 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled. 835 * 2. other client can send request to disable gfx off feature, the request should be honored. 836 * 3. other client can cancel their request of disable gfx off feature 837 * 4. other client should not send request to enable gfx off feature before disable gfx off feature. 838 * 839 * gfx off allow will be delayed by GFX_OFF_DELAY_ENABLE ms. 840 */ 841 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) 842 { 843 /* If going to s2idle, no need to wait */ 844 bool no_delay = adev->in_s0ix ? true : false; 845 846 amdgpu_gfx_do_off_ctrl(adev, enable, no_delay); 847 } 848 849 /* amdgpu_gfx_off_ctrl_immediate - Handle gfx off feature enable/disable 850 * 851 * @adev: amdgpu_device pointer 852 * @bool enable true: enable gfx off feature, false: disable gfx off feature 853 * 854 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled. 855 * 2. other client can send request to disable gfx off feature, the request should be honored. 856 * 3. other client can cancel their request of disable gfx off feature 857 * 4. other client should not send request to enable gfx off feature before disable gfx off feature. 858 * 859 * gfx off allow will be issued immediately. 860 */ 861 void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable) 862 { 863 amdgpu_gfx_do_off_ctrl(adev, enable, true); 864 } 865 866 int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value) 867 { 868 int r = 0; 869 870 mutex_lock(&adev->gfx.gfx_off_mutex); 871 872 r = amdgpu_dpm_set_residency_gfxoff(adev, value); 873 874 mutex_unlock(&adev->gfx.gfx_off_mutex); 875 876 return r; 877 } 878 879 int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value) 880 { 881 int r = 0; 882 883 mutex_lock(&adev->gfx.gfx_off_mutex); 884 885 r = amdgpu_dpm_get_residency_gfxoff(adev, value); 886 887 mutex_unlock(&adev->gfx.gfx_off_mutex); 888 889 return r; 890 } 891 892 int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value) 893 { 894 int r = 0; 895 896 mutex_lock(&adev->gfx.gfx_off_mutex); 897 898 r = amdgpu_dpm_get_entrycount_gfxoff(adev, value); 899 900 mutex_unlock(&adev->gfx.gfx_off_mutex); 901 902 return r; 903 } 904 905 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value) 906 { 907 908 int r = 0; 909 910 mutex_lock(&adev->gfx.gfx_off_mutex); 911 912 r = amdgpu_dpm_get_status_gfxoff(adev, value); 913 914 mutex_unlock(&adev->gfx.gfx_off_mutex); 915 916 return r; 917 } 918 919 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 920 { 921 int r; 922 923 if (amdgpu_ras_is_supported(adev, ras_block->block)) { 924 if (!amdgpu_persistent_edc_harvesting_supported(adev)) { 925 r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX); 926 if (r) 927 return r; 928 } 929 930 r = amdgpu_ras_block_late_init(adev, ras_block); 931 if (r) 932 return r; 933 934 if (amdgpu_sriov_vf(adev)) 935 return r; 936 937 if (adev->gfx.cp_ecc_error_irq.funcs) { 938 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 939 if (r) 940 goto late_fini; 941 } 942 } else { 943 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 944 } 945 946 return 0; 947 late_fini: 948 amdgpu_ras_block_late_fini(adev, ras_block); 949 return r; 950 } 951 952 int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) 953 { 954 int err = 0; 955 struct amdgpu_gfx_ras *ras = NULL; 956 957 /* adev->gfx.ras is NULL, which means gfx does not 958 * support ras function, then do nothing here. 959 */ 960 if (!adev->gfx.ras) 961 return 0; 962 963 ras = adev->gfx.ras; 964 965 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); 966 if (err) { 967 dev_err(adev->dev, "Failed to register gfx ras block!\n"); 968 return err; 969 } 970 971 strcpy(ras->ras_block.ras_comm.name, "gfx"); 972 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX; 973 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 974 adev->gfx.ras_if = &ras->ras_block.ras_comm; 975 976 /* If not define special ras_late_init function, use gfx default ras_late_init */ 977 if (!ras->ras_block.ras_late_init) 978 ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init; 979 980 /* If not defined special ras_cb function, use default ras_cb */ 981 if (!ras->ras_block.ras_cb) 982 ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb; 983 984 return 0; 985 } 986 987 int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev, 988 struct amdgpu_iv_entry *entry) 989 { 990 if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler) 991 return adev->gfx.ras->poison_consumption_handler(adev, entry); 992 993 return 0; 994 } 995 996 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, 997 void *err_data, 998 struct amdgpu_iv_entry *entry) 999 { 1000 /* TODO ue will trigger an interrupt. 1001 * 1002 * When “Full RAS” is enabled, the per-IP interrupt sources should 1003 * be disabled and the driver should only look for the aggregated 1004 * interrupt via sync flood 1005 */ 1006 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) { 1007 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 1008 if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops && 1009 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count) 1010 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1011 amdgpu_ras_reset_gpu(adev); 1012 } 1013 return AMDGPU_RAS_SUCCESS; 1014 } 1015 1016 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, 1017 struct amdgpu_irq_src *source, 1018 struct amdgpu_iv_entry *entry) 1019 { 1020 struct ras_common_if *ras_if = adev->gfx.ras_if; 1021 struct ras_dispatch_if ih_data = { 1022 .entry = entry, 1023 }; 1024 1025 if (!ras_if) 1026 return 0; 1027 1028 ih_data.head = *ras_if; 1029 1030 DRM_ERROR("CP ECC ERROR IRQ\n"); 1031 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 1032 return 0; 1033 } 1034 1035 void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev, 1036 void *ras_error_status, 1037 void (*func)(struct amdgpu_device *adev, void *ras_error_status, 1038 int xcc_id)) 1039 { 1040 int i; 1041 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 1042 uint32_t xcc_mask = GENMASK(num_xcc - 1, 0); 1043 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 1044 1045 if (err_data) { 1046 err_data->ue_count = 0; 1047 err_data->ce_count = 0; 1048 } 1049 1050 for_each_inst(i, xcc_mask) 1051 func(adev, ras_error_status, i); 1052 } 1053 1054 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id) 1055 { 1056 signed long r, cnt = 0; 1057 unsigned long flags; 1058 uint32_t seq, reg_val_offs = 0, value = 0; 1059 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 1060 struct amdgpu_ring *ring = &kiq->ring; 1061 1062 if (amdgpu_device_skip_hw_access(adev)) 1063 return 0; 1064 1065 if (adev->mes.ring[0].sched.ready) 1066 return amdgpu_mes_rreg(adev, reg); 1067 1068 BUG_ON(!ring->funcs->emit_rreg); 1069 1070 spin_lock_irqsave(&kiq->ring_lock, flags); 1071 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 1072 pr_err("critical bug! too many kiq readers\n"); 1073 goto failed_unlock; 1074 } 1075 r = amdgpu_ring_alloc(ring, 32); 1076 if (r) 1077 goto failed_unlock; 1078 1079 amdgpu_ring_emit_rreg(ring, reg, reg_val_offs); 1080 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 1081 if (r) 1082 goto failed_undo; 1083 1084 amdgpu_ring_commit(ring); 1085 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1086 1087 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1088 1089 /* don't wait anymore for gpu reset case because this way may 1090 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 1091 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 1092 * never return if we keep waiting in virt_kiq_rreg, which cause 1093 * gpu_recover() hang there. 1094 * 1095 * also don't wait anymore for IRQ context 1096 * */ 1097 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 1098 goto failed_kiq_read; 1099 1100 might_sleep(); 1101 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 1102 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 1103 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1104 } 1105 1106 if (cnt > MAX_KIQ_REG_TRY) 1107 goto failed_kiq_read; 1108 1109 mb(); 1110 value = adev->wb.wb[reg_val_offs]; 1111 amdgpu_device_wb_free(adev, reg_val_offs); 1112 return value; 1113 1114 failed_undo: 1115 amdgpu_ring_undo(ring); 1116 failed_unlock: 1117 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1118 failed_kiq_read: 1119 if (reg_val_offs) 1120 amdgpu_device_wb_free(adev, reg_val_offs); 1121 dev_err(adev->dev, "failed to read reg:%x\n", reg); 1122 return ~0; 1123 } 1124 1125 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id) 1126 { 1127 signed long r, cnt = 0; 1128 unsigned long flags; 1129 uint32_t seq; 1130 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 1131 struct amdgpu_ring *ring = &kiq->ring; 1132 1133 BUG_ON(!ring->funcs->emit_wreg); 1134 1135 if (amdgpu_device_skip_hw_access(adev)) 1136 return; 1137 1138 if (adev->mes.ring[0].sched.ready) { 1139 amdgpu_mes_wreg(adev, reg, v); 1140 return; 1141 } 1142 1143 spin_lock_irqsave(&kiq->ring_lock, flags); 1144 r = amdgpu_ring_alloc(ring, 32); 1145 if (r) 1146 goto failed_unlock; 1147 1148 amdgpu_ring_emit_wreg(ring, reg, v); 1149 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 1150 if (r) 1151 goto failed_undo; 1152 1153 amdgpu_ring_commit(ring); 1154 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1155 1156 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1157 1158 /* don't wait anymore for gpu reset case because this way may 1159 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 1160 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 1161 * never return if we keep waiting in virt_kiq_rreg, which cause 1162 * gpu_recover() hang there. 1163 * 1164 * also don't wait anymore for IRQ context 1165 * */ 1166 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 1167 goto failed_kiq_write; 1168 1169 might_sleep(); 1170 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 1171 1172 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 1173 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1174 } 1175 1176 if (cnt > MAX_KIQ_REG_TRY) 1177 goto failed_kiq_write; 1178 1179 return; 1180 1181 failed_undo: 1182 amdgpu_ring_undo(ring); 1183 failed_unlock: 1184 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1185 failed_kiq_write: 1186 dev_err(adev->dev, "failed to write reg:%x\n", reg); 1187 } 1188 1189 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev) 1190 { 1191 if (amdgpu_num_kcq == -1) { 1192 return 8; 1193 } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) { 1194 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n"); 1195 return 8; 1196 } 1197 return amdgpu_num_kcq; 1198 } 1199 1200 void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, 1201 uint32_t ucode_id) 1202 { 1203 const struct gfx_firmware_header_v1_0 *cp_hdr; 1204 const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0; 1205 struct amdgpu_firmware_info *info = NULL; 1206 const struct firmware *ucode_fw; 1207 unsigned int fw_size; 1208 1209 switch (ucode_id) { 1210 case AMDGPU_UCODE_ID_CP_PFP: 1211 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1212 adev->gfx.pfp_fw->data; 1213 adev->gfx.pfp_fw_version = 1214 le32_to_cpu(cp_hdr->header.ucode_version); 1215 adev->gfx.pfp_feature_version = 1216 le32_to_cpu(cp_hdr->ucode_feature_version); 1217 ucode_fw = adev->gfx.pfp_fw; 1218 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1219 break; 1220 case AMDGPU_UCODE_ID_CP_RS64_PFP: 1221 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1222 adev->gfx.pfp_fw->data; 1223 adev->gfx.pfp_fw_version = 1224 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1225 adev->gfx.pfp_feature_version = 1226 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1227 ucode_fw = adev->gfx.pfp_fw; 1228 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1229 break; 1230 case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK: 1231 case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK: 1232 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1233 adev->gfx.pfp_fw->data; 1234 ucode_fw = adev->gfx.pfp_fw; 1235 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1236 break; 1237 case AMDGPU_UCODE_ID_CP_ME: 1238 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1239 adev->gfx.me_fw->data; 1240 adev->gfx.me_fw_version = 1241 le32_to_cpu(cp_hdr->header.ucode_version); 1242 adev->gfx.me_feature_version = 1243 le32_to_cpu(cp_hdr->ucode_feature_version); 1244 ucode_fw = adev->gfx.me_fw; 1245 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1246 break; 1247 case AMDGPU_UCODE_ID_CP_RS64_ME: 1248 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1249 adev->gfx.me_fw->data; 1250 adev->gfx.me_fw_version = 1251 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1252 adev->gfx.me_feature_version = 1253 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1254 ucode_fw = adev->gfx.me_fw; 1255 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1256 break; 1257 case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK: 1258 case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK: 1259 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1260 adev->gfx.me_fw->data; 1261 ucode_fw = adev->gfx.me_fw; 1262 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1263 break; 1264 case AMDGPU_UCODE_ID_CP_CE: 1265 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1266 adev->gfx.ce_fw->data; 1267 adev->gfx.ce_fw_version = 1268 le32_to_cpu(cp_hdr->header.ucode_version); 1269 adev->gfx.ce_feature_version = 1270 le32_to_cpu(cp_hdr->ucode_feature_version); 1271 ucode_fw = adev->gfx.ce_fw; 1272 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1273 break; 1274 case AMDGPU_UCODE_ID_CP_MEC1: 1275 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1276 adev->gfx.mec_fw->data; 1277 adev->gfx.mec_fw_version = 1278 le32_to_cpu(cp_hdr->header.ucode_version); 1279 adev->gfx.mec_feature_version = 1280 le32_to_cpu(cp_hdr->ucode_feature_version); 1281 ucode_fw = adev->gfx.mec_fw; 1282 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1283 le32_to_cpu(cp_hdr->jt_size) * 4; 1284 break; 1285 case AMDGPU_UCODE_ID_CP_MEC1_JT: 1286 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1287 adev->gfx.mec_fw->data; 1288 ucode_fw = adev->gfx.mec_fw; 1289 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; 1290 break; 1291 case AMDGPU_UCODE_ID_CP_MEC2: 1292 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1293 adev->gfx.mec2_fw->data; 1294 adev->gfx.mec2_fw_version = 1295 le32_to_cpu(cp_hdr->header.ucode_version); 1296 adev->gfx.mec2_feature_version = 1297 le32_to_cpu(cp_hdr->ucode_feature_version); 1298 ucode_fw = adev->gfx.mec2_fw; 1299 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1300 le32_to_cpu(cp_hdr->jt_size) * 4; 1301 break; 1302 case AMDGPU_UCODE_ID_CP_MEC2_JT: 1303 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1304 adev->gfx.mec2_fw->data; 1305 ucode_fw = adev->gfx.mec2_fw; 1306 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; 1307 break; 1308 case AMDGPU_UCODE_ID_CP_RS64_MEC: 1309 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1310 adev->gfx.mec_fw->data; 1311 adev->gfx.mec_fw_version = 1312 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1313 adev->gfx.mec_feature_version = 1314 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1315 ucode_fw = adev->gfx.mec_fw; 1316 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1317 break; 1318 case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK: 1319 case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK: 1320 case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK: 1321 case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK: 1322 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1323 adev->gfx.mec_fw->data; 1324 ucode_fw = adev->gfx.mec_fw; 1325 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1326 break; 1327 default: 1328 dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id); 1329 return; 1330 } 1331 1332 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1333 info = &adev->firmware.ucode[ucode_id]; 1334 info->ucode_id = ucode_id; 1335 info->fw = ucode_fw; 1336 adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE); 1337 } 1338 } 1339 1340 bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id) 1341 { 1342 return !(xcc_id % (adev->gfx.num_xcc_per_xcp ? 1343 adev->gfx.num_xcc_per_xcp : 1)); 1344 } 1345 1346 static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev, 1347 struct device_attribute *addr, 1348 char *buf) 1349 { 1350 struct drm_device *ddev = dev_get_drvdata(dev); 1351 struct amdgpu_device *adev = drm_to_adev(ddev); 1352 int mode; 1353 1354 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr, 1355 AMDGPU_XCP_FL_NONE); 1356 1357 return sysfs_emit(buf, "%s\n", amdgpu_gfx_compute_mode_desc(mode)); 1358 } 1359 1360 static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev, 1361 struct device_attribute *addr, 1362 const char *buf, size_t count) 1363 { 1364 struct drm_device *ddev = dev_get_drvdata(dev); 1365 struct amdgpu_device *adev = drm_to_adev(ddev); 1366 enum amdgpu_gfx_partition mode; 1367 int ret = 0, num_xcc; 1368 1369 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1370 if (num_xcc % 2 != 0) 1371 return -EINVAL; 1372 1373 if (!strncasecmp("SPX", buf, strlen("SPX"))) { 1374 mode = AMDGPU_SPX_PARTITION_MODE; 1375 } else if (!strncasecmp("DPX", buf, strlen("DPX"))) { 1376 /* 1377 * DPX mode needs AIDs to be in multiple of 2. 1378 * Each AID connects 2 XCCs. 1379 */ 1380 if (num_xcc%4) 1381 return -EINVAL; 1382 mode = AMDGPU_DPX_PARTITION_MODE; 1383 } else if (!strncasecmp("TPX", buf, strlen("TPX"))) { 1384 if (num_xcc != 6) 1385 return -EINVAL; 1386 mode = AMDGPU_TPX_PARTITION_MODE; 1387 } else if (!strncasecmp("QPX", buf, strlen("QPX"))) { 1388 if (num_xcc != 8) 1389 return -EINVAL; 1390 mode = AMDGPU_QPX_PARTITION_MODE; 1391 } else if (!strncasecmp("CPX", buf, strlen("CPX"))) { 1392 mode = AMDGPU_CPX_PARTITION_MODE; 1393 } else { 1394 return -EINVAL; 1395 } 1396 1397 ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode); 1398 1399 if (ret) 1400 return ret; 1401 1402 return count; 1403 } 1404 1405 static const char *xcp_desc[] = { 1406 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 1407 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 1408 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 1409 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 1410 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 1411 }; 1412 1413 static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev, 1414 struct device_attribute *addr, 1415 char *buf) 1416 { 1417 struct drm_device *ddev = dev_get_drvdata(dev); 1418 struct amdgpu_device *adev = drm_to_adev(ddev); 1419 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1420 int size = 0, mode; 1421 char *sep = ""; 1422 1423 if (!xcp_mgr || !xcp_mgr->avail_xcp_modes) 1424 return sysfs_emit(buf, "Not supported\n"); 1425 1426 for_each_inst(mode, xcp_mgr->avail_xcp_modes) { 1427 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 1428 sep = ", "; 1429 } 1430 1431 size += sysfs_emit_at(buf, size, "\n"); 1432 1433 return size; 1434 } 1435 1436 static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) 1437 { 1438 struct amdgpu_device *adev = ring->adev; 1439 struct drm_gpu_scheduler *sched = &ring->sched; 1440 struct drm_sched_entity entity; 1441 struct dma_fence *f; 1442 struct amdgpu_job *job; 1443 struct amdgpu_ib *ib; 1444 int i, r; 1445 1446 /* Initialize the scheduler entity */ 1447 r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL, 1448 &sched, 1, NULL); 1449 if (r) { 1450 dev_err(adev->dev, "Failed setting up GFX kernel entity.\n"); 1451 goto err; 1452 } 1453 1454 r = amdgpu_job_alloc_with_ib(ring->adev, &entity, NULL, 1455 64, 0, 1456 &job); 1457 if (r) 1458 goto err; 1459 1460 job->enforce_isolation = true; 1461 1462 ib = &job->ibs[0]; 1463 for (i = 0; i <= ring->funcs->align_mask; ++i) 1464 ib->ptr[i] = ring->funcs->nop; 1465 ib->length_dw = ring->funcs->align_mask + 1; 1466 1467 f = amdgpu_job_submit(job); 1468 1469 r = dma_fence_wait(f, false); 1470 if (r) 1471 goto err; 1472 1473 dma_fence_put(f); 1474 1475 /* Clean up the scheduler entity */ 1476 drm_sched_entity_destroy(&entity); 1477 return 0; 1478 1479 err: 1480 return r; 1481 } 1482 1483 static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id) 1484 { 1485 int num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1486 struct amdgpu_ring *ring; 1487 int num_xcc_to_clear; 1488 int i, r, xcc_id; 1489 1490 if (adev->gfx.num_xcc_per_xcp) 1491 num_xcc_to_clear = adev->gfx.num_xcc_per_xcp; 1492 else 1493 num_xcc_to_clear = 1; 1494 1495 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1496 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 1497 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings]; 1498 if ((ring->xcp_id == xcp_id) && ring->sched.ready) { 1499 r = amdgpu_gfx_run_cleaner_shader_job(ring); 1500 if (r) 1501 return r; 1502 num_xcc_to_clear--; 1503 break; 1504 } 1505 } 1506 } 1507 1508 if (num_xcc_to_clear) 1509 return -ENOENT; 1510 1511 return 0; 1512 } 1513 1514 /** 1515 * amdgpu_gfx_set_run_cleaner_shader - Execute the AMDGPU GFX Cleaner Shader 1516 * @dev: The device structure 1517 * @attr: The device attribute structure 1518 * @buf: The buffer containing the input data 1519 * @count: The size of the input data 1520 * 1521 * Provides the sysfs interface to manually run a cleaner shader, which is 1522 * used to clear the GPU state between different tasks. Writing a value to the 1523 * 'run_cleaner_shader' sysfs file triggers the cleaner shader execution. 1524 * The value written corresponds to the partition index on multi-partition 1525 * devices. On single-partition devices, the value should be '0'. 1526 * 1527 * The cleaner shader clears the Local Data Store (LDS) and General Purpose 1528 * Registers (GPRs) to ensure data isolation between GPU workloads. 1529 * 1530 * Return: The number of bytes written to the sysfs file. 1531 */ 1532 static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev, 1533 struct device_attribute *attr, 1534 const char *buf, 1535 size_t count) 1536 { 1537 struct drm_device *ddev = dev_get_drvdata(dev); 1538 struct amdgpu_device *adev = drm_to_adev(ddev); 1539 int ret; 1540 long value; 1541 1542 if (amdgpu_in_reset(adev)) 1543 return -EPERM; 1544 if (adev->in_suspend && !adev->in_runpm) 1545 return -EPERM; 1546 1547 ret = kstrtol(buf, 0, &value); 1548 1549 if (ret) 1550 return -EINVAL; 1551 1552 if (value < 0) 1553 return -EINVAL; 1554 1555 if (adev->xcp_mgr) { 1556 if (value >= adev->xcp_mgr->num_xcps) 1557 return -EINVAL; 1558 } else { 1559 if (value > 1) 1560 return -EINVAL; 1561 } 1562 1563 ret = pm_runtime_get_sync(ddev->dev); 1564 if (ret < 0) { 1565 pm_runtime_put_autosuspend(ddev->dev); 1566 return ret; 1567 } 1568 1569 ret = amdgpu_gfx_run_cleaner_shader(adev, value); 1570 1571 pm_runtime_mark_last_busy(ddev->dev); 1572 pm_runtime_put_autosuspend(ddev->dev); 1573 1574 if (ret) 1575 return ret; 1576 1577 return count; 1578 } 1579 1580 /** 1581 * amdgpu_gfx_get_enforce_isolation - Query AMDGPU GFX Enforce Isolation Settings 1582 * @dev: The device structure 1583 * @attr: The device attribute structure 1584 * @buf: The buffer to store the output data 1585 * 1586 * Provides the sysfs read interface to get the current settings of the 'enforce_isolation' 1587 * feature for each GPU partition. Reading from the 'enforce_isolation' 1588 * sysfs file returns the isolation settings for all partitions, where '0' 1589 * indicates disabled and '1' indicates enabled. 1590 * 1591 * Return: The number of bytes read from the sysfs file. 1592 */ 1593 static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev, 1594 struct device_attribute *attr, 1595 char *buf) 1596 { 1597 struct drm_device *ddev = dev_get_drvdata(dev); 1598 struct amdgpu_device *adev = drm_to_adev(ddev); 1599 int i; 1600 ssize_t size = 0; 1601 1602 if (adev->xcp_mgr) { 1603 for (i = 0; i < adev->xcp_mgr->num_xcps; i++) { 1604 size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]); 1605 if (i < (adev->xcp_mgr->num_xcps - 1)) 1606 size += sysfs_emit_at(buf, size, " "); 1607 } 1608 buf[size++] = '\n'; 1609 } else { 1610 size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]); 1611 } 1612 1613 return size; 1614 } 1615 1616 /** 1617 * amdgpu_gfx_set_enforce_isolation - Control AMDGPU GFX Enforce Isolation 1618 * @dev: The device structure 1619 * @attr: The device attribute structure 1620 * @buf: The buffer containing the input data 1621 * @count: The size of the input data 1622 * 1623 * This function allows control over the 'enforce_isolation' feature, which 1624 * serializes access to the graphics engine. Writing '1' or '0' to the 1625 * 'enforce_isolation' sysfs file enables or disables process isolation for 1626 * each partition. The input should specify the setting for all partitions. 1627 * 1628 * Return: The number of bytes written to the sysfs file. 1629 */ 1630 static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev, 1631 struct device_attribute *attr, 1632 const char *buf, size_t count) 1633 { 1634 struct drm_device *ddev = dev_get_drvdata(dev); 1635 struct amdgpu_device *adev = drm_to_adev(ddev); 1636 long partition_values[MAX_XCP] = {0}; 1637 int ret, i, num_partitions; 1638 const char *input_buf = buf; 1639 1640 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { 1641 ret = sscanf(input_buf, "%ld", &partition_values[i]); 1642 if (ret <= 0) 1643 break; 1644 1645 /* Move the pointer to the next value in the string */ 1646 input_buf = strchr(input_buf, ' '); 1647 if (input_buf) { 1648 input_buf++; 1649 } else { 1650 i++; 1651 break; 1652 } 1653 } 1654 num_partitions = i; 1655 1656 if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps) 1657 return -EINVAL; 1658 1659 if (!adev->xcp_mgr && num_partitions != 1) 1660 return -EINVAL; 1661 1662 for (i = 0; i < num_partitions; i++) { 1663 if (partition_values[i] != 0 && partition_values[i] != 1) 1664 return -EINVAL; 1665 } 1666 1667 mutex_lock(&adev->enforce_isolation_mutex); 1668 for (i = 0; i < num_partitions; i++) 1669 adev->enforce_isolation[i] = partition_values[i]; 1670 mutex_unlock(&adev->enforce_isolation_mutex); 1671 1672 amdgpu_mes_update_enforce_isolation(adev); 1673 1674 return count; 1675 } 1676 1677 static ssize_t amdgpu_gfx_get_gfx_reset_mask(struct device *dev, 1678 struct device_attribute *attr, 1679 char *buf) 1680 { 1681 struct drm_device *ddev = dev_get_drvdata(dev); 1682 struct amdgpu_device *adev = drm_to_adev(ddev); 1683 1684 if (!adev) 1685 return -ENODEV; 1686 1687 return amdgpu_show_reset_mask(buf, adev->gfx.gfx_supported_reset); 1688 } 1689 1690 static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev, 1691 struct device_attribute *attr, 1692 char *buf) 1693 { 1694 struct drm_device *ddev = dev_get_drvdata(dev); 1695 struct amdgpu_device *adev = drm_to_adev(ddev); 1696 1697 if (!adev) 1698 return -ENODEV; 1699 1700 return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset); 1701 } 1702 1703 static DEVICE_ATTR(run_cleaner_shader, 0200, 1704 NULL, amdgpu_gfx_set_run_cleaner_shader); 1705 1706 static DEVICE_ATTR(enforce_isolation, 0644, 1707 amdgpu_gfx_get_enforce_isolation, 1708 amdgpu_gfx_set_enforce_isolation); 1709 1710 static DEVICE_ATTR(current_compute_partition, 0644, 1711 amdgpu_gfx_get_current_compute_partition, 1712 amdgpu_gfx_set_compute_partition); 1713 1714 static DEVICE_ATTR(available_compute_partition, 0444, 1715 amdgpu_gfx_get_available_compute_partition, NULL); 1716 static DEVICE_ATTR(gfx_reset_mask, 0444, 1717 amdgpu_gfx_get_gfx_reset_mask, NULL); 1718 1719 static DEVICE_ATTR(compute_reset_mask, 0444, 1720 amdgpu_gfx_get_compute_reset_mask, NULL); 1721 1722 static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev) 1723 { 1724 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1725 bool xcp_switch_supported; 1726 int r; 1727 1728 if (!xcp_mgr) 1729 return 0; 1730 1731 xcp_switch_supported = 1732 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode); 1733 1734 if (!xcp_switch_supported) 1735 dev_attr_current_compute_partition.attr.mode &= 1736 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1737 1738 r = device_create_file(adev->dev, &dev_attr_current_compute_partition); 1739 if (r) 1740 return r; 1741 1742 if (xcp_switch_supported) 1743 r = device_create_file(adev->dev, 1744 &dev_attr_available_compute_partition); 1745 1746 return r; 1747 } 1748 1749 static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev) 1750 { 1751 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1752 bool xcp_switch_supported; 1753 1754 if (!xcp_mgr) 1755 return; 1756 1757 xcp_switch_supported = 1758 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode); 1759 device_remove_file(adev->dev, &dev_attr_current_compute_partition); 1760 1761 if (xcp_switch_supported) 1762 device_remove_file(adev->dev, 1763 &dev_attr_available_compute_partition); 1764 } 1765 1766 static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev) 1767 { 1768 int r; 1769 1770 r = device_create_file(adev->dev, &dev_attr_enforce_isolation); 1771 if (r) 1772 return r; 1773 if (adev->gfx.enable_cleaner_shader) 1774 r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader); 1775 1776 return r; 1777 } 1778 1779 static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev) 1780 { 1781 device_remove_file(adev->dev, &dev_attr_enforce_isolation); 1782 if (adev->gfx.enable_cleaner_shader) 1783 device_remove_file(adev->dev, &dev_attr_run_cleaner_shader); 1784 } 1785 1786 static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev) 1787 { 1788 int r = 0; 1789 1790 if (!amdgpu_gpu_recovery) 1791 return r; 1792 1793 if (adev->gfx.num_gfx_rings) { 1794 r = device_create_file(adev->dev, &dev_attr_gfx_reset_mask); 1795 if (r) 1796 return r; 1797 } 1798 1799 if (adev->gfx.num_compute_rings) { 1800 r = device_create_file(adev->dev, &dev_attr_compute_reset_mask); 1801 if (r) 1802 return r; 1803 } 1804 1805 return r; 1806 } 1807 1808 static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev) 1809 { 1810 if (!amdgpu_gpu_recovery) 1811 return; 1812 1813 if (adev->gfx.num_gfx_rings) 1814 device_remove_file(adev->dev, &dev_attr_gfx_reset_mask); 1815 1816 if (adev->gfx.num_compute_rings) 1817 device_remove_file(adev->dev, &dev_attr_compute_reset_mask); 1818 } 1819 1820 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev) 1821 { 1822 int r; 1823 1824 r = amdgpu_gfx_sysfs_xcp_init(adev); 1825 if (r) { 1826 dev_err(adev->dev, "failed to create xcp sysfs files"); 1827 return r; 1828 } 1829 1830 r = amdgpu_gfx_sysfs_isolation_shader_init(adev); 1831 if (r) 1832 dev_err(adev->dev, "failed to create isolation sysfs files"); 1833 1834 r = amdgpu_gfx_sysfs_reset_mask_init(adev); 1835 if (r) 1836 dev_err(adev->dev, "failed to create reset mask sysfs files"); 1837 1838 return r; 1839 } 1840 1841 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev) 1842 { 1843 if (adev->dev->kobj.sd) { 1844 amdgpu_gfx_sysfs_xcp_fini(adev); 1845 amdgpu_gfx_sysfs_isolation_shader_fini(adev); 1846 amdgpu_gfx_sysfs_reset_mask_fini(adev); 1847 } 1848 } 1849 1850 int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev, 1851 unsigned int cleaner_shader_size) 1852 { 1853 if (!adev->gfx.enable_cleaner_shader) 1854 return -EOPNOTSUPP; 1855 1856 return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE, 1857 AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT, 1858 &adev->gfx.cleaner_shader_obj, 1859 &adev->gfx.cleaner_shader_gpu_addr, 1860 (void **)&adev->gfx.cleaner_shader_cpu_ptr); 1861 } 1862 1863 void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev) 1864 { 1865 if (!adev->gfx.enable_cleaner_shader) 1866 return; 1867 1868 amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj, 1869 &adev->gfx.cleaner_shader_gpu_addr, 1870 (void **)&adev->gfx.cleaner_shader_cpu_ptr); 1871 } 1872 1873 void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev, 1874 unsigned int cleaner_shader_size, 1875 const void *cleaner_shader_ptr) 1876 { 1877 if (!adev->gfx.enable_cleaner_shader) 1878 return; 1879 1880 if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr) 1881 memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr, 1882 cleaner_shader_size); 1883 } 1884 1885 /** 1886 * amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver) 1887 * @adev: amdgpu_device pointer 1888 * @idx: Index of the scheduler to control 1889 * @enable: Whether to enable or disable the KFD scheduler 1890 * 1891 * This function is used to control the KFD (Kernel Fusion Driver) scheduler 1892 * from the KGD. It is part of the cleaner shader feature. This function plays 1893 * a key role in enforcing process isolation on the GPU. 1894 * 1895 * The function uses a reference count mechanism (kfd_sch_req_count) to keep 1896 * track of the number of requests to enable the KFD scheduler. When a request 1897 * to enable the KFD scheduler is made, the reference count is decremented. 1898 * When the reference count reaches zero, a delayed work is scheduled to 1899 * enforce isolation after a delay of GFX_SLICE_PERIOD. 1900 * 1901 * When a request to disable the KFD scheduler is made, the function first 1902 * checks if the reference count is zero. If it is, it cancels the delayed work 1903 * for enforcing isolation and checks if the KFD scheduler is active. If the 1904 * KFD scheduler is active, it sends a request to stop the KFD scheduler and 1905 * sets the KFD scheduler state to inactive. Then, it increments the reference 1906 * count. 1907 * 1908 * The function is synchronized using the kfd_sch_mutex to ensure that the KFD 1909 * scheduler state and reference count are updated atomically. 1910 * 1911 * Note: If the reference count is already zero when a request to enable the 1912 * KFD scheduler is made, it means there's an imbalance bug somewhere. The 1913 * function triggers a warning in this case. 1914 */ 1915 static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx, 1916 bool enable) 1917 { 1918 mutex_lock(&adev->gfx.kfd_sch_mutex); 1919 1920 if (enable) { 1921 /* If the count is already 0, it means there's an imbalance bug somewhere. 1922 * Note that the bug may be in a different caller than the one which triggers the 1923 * WARN_ON_ONCE. 1924 */ 1925 if (WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx] == 0)) { 1926 dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n"); 1927 goto unlock; 1928 } 1929 1930 adev->gfx.kfd_sch_req_count[idx]--; 1931 1932 if (adev->gfx.kfd_sch_req_count[idx] == 0 && 1933 adev->gfx.kfd_sch_inactive[idx]) { 1934 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, 1935 msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx])); 1936 } 1937 } else { 1938 if (adev->gfx.kfd_sch_req_count[idx] == 0) { 1939 cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work); 1940 if (!adev->gfx.kfd_sch_inactive[idx]) { 1941 amdgpu_amdkfd_stop_sched(adev, idx); 1942 adev->gfx.kfd_sch_inactive[idx] = true; 1943 } 1944 } 1945 1946 adev->gfx.kfd_sch_req_count[idx]++; 1947 } 1948 1949 unlock: 1950 mutex_unlock(&adev->gfx.kfd_sch_mutex); 1951 } 1952 1953 /** 1954 * amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation 1955 * 1956 * @work: work_struct. 1957 * 1958 * This function is the work handler for enforcing shader isolation on AMD GPUs. 1959 * It counts the number of emitted fences for each GFX and compute ring. If there 1960 * are any fences, it schedules the `enforce_isolation_work` to be run after a 1961 * delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion 1962 * Driver (KFD) to resume the runqueue. The function is synchronized using the 1963 * `enforce_isolation_mutex`. 1964 */ 1965 void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work) 1966 { 1967 struct amdgpu_isolation_work *isolation_work = 1968 container_of(work, struct amdgpu_isolation_work, work.work); 1969 struct amdgpu_device *adev = isolation_work->adev; 1970 u32 i, idx, fences = 0; 1971 1972 if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION) 1973 idx = 0; 1974 else 1975 idx = isolation_work->xcp_id; 1976 1977 if (idx >= MAX_XCP) 1978 return; 1979 1980 mutex_lock(&adev->enforce_isolation_mutex); 1981 for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) { 1982 if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id) 1983 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]); 1984 } 1985 for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) { 1986 if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id) 1987 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]); 1988 } 1989 if (fences) { 1990 /* we've already had our timeslice, so let's wrap this up */ 1991 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, 1992 msecs_to_jiffies(1)); 1993 } else { 1994 /* Tell KFD to resume the runqueue */ 1995 if (adev->kfd.init_complete) { 1996 WARN_ON_ONCE(!adev->gfx.kfd_sch_inactive[idx]); 1997 WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx]); 1998 amdgpu_amdkfd_start_sched(adev, idx); 1999 adev->gfx.kfd_sch_inactive[idx] = false; 2000 } 2001 } 2002 mutex_unlock(&adev->enforce_isolation_mutex); 2003 } 2004 2005 /** 2006 * amdgpu_gfx_enforce_isolation_wait_for_kfd - Manage KFD wait period for process isolation 2007 * @adev: amdgpu_device pointer 2008 * @idx: Index of the GPU partition 2009 * 2010 * When kernel submissions come in, the jobs are given a time slice and once 2011 * that time slice is up, if there are KFD user queues active, kernel 2012 * submissions are blocked until KFD has had its time slice. Once the KFD time 2013 * slice is up, KFD user queues are preempted and kernel submissions are 2014 * unblocked and allowed to run again. 2015 */ 2016 static void 2017 amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev, 2018 u32 idx) 2019 { 2020 unsigned long cjiffies; 2021 bool wait = false; 2022 2023 mutex_lock(&adev->enforce_isolation_mutex); 2024 if (adev->enforce_isolation[idx]) { 2025 /* set the initial values if nothing is set */ 2026 if (!adev->gfx.enforce_isolation_jiffies[idx]) { 2027 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 2028 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 2029 } 2030 /* Make sure KFD gets a chance to run */ 2031 if (amdgpu_amdkfd_compute_active(adev, idx)) { 2032 cjiffies = jiffies; 2033 if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) { 2034 cjiffies -= adev->gfx.enforce_isolation_jiffies[idx]; 2035 if ((jiffies_to_msecs(cjiffies) >= GFX_SLICE_PERIOD_MS)) { 2036 /* if our time is up, let KGD work drain before scheduling more */ 2037 wait = true; 2038 /* reset the timer period */ 2039 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 2040 } else { 2041 /* set the timer period to what's left in our time slice */ 2042 adev->gfx.enforce_isolation_time[idx] = 2043 GFX_SLICE_PERIOD_MS - jiffies_to_msecs(cjiffies); 2044 } 2045 } else { 2046 /* if jiffies wrap around we will just wait a little longer */ 2047 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 2048 } 2049 } else { 2050 /* if there is no KFD work, then set the full slice period */ 2051 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 2052 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 2053 } 2054 } 2055 mutex_unlock(&adev->enforce_isolation_mutex); 2056 2057 if (wait) 2058 msleep(GFX_SLICE_PERIOD_MS); 2059 } 2060 2061 /** 2062 * amdgpu_gfx_enforce_isolation_ring_begin_use - Begin use of a ring with enforced isolation 2063 * @ring: Pointer to the amdgpu_ring structure 2064 * 2065 * Ring begin_use helper implementation for gfx which serializes access to the 2066 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation 2067 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues 2068 * each get a time slice when both are active. 2069 */ 2070 void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring) 2071 { 2072 struct amdgpu_device *adev = ring->adev; 2073 u32 idx; 2074 bool sched_work = false; 2075 2076 if (!adev->gfx.enable_cleaner_shader) 2077 return; 2078 2079 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION) 2080 idx = 0; 2081 else 2082 idx = ring->xcp_id; 2083 2084 if (idx >= MAX_XCP) 2085 return; 2086 2087 /* Don't submit more work until KFD has had some time */ 2088 amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx); 2089 2090 mutex_lock(&adev->enforce_isolation_mutex); 2091 if (adev->enforce_isolation[idx]) { 2092 if (adev->kfd.init_complete) 2093 sched_work = true; 2094 } 2095 mutex_unlock(&adev->enforce_isolation_mutex); 2096 2097 if (sched_work) 2098 amdgpu_gfx_kfd_sch_ctrl(adev, idx, false); 2099 } 2100 2101 /** 2102 * amdgpu_gfx_enforce_isolation_ring_end_use - End use of a ring with enforced isolation 2103 * @ring: Pointer to the amdgpu_ring structure 2104 * 2105 * Ring end_use helper implementation for gfx which serializes access to the 2106 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation 2107 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues 2108 * each get a time slice when both are active. 2109 */ 2110 void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring) 2111 { 2112 struct amdgpu_device *adev = ring->adev; 2113 u32 idx; 2114 bool sched_work = false; 2115 2116 if (!adev->gfx.enable_cleaner_shader) 2117 return; 2118 2119 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION) 2120 idx = 0; 2121 else 2122 idx = ring->xcp_id; 2123 2124 if (idx >= MAX_XCP) 2125 return; 2126 2127 mutex_lock(&adev->enforce_isolation_mutex); 2128 if (adev->enforce_isolation[idx]) { 2129 if (adev->kfd.init_complete) 2130 sched_work = true; 2131 } 2132 mutex_unlock(&adev->enforce_isolation_mutex); 2133 2134 if (sched_work) 2135 amdgpu_gfx_kfd_sch_ctrl(adev, idx, true); 2136 } 2137 2138 void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work) 2139 { 2140 struct amdgpu_device *adev = 2141 container_of(work, struct amdgpu_device, gfx.idle_work.work); 2142 enum PP_SMC_POWER_PROFILE profile; 2143 u32 i, fences = 0; 2144 int r; 2145 2146 if (adev->gfx.num_gfx_rings) 2147 profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D; 2148 else 2149 profile = PP_SMC_POWER_PROFILE_COMPUTE; 2150 2151 for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) 2152 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]); 2153 for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) 2154 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]); 2155 if (!fences && !atomic_read(&adev->gfx.total_submission_cnt)) { 2156 mutex_lock(&adev->gfx.workload_profile_mutex); 2157 if (adev->gfx.workload_profile_active) { 2158 r = amdgpu_dpm_switch_power_profile(adev, profile, false); 2159 if (r) 2160 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r, 2161 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ? 2162 "fullscreen 3D" : "compute"); 2163 adev->gfx.workload_profile_active = false; 2164 } 2165 mutex_unlock(&adev->gfx.workload_profile_mutex); 2166 } else { 2167 schedule_delayed_work(&adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT); 2168 } 2169 } 2170 2171 void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring) 2172 { 2173 struct amdgpu_device *adev = ring->adev; 2174 enum PP_SMC_POWER_PROFILE profile; 2175 int r; 2176 2177 if (adev->gfx.num_gfx_rings) 2178 profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D; 2179 else 2180 profile = PP_SMC_POWER_PROFILE_COMPUTE; 2181 2182 atomic_inc(&adev->gfx.total_submission_cnt); 2183 2184 cancel_delayed_work_sync(&adev->gfx.idle_work); 2185 2186 /* We can safely return early here because we've cancelled the 2187 * the delayed work so there is no one else to set it to false 2188 * and we don't care if someone else sets it to true. 2189 */ 2190 if (adev->gfx.workload_profile_active) 2191 return; 2192 2193 mutex_lock(&adev->gfx.workload_profile_mutex); 2194 if (!adev->gfx.workload_profile_active) { 2195 r = amdgpu_dpm_switch_power_profile(adev, profile, true); 2196 if (r) 2197 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r, 2198 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ? 2199 "fullscreen 3D" : "compute"); 2200 adev->gfx.workload_profile_active = true; 2201 } 2202 mutex_unlock(&adev->gfx.workload_profile_mutex); 2203 } 2204 2205 void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring) 2206 { 2207 atomic_dec(&ring->adev->gfx.total_submission_cnt); 2208 2209 schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT); 2210 } 2211 2212 /* 2213 * debugfs for to enable/disable gfx job submission to specific core. 2214 */ 2215 #if defined(CONFIG_DEBUG_FS) 2216 static int amdgpu_debugfs_gfx_sched_mask_set(void *data, u64 val) 2217 { 2218 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2219 u32 i; 2220 u64 mask = 0; 2221 struct amdgpu_ring *ring; 2222 2223 if (!adev) 2224 return -ENODEV; 2225 2226 mask = (1ULL << adev->gfx.num_gfx_rings) - 1; 2227 if ((val & mask) == 0) 2228 return -EINVAL; 2229 2230 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) { 2231 ring = &adev->gfx.gfx_ring[i]; 2232 if (val & (1 << i)) 2233 ring->sched.ready = true; 2234 else 2235 ring->sched.ready = false; 2236 } 2237 /* publish sched.ready flag update effective immediately across smp */ 2238 smp_rmb(); 2239 return 0; 2240 } 2241 2242 static int amdgpu_debugfs_gfx_sched_mask_get(void *data, u64 *val) 2243 { 2244 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2245 u32 i; 2246 u64 mask = 0; 2247 struct amdgpu_ring *ring; 2248 2249 if (!adev) 2250 return -ENODEV; 2251 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) { 2252 ring = &adev->gfx.gfx_ring[i]; 2253 if (ring->sched.ready) 2254 mask |= 1ULL << i; 2255 } 2256 2257 *val = mask; 2258 return 0; 2259 } 2260 2261 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gfx_sched_mask_fops, 2262 amdgpu_debugfs_gfx_sched_mask_get, 2263 amdgpu_debugfs_gfx_sched_mask_set, "%llx\n"); 2264 2265 #endif 2266 2267 void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev) 2268 { 2269 #if defined(CONFIG_DEBUG_FS) 2270 struct drm_minor *minor = adev_to_drm(adev)->primary; 2271 struct dentry *root = minor->debugfs_root; 2272 char name[32]; 2273 2274 if (!(adev->gfx.num_gfx_rings > 1)) 2275 return; 2276 sprintf(name, "amdgpu_gfx_sched_mask"); 2277 debugfs_create_file(name, 0600, root, adev, 2278 &amdgpu_debugfs_gfx_sched_mask_fops); 2279 #endif 2280 } 2281 2282 /* 2283 * debugfs for to enable/disable compute job submission to specific core. 2284 */ 2285 #if defined(CONFIG_DEBUG_FS) 2286 static int amdgpu_debugfs_compute_sched_mask_set(void *data, u64 val) 2287 { 2288 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2289 u32 i; 2290 u64 mask = 0; 2291 struct amdgpu_ring *ring; 2292 2293 if (!adev) 2294 return -ENODEV; 2295 2296 mask = (1ULL << adev->gfx.num_compute_rings) - 1; 2297 if ((val & mask) == 0) 2298 return -EINVAL; 2299 2300 for (i = 0; i < adev->gfx.num_compute_rings; ++i) { 2301 ring = &adev->gfx.compute_ring[i]; 2302 if (val & (1 << i)) 2303 ring->sched.ready = true; 2304 else 2305 ring->sched.ready = false; 2306 } 2307 2308 /* publish sched.ready flag update effective immediately across smp */ 2309 smp_rmb(); 2310 return 0; 2311 } 2312 2313 static int amdgpu_debugfs_compute_sched_mask_get(void *data, u64 *val) 2314 { 2315 struct amdgpu_device *adev = (struct amdgpu_device *)data; 2316 u32 i; 2317 u64 mask = 0; 2318 struct amdgpu_ring *ring; 2319 2320 if (!adev) 2321 return -ENODEV; 2322 for (i = 0; i < adev->gfx.num_compute_rings; ++i) { 2323 ring = &adev->gfx.compute_ring[i]; 2324 if (ring->sched.ready) 2325 mask |= 1ULL << i; 2326 } 2327 2328 *val = mask; 2329 return 0; 2330 } 2331 2332 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_compute_sched_mask_fops, 2333 amdgpu_debugfs_compute_sched_mask_get, 2334 amdgpu_debugfs_compute_sched_mask_set, "%llx\n"); 2335 2336 #endif 2337 2338 void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev) 2339 { 2340 #if defined(CONFIG_DEBUG_FS) 2341 struct drm_minor *minor = adev_to_drm(adev)->primary; 2342 struct dentry *root = minor->debugfs_root; 2343 char name[32]; 2344 2345 if (!(adev->gfx.num_compute_rings > 1)) 2346 return; 2347 sprintf(name, "amdgpu_compute_sched_mask"); 2348 debugfs_create_file(name, 0600, root, adev, 2349 &amdgpu_debugfs_compute_sched_mask_fops); 2350 #endif 2351 } 2352