1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 */ 25 26 #include <linux/firmware.h> 27 #include <linux/pm_runtime.h> 28 29 #include "amdgpu.h" 30 #include "amdgpu_gfx.h" 31 #include "amdgpu_rlc.h" 32 #include "amdgpu_ras.h" 33 #include "amdgpu_reset.h" 34 #include "amdgpu_xcp.h" 35 #include "amdgpu_xgmi.h" 36 37 /* delay 0.1 second to enable gfx off feature */ 38 #define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100) 39 40 #define GFX_OFF_NO_DELAY 0 41 42 /* 43 * GPU GFX IP block helpers function. 44 */ 45 46 int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec, 47 int pipe, int queue) 48 { 49 int bit = 0; 50 51 bit += mec * adev->gfx.mec.num_pipe_per_mec 52 * adev->gfx.mec.num_queue_per_pipe; 53 bit += pipe * adev->gfx.mec.num_queue_per_pipe; 54 bit += queue; 55 56 return bit; 57 } 58 59 void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, 60 int *mec, int *pipe, int *queue) 61 { 62 *queue = bit % adev->gfx.mec.num_queue_per_pipe; 63 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe) 64 % adev->gfx.mec.num_pipe_per_mec; 65 *mec = (bit / adev->gfx.mec.num_queue_per_pipe) 66 / adev->gfx.mec.num_pipe_per_mec; 67 68 } 69 70 bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, 71 int xcc_id, int mec, int pipe, int queue) 72 { 73 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue), 74 adev->gfx.mec_bitmap[xcc_id].queue_bitmap); 75 } 76 77 int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, 78 int me, int pipe, int queue) 79 { 80 int bit = 0; 81 82 bit += me * adev->gfx.me.num_pipe_per_me 83 * adev->gfx.me.num_queue_per_pipe; 84 bit += pipe * adev->gfx.me.num_queue_per_pipe; 85 bit += queue; 86 87 return bit; 88 } 89 90 bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, 91 int me, int pipe, int queue) 92 { 93 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue), 94 adev->gfx.me.queue_bitmap); 95 } 96 97 /** 98 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter 99 * 100 * @mask: array in which the per-shader array disable masks will be stored 101 * @max_se: number of SEs 102 * @max_sh: number of SHs 103 * 104 * The bitmask of CUs to be disabled in the shader array determined by se and 105 * sh is stored in mask[se * max_sh + sh]. 106 */ 107 void amdgpu_gfx_parse_disable_cu(unsigned int *mask, unsigned int max_se, unsigned int max_sh) 108 { 109 unsigned int se, sh, cu; 110 const char *p; 111 112 memset(mask, 0, sizeof(*mask) * max_se * max_sh); 113 114 if (!amdgpu_disable_cu || !*amdgpu_disable_cu) 115 return; 116 117 p = amdgpu_disable_cu; 118 for (;;) { 119 char *next; 120 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); 121 122 if (ret < 3) { 123 DRM_ERROR("amdgpu: could not parse disable_cu\n"); 124 return; 125 } 126 127 if (se < max_se && sh < max_sh && cu < 16) { 128 DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); 129 mask[se * max_sh + sh] |= 1u << cu; 130 } else { 131 DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", 132 se, sh, cu); 133 } 134 135 next = strchr(p, ','); 136 if (!next) 137 break; 138 p = next + 1; 139 } 140 } 141 142 static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev) 143 { 144 return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1; 145 } 146 147 static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev) 148 { 149 if (amdgpu_compute_multipipe != -1) { 150 DRM_INFO("amdgpu: forcing compute pipe policy %d\n", 151 amdgpu_compute_multipipe); 152 return amdgpu_compute_multipipe == 1; 153 } 154 155 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 156 return true; 157 158 /* FIXME: spreading the queues across pipes causes perf regressions 159 * on POLARIS11 compute workloads */ 160 if (adev->asic_type == CHIP_POLARIS11) 161 return false; 162 163 return adev->gfx.mec.num_mec > 1; 164 } 165 166 bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev, 167 struct amdgpu_ring *ring) 168 { 169 int queue = ring->queue; 170 int pipe = ring->pipe; 171 172 /* Policy: use pipe1 queue0 as high priority graphics queue if we 173 * have more than one gfx pipe. 174 */ 175 if (amdgpu_gfx_is_graphics_multipipe_capable(adev) && 176 adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) { 177 int me = ring->me; 178 int bit; 179 180 bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue); 181 if (ring == &adev->gfx.gfx_ring[bit]) 182 return true; 183 } 184 185 return false; 186 } 187 188 bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, 189 struct amdgpu_ring *ring) 190 { 191 /* Policy: use 1st queue as high priority compute queue if we 192 * have more than one compute queue. 193 */ 194 if (adev->gfx.num_compute_rings > 1 && 195 ring == &adev->gfx.compute_ring[0]) 196 return true; 197 198 return false; 199 } 200 201 void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) 202 { 203 int i, j, queue, pipe; 204 bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev); 205 int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec * 206 adev->gfx.mec.num_queue_per_pipe, 207 adev->gfx.num_compute_rings); 208 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 209 210 if (multipipe_policy) { 211 /* policy: make queues evenly cross all pipes on MEC1 only 212 * for multiple xcc, just use the original policy for simplicity */ 213 for (j = 0; j < num_xcc; j++) { 214 for (i = 0; i < max_queues_per_mec; i++) { 215 pipe = i % adev->gfx.mec.num_pipe_per_mec; 216 queue = (i / adev->gfx.mec.num_pipe_per_mec) % 217 adev->gfx.mec.num_queue_per_pipe; 218 219 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue, 220 adev->gfx.mec_bitmap[j].queue_bitmap); 221 } 222 } 223 } else { 224 /* policy: amdgpu owns all queues in the given pipe */ 225 for (j = 0; j < num_xcc; j++) { 226 for (i = 0; i < max_queues_per_mec; ++i) 227 set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap); 228 } 229 } 230 231 for (j = 0; j < num_xcc; j++) { 232 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n", 233 bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)); 234 } 235 } 236 237 void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev) 238 { 239 int i, queue, pipe; 240 bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev); 241 int max_queues_per_me = adev->gfx.me.num_pipe_per_me * 242 adev->gfx.me.num_queue_per_pipe; 243 244 if (multipipe_policy) { 245 /* policy: amdgpu owns the first queue per pipe at this stage 246 * will extend to mulitple queues per pipe later */ 247 for (i = 0; i < max_queues_per_me; i++) { 248 pipe = i % adev->gfx.me.num_pipe_per_me; 249 queue = (i / adev->gfx.me.num_pipe_per_me) % 250 adev->gfx.me.num_queue_per_pipe; 251 252 set_bit(pipe * adev->gfx.me.num_queue_per_pipe + queue, 253 adev->gfx.me.queue_bitmap); 254 } 255 } else { 256 for (i = 0; i < max_queues_per_me; ++i) 257 set_bit(i, adev->gfx.me.queue_bitmap); 258 } 259 260 /* update the number of active graphics rings */ 261 adev->gfx.num_gfx_rings = 262 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 263 } 264 265 static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev, 266 struct amdgpu_ring *ring, int xcc_id) 267 { 268 int queue_bit; 269 int mec, pipe, queue; 270 271 queue_bit = adev->gfx.mec.num_mec 272 * adev->gfx.mec.num_pipe_per_mec 273 * adev->gfx.mec.num_queue_per_pipe; 274 275 while (--queue_bit >= 0) { 276 if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) 277 continue; 278 279 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 280 281 /* 282 * 1. Using pipes 2/3 from MEC 2 seems cause problems. 283 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN 284 * only can be issued on queue 0. 285 */ 286 if ((mec == 1 && pipe > 1) || queue != 0) 287 continue; 288 289 ring->me = mec + 1; 290 ring->pipe = pipe; 291 ring->queue = queue; 292 293 return 0; 294 } 295 296 dev_err(adev->dev, "Failed to find a queue for KIQ\n"); 297 return -EINVAL; 298 } 299 300 int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id) 301 { 302 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 303 struct amdgpu_irq_src *irq = &kiq->irq; 304 struct amdgpu_ring *ring = &kiq->ring; 305 int r = 0; 306 307 spin_lock_init(&kiq->ring_lock); 308 309 ring->adev = NULL; 310 ring->ring_obj = NULL; 311 ring->use_doorbell = true; 312 ring->xcc_id = xcc_id; 313 ring->vm_hub = AMDGPU_GFXHUB(xcc_id); 314 ring->doorbell_index = 315 (adev->doorbell_index.kiq + 316 xcc_id * adev->doorbell_index.xcc_doorbell_range) 317 << 1; 318 319 r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id); 320 if (r) 321 return r; 322 323 ring->eop_gpu_addr = kiq->eop_gpu_addr; 324 ring->no_scheduler = true; 325 snprintf(ring->name, sizeof(ring->name), "kiq_%hhu.%hhu.%hhu.%hhu", 326 (unsigned char)xcc_id, (unsigned char)ring->me, 327 (unsigned char)ring->pipe, (unsigned char)ring->queue); 328 r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0, 329 AMDGPU_RING_PRIO_DEFAULT, NULL); 330 if (r) 331 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r); 332 333 return r; 334 } 335 336 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring) 337 { 338 amdgpu_ring_fini(ring); 339 } 340 341 void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id) 342 { 343 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 344 345 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL); 346 } 347 348 int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, 349 unsigned int hpd_size, int xcc_id) 350 { 351 int r; 352 u32 *hpd; 353 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 354 355 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE, 356 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj, 357 &kiq->eop_gpu_addr, (void **)&hpd); 358 if (r) { 359 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r); 360 return r; 361 } 362 363 memset(hpd, 0, hpd_size); 364 365 r = amdgpu_bo_reserve(kiq->eop_obj, true); 366 if (unlikely(r != 0)) 367 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r); 368 amdgpu_bo_kunmap(kiq->eop_obj); 369 amdgpu_bo_unreserve(kiq->eop_obj); 370 371 return 0; 372 } 373 374 /* create MQD for each compute/gfx queue */ 375 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, 376 unsigned int mqd_size, int xcc_id) 377 { 378 int r, i, j; 379 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 380 struct amdgpu_ring *ring = &kiq->ring; 381 u32 domain = AMDGPU_GEM_DOMAIN_GTT; 382 383 #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) 384 /* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */ 385 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0)) 386 domain |= AMDGPU_GEM_DOMAIN_VRAM; 387 #endif 388 389 /* create MQD for KIQ */ 390 if (!adev->enable_mes_kiq && !ring->mqd_obj) { 391 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must 392 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD 393 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for 394 * KIQ MQD no matter SRIOV or Bare-metal 395 */ 396 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 397 AMDGPU_GEM_DOMAIN_VRAM | 398 AMDGPU_GEM_DOMAIN_GTT, 399 &ring->mqd_obj, 400 &ring->mqd_gpu_addr, 401 &ring->mqd_ptr); 402 if (r) { 403 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r); 404 return r; 405 } 406 407 /* prepare MQD backup */ 408 kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL); 409 if (!kiq->mqd_backup) { 410 dev_warn(adev->dev, 411 "no memory to create MQD backup for ring %s\n", ring->name); 412 return -ENOMEM; 413 } 414 } 415 416 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 417 /* create MQD for each KGQ */ 418 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 419 ring = &adev->gfx.gfx_ring[i]; 420 if (!ring->mqd_obj) { 421 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 422 domain, &ring->mqd_obj, 423 &ring->mqd_gpu_addr, &ring->mqd_ptr); 424 if (r) { 425 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 426 return r; 427 } 428 429 ring->mqd_size = mqd_size; 430 /* prepare MQD backup */ 431 adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL); 432 if (!adev->gfx.me.mqd_backup[i]) { 433 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 434 return -ENOMEM; 435 } 436 } 437 } 438 } 439 440 /* create MQD for each KCQ */ 441 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 442 j = i + xcc_id * adev->gfx.num_compute_rings; 443 ring = &adev->gfx.compute_ring[j]; 444 if (!ring->mqd_obj) { 445 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE, 446 domain, &ring->mqd_obj, 447 &ring->mqd_gpu_addr, &ring->mqd_ptr); 448 if (r) { 449 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r); 450 return r; 451 } 452 453 ring->mqd_size = mqd_size; 454 /* prepare MQD backup */ 455 adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL); 456 if (!adev->gfx.mec.mqd_backup[j]) { 457 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name); 458 return -ENOMEM; 459 } 460 } 461 } 462 463 return 0; 464 } 465 466 void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id) 467 { 468 struct amdgpu_ring *ring = NULL; 469 int i, j; 470 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 471 472 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) { 473 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 474 ring = &adev->gfx.gfx_ring[i]; 475 kfree(adev->gfx.me.mqd_backup[i]); 476 amdgpu_bo_free_kernel(&ring->mqd_obj, 477 &ring->mqd_gpu_addr, 478 &ring->mqd_ptr); 479 } 480 } 481 482 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 483 j = i + xcc_id * adev->gfx.num_compute_rings; 484 ring = &adev->gfx.compute_ring[j]; 485 kfree(adev->gfx.mec.mqd_backup[j]); 486 amdgpu_bo_free_kernel(&ring->mqd_obj, 487 &ring->mqd_gpu_addr, 488 &ring->mqd_ptr); 489 } 490 491 ring = &kiq->ring; 492 kfree(kiq->mqd_backup); 493 amdgpu_bo_free_kernel(&ring->mqd_obj, 494 &ring->mqd_gpu_addr, 495 &ring->mqd_ptr); 496 } 497 498 int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id) 499 { 500 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 501 struct amdgpu_ring *kiq_ring = &kiq->ring; 502 int i, r = 0; 503 int j; 504 505 if (adev->enable_mes) { 506 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 507 j = i + xcc_id * adev->gfx.num_compute_rings; 508 amdgpu_mes_unmap_legacy_queue(adev, 509 &adev->gfx.compute_ring[j], 510 RESET_QUEUES, 0, 0); 511 } 512 return 0; 513 } 514 515 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 516 return -EINVAL; 517 518 spin_lock(&kiq->ring_lock); 519 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 520 adev->gfx.num_compute_rings)) { 521 spin_unlock(&kiq->ring_lock); 522 return -ENOMEM; 523 } 524 525 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 526 j = i + xcc_id * adev->gfx.num_compute_rings; 527 kiq->pmf->kiq_unmap_queues(kiq_ring, 528 &adev->gfx.compute_ring[j], 529 RESET_QUEUES, 0, 0); 530 } 531 532 /** 533 * This is workaround: only skip kiq_ring test 534 * during ras recovery in suspend stage for gfx9.4.3 535 */ 536 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 537 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) && 538 amdgpu_ras_in_recovery(adev)) { 539 spin_unlock(&kiq->ring_lock); 540 return 0; 541 } 542 543 if (kiq_ring->sched.ready && !adev->job_hang) 544 r = amdgpu_ring_test_helper(kiq_ring); 545 spin_unlock(&kiq->ring_lock); 546 547 return r; 548 } 549 550 int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id) 551 { 552 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 553 struct amdgpu_ring *kiq_ring = &kiq->ring; 554 int i, r = 0; 555 int j; 556 557 if (adev->enable_mes) { 558 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 559 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 560 j = i + xcc_id * adev->gfx.num_gfx_rings; 561 amdgpu_mes_unmap_legacy_queue(adev, 562 &adev->gfx.gfx_ring[j], 563 PREEMPT_QUEUES, 0, 0); 564 } 565 } 566 return 0; 567 } 568 569 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 570 return -EINVAL; 571 572 spin_lock(&kiq->ring_lock); 573 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 574 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size * 575 adev->gfx.num_gfx_rings)) { 576 spin_unlock(&kiq->ring_lock); 577 return -ENOMEM; 578 } 579 580 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 581 j = i + xcc_id * adev->gfx.num_gfx_rings; 582 kiq->pmf->kiq_unmap_queues(kiq_ring, 583 &adev->gfx.gfx_ring[j], 584 PREEMPT_QUEUES, 0, 0); 585 } 586 } 587 588 if (adev->gfx.kiq[0].ring.sched.ready && !adev->job_hang) 589 r = amdgpu_ring_test_helper(kiq_ring); 590 spin_unlock(&kiq->ring_lock); 591 592 return r; 593 } 594 595 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, 596 int queue_bit) 597 { 598 int mec, pipe, queue; 599 int set_resource_bit = 0; 600 601 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue); 602 603 set_resource_bit = mec * 4 * 8 + pipe * 8 + queue; 604 605 return set_resource_bit; 606 } 607 608 static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id) 609 { 610 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 611 struct amdgpu_ring *kiq_ring = &kiq->ring; 612 uint64_t queue_mask = ~0ULL; 613 int r, i, j; 614 615 amdgpu_device_flush_hdp(adev, NULL); 616 617 if (!adev->enable_uni_mes) { 618 spin_lock(&kiq->ring_lock); 619 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->set_resources_size); 620 if (r) { 621 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r); 622 spin_unlock(&kiq->ring_lock); 623 return r; 624 } 625 626 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 627 r = amdgpu_ring_test_helper(kiq_ring); 628 spin_unlock(&kiq->ring_lock); 629 if (r) 630 dev_err(adev->dev, "KIQ failed to set resources\n"); 631 } 632 633 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 634 j = i + xcc_id * adev->gfx.num_compute_rings; 635 r = amdgpu_mes_map_legacy_queue(adev, 636 &adev->gfx.compute_ring[j]); 637 if (r) { 638 dev_err(adev->dev, "failed to map compute queue\n"); 639 return r; 640 } 641 } 642 643 return 0; 644 } 645 646 int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id) 647 { 648 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 649 struct amdgpu_ring *kiq_ring = &kiq->ring; 650 uint64_t queue_mask = 0; 651 int r, i, j; 652 653 if (adev->mes.enable_legacy_queue_map) 654 return amdgpu_gfx_mes_enable_kcq(adev, xcc_id); 655 656 if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources) 657 return -EINVAL; 658 659 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 660 if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap)) 661 continue; 662 663 /* This situation may be hit in the future if a new HW 664 * generation exposes more than 64 queues. If so, the 665 * definition of queue_mask needs updating */ 666 if (WARN_ON(i > (sizeof(queue_mask)*8))) { 667 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 668 break; 669 } 670 671 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i)); 672 } 673 674 amdgpu_device_flush_hdp(adev, NULL); 675 676 DRM_INFO("kiq ring mec %d pipe %d q %d\n", kiq_ring->me, kiq_ring->pipe, 677 kiq_ring->queue); 678 679 spin_lock(&kiq->ring_lock); 680 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 681 adev->gfx.num_compute_rings + 682 kiq->pmf->set_resources_size); 683 if (r) { 684 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 685 spin_unlock(&kiq->ring_lock); 686 return r; 687 } 688 689 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask); 690 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 691 j = i + xcc_id * adev->gfx.num_compute_rings; 692 kiq->pmf->kiq_map_queues(kiq_ring, 693 &adev->gfx.compute_ring[j]); 694 } 695 696 r = amdgpu_ring_test_helper(kiq_ring); 697 spin_unlock(&kiq->ring_lock); 698 if (r) 699 DRM_ERROR("KCQ enable failed\n"); 700 701 return r; 702 } 703 704 int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id) 705 { 706 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 707 struct amdgpu_ring *kiq_ring = &kiq->ring; 708 int r, i, j; 709 710 if (!kiq->pmf || !kiq->pmf->kiq_map_queues) 711 return -EINVAL; 712 713 amdgpu_device_flush_hdp(adev, NULL); 714 715 if (adev->mes.enable_legacy_queue_map) { 716 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 717 j = i + xcc_id * adev->gfx.num_gfx_rings; 718 r = amdgpu_mes_map_legacy_queue(adev, 719 &adev->gfx.gfx_ring[j]); 720 if (r) { 721 DRM_ERROR("failed to map gfx queue\n"); 722 return r; 723 } 724 } 725 726 return 0; 727 } 728 729 spin_lock(&kiq->ring_lock); 730 /* No need to map kcq on the slave */ 731 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) { 732 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size * 733 adev->gfx.num_gfx_rings); 734 if (r) { 735 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 736 spin_unlock(&kiq->ring_lock); 737 return r; 738 } 739 740 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 741 j = i + xcc_id * adev->gfx.num_gfx_rings; 742 kiq->pmf->kiq_map_queues(kiq_ring, 743 &adev->gfx.gfx_ring[j]); 744 } 745 } 746 747 r = amdgpu_ring_test_helper(kiq_ring); 748 spin_unlock(&kiq->ring_lock); 749 if (r) 750 DRM_ERROR("KGQ enable failed\n"); 751 752 return r; 753 } 754 755 /* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable 756 * 757 * @adev: amdgpu_device pointer 758 * @bool enable true: enable gfx off feature, false: disable gfx off feature 759 * 760 * 1. gfx off feature will be enabled by gfx ip after gfx cg gp enabled. 761 * 2. other client can send request to disable gfx off feature, the request should be honored. 762 * 3. other client can cancel their request of disable gfx off feature 763 * 4. other client should not send request to enable gfx off feature before disable gfx off feature. 764 */ 765 766 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) 767 { 768 unsigned long delay = GFX_OFF_DELAY_ENABLE; 769 770 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) 771 return; 772 773 mutex_lock(&adev->gfx.gfx_off_mutex); 774 775 if (enable) { 776 /* If the count is already 0, it means there's an imbalance bug somewhere. 777 * Note that the bug may be in a different caller than the one which triggers the 778 * WARN_ON_ONCE. 779 */ 780 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0)) 781 goto unlock; 782 783 adev->gfx.gfx_off_req_count--; 784 785 if (adev->gfx.gfx_off_req_count == 0 && 786 !adev->gfx.gfx_off_state) { 787 /* If going to s2idle, no need to wait */ 788 if (adev->in_s0ix) { 789 if (!amdgpu_dpm_set_powergating_by_smu(adev, 790 AMD_IP_BLOCK_TYPE_GFX, true)) 791 adev->gfx.gfx_off_state = true; 792 } else { 793 schedule_delayed_work(&adev->gfx.gfx_off_delay_work, 794 delay); 795 } 796 } 797 } else { 798 if (adev->gfx.gfx_off_req_count == 0) { 799 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work); 800 801 if (adev->gfx.gfx_off_state && 802 !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false)) { 803 adev->gfx.gfx_off_state = false; 804 805 if (adev->gfx.funcs->init_spm_golden) { 806 dev_dbg(adev->dev, 807 "GFXOFF is disabled, re-init SPM golden settings\n"); 808 amdgpu_gfx_init_spm_golden(adev); 809 } 810 } 811 } 812 813 adev->gfx.gfx_off_req_count++; 814 } 815 816 unlock: 817 mutex_unlock(&adev->gfx.gfx_off_mutex); 818 } 819 820 int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value) 821 { 822 int r = 0; 823 824 mutex_lock(&adev->gfx.gfx_off_mutex); 825 826 r = amdgpu_dpm_set_residency_gfxoff(adev, value); 827 828 mutex_unlock(&adev->gfx.gfx_off_mutex); 829 830 return r; 831 } 832 833 int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value) 834 { 835 int r = 0; 836 837 mutex_lock(&adev->gfx.gfx_off_mutex); 838 839 r = amdgpu_dpm_get_residency_gfxoff(adev, value); 840 841 mutex_unlock(&adev->gfx.gfx_off_mutex); 842 843 return r; 844 } 845 846 int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value) 847 { 848 int r = 0; 849 850 mutex_lock(&adev->gfx.gfx_off_mutex); 851 852 r = amdgpu_dpm_get_entrycount_gfxoff(adev, value); 853 854 mutex_unlock(&adev->gfx.gfx_off_mutex); 855 856 return r; 857 } 858 859 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value) 860 { 861 862 int r = 0; 863 864 mutex_lock(&adev->gfx.gfx_off_mutex); 865 866 r = amdgpu_dpm_get_status_gfxoff(adev, value); 867 868 mutex_unlock(&adev->gfx.gfx_off_mutex); 869 870 return r; 871 } 872 873 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 874 { 875 int r; 876 877 if (amdgpu_ras_is_supported(adev, ras_block->block)) { 878 if (!amdgpu_persistent_edc_harvesting_supported(adev)) { 879 r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX); 880 if (r) 881 return r; 882 } 883 884 r = amdgpu_ras_block_late_init(adev, ras_block); 885 if (r) 886 return r; 887 888 if (adev->gfx.cp_ecc_error_irq.funcs) { 889 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 890 if (r) 891 goto late_fini; 892 } 893 } else { 894 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 895 } 896 897 return 0; 898 late_fini: 899 amdgpu_ras_block_late_fini(adev, ras_block); 900 return r; 901 } 902 903 int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) 904 { 905 int err = 0; 906 struct amdgpu_gfx_ras *ras = NULL; 907 908 /* adev->gfx.ras is NULL, which means gfx does not 909 * support ras function, then do nothing here. 910 */ 911 if (!adev->gfx.ras) 912 return 0; 913 914 ras = adev->gfx.ras; 915 916 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); 917 if (err) { 918 dev_err(adev->dev, "Failed to register gfx ras block!\n"); 919 return err; 920 } 921 922 strcpy(ras->ras_block.ras_comm.name, "gfx"); 923 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX; 924 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 925 adev->gfx.ras_if = &ras->ras_block.ras_comm; 926 927 /* If not define special ras_late_init function, use gfx default ras_late_init */ 928 if (!ras->ras_block.ras_late_init) 929 ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init; 930 931 /* If not defined special ras_cb function, use default ras_cb */ 932 if (!ras->ras_block.ras_cb) 933 ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb; 934 935 return 0; 936 } 937 938 int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev, 939 struct amdgpu_iv_entry *entry) 940 { 941 if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler) 942 return adev->gfx.ras->poison_consumption_handler(adev, entry); 943 944 return 0; 945 } 946 947 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev, 948 void *err_data, 949 struct amdgpu_iv_entry *entry) 950 { 951 /* TODO ue will trigger an interrupt. 952 * 953 * When “Full RAS” is enabled, the per-IP interrupt sources should 954 * be disabled and the driver should only look for the aggregated 955 * interrupt via sync flood 956 */ 957 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) { 958 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 959 if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops && 960 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count) 961 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 962 amdgpu_ras_reset_gpu(adev); 963 } 964 return AMDGPU_RAS_SUCCESS; 965 } 966 967 int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, 968 struct amdgpu_irq_src *source, 969 struct amdgpu_iv_entry *entry) 970 { 971 struct ras_common_if *ras_if = adev->gfx.ras_if; 972 struct ras_dispatch_if ih_data = { 973 .entry = entry, 974 }; 975 976 if (!ras_if) 977 return 0; 978 979 ih_data.head = *ras_if; 980 981 DRM_ERROR("CP ECC ERROR IRQ\n"); 982 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 983 return 0; 984 } 985 986 void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev, 987 void *ras_error_status, 988 void (*func)(struct amdgpu_device *adev, void *ras_error_status, 989 int xcc_id)) 990 { 991 int i; 992 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 993 uint32_t xcc_mask = GENMASK(num_xcc - 1, 0); 994 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 995 996 if (err_data) { 997 err_data->ue_count = 0; 998 err_data->ce_count = 0; 999 } 1000 1001 for_each_inst(i, xcc_mask) 1002 func(adev, ras_error_status, i); 1003 } 1004 1005 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id) 1006 { 1007 signed long r, cnt = 0; 1008 unsigned long flags; 1009 uint32_t seq, reg_val_offs = 0, value = 0; 1010 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 1011 struct amdgpu_ring *ring = &kiq->ring; 1012 1013 if (amdgpu_device_skip_hw_access(adev)) 1014 return 0; 1015 1016 if (adev->mes.ring[0].sched.ready) 1017 return amdgpu_mes_rreg(adev, reg); 1018 1019 BUG_ON(!ring->funcs->emit_rreg); 1020 1021 spin_lock_irqsave(&kiq->ring_lock, flags); 1022 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 1023 pr_err("critical bug! too many kiq readers\n"); 1024 goto failed_unlock; 1025 } 1026 r = amdgpu_ring_alloc(ring, 32); 1027 if (r) 1028 goto failed_unlock; 1029 1030 amdgpu_ring_emit_rreg(ring, reg, reg_val_offs); 1031 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 1032 if (r) 1033 goto failed_undo; 1034 1035 amdgpu_ring_commit(ring); 1036 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1037 1038 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1039 1040 /* don't wait anymore for gpu reset case because this way may 1041 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 1042 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 1043 * never return if we keep waiting in virt_kiq_rreg, which cause 1044 * gpu_recover() hang there. 1045 * 1046 * also don't wait anymore for IRQ context 1047 * */ 1048 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 1049 goto failed_kiq_read; 1050 1051 might_sleep(); 1052 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 1053 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 1054 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1055 } 1056 1057 if (cnt > MAX_KIQ_REG_TRY) 1058 goto failed_kiq_read; 1059 1060 mb(); 1061 value = adev->wb.wb[reg_val_offs]; 1062 amdgpu_device_wb_free(adev, reg_val_offs); 1063 return value; 1064 1065 failed_undo: 1066 amdgpu_ring_undo(ring); 1067 failed_unlock: 1068 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1069 failed_kiq_read: 1070 if (reg_val_offs) 1071 amdgpu_device_wb_free(adev, reg_val_offs); 1072 dev_err(adev->dev, "failed to read reg:%x\n", reg); 1073 return ~0; 1074 } 1075 1076 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id) 1077 { 1078 signed long r, cnt = 0; 1079 unsigned long flags; 1080 uint32_t seq; 1081 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id]; 1082 struct amdgpu_ring *ring = &kiq->ring; 1083 1084 BUG_ON(!ring->funcs->emit_wreg); 1085 1086 if (amdgpu_device_skip_hw_access(adev)) 1087 return; 1088 1089 if (adev->mes.ring[0].sched.ready) { 1090 amdgpu_mes_wreg(adev, reg, v); 1091 return; 1092 } 1093 1094 spin_lock_irqsave(&kiq->ring_lock, flags); 1095 r = amdgpu_ring_alloc(ring, 32); 1096 if (r) 1097 goto failed_unlock; 1098 1099 amdgpu_ring_emit_wreg(ring, reg, v); 1100 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 1101 if (r) 1102 goto failed_undo; 1103 1104 amdgpu_ring_commit(ring); 1105 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1106 1107 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1108 1109 /* don't wait anymore for gpu reset case because this way may 1110 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 1111 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 1112 * never return if we keep waiting in virt_kiq_rreg, which cause 1113 * gpu_recover() hang there. 1114 * 1115 * also don't wait anymore for IRQ context 1116 * */ 1117 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 1118 goto failed_kiq_write; 1119 1120 might_sleep(); 1121 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 1122 1123 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 1124 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1125 } 1126 1127 if (cnt > MAX_KIQ_REG_TRY) 1128 goto failed_kiq_write; 1129 1130 return; 1131 1132 failed_undo: 1133 amdgpu_ring_undo(ring); 1134 failed_unlock: 1135 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1136 failed_kiq_write: 1137 dev_err(adev->dev, "failed to write reg:%x\n", reg); 1138 } 1139 1140 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev) 1141 { 1142 if (amdgpu_num_kcq == -1) { 1143 return 8; 1144 } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) { 1145 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n"); 1146 return 8; 1147 } 1148 return amdgpu_num_kcq; 1149 } 1150 1151 void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, 1152 uint32_t ucode_id) 1153 { 1154 const struct gfx_firmware_header_v1_0 *cp_hdr; 1155 const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0; 1156 struct amdgpu_firmware_info *info = NULL; 1157 const struct firmware *ucode_fw; 1158 unsigned int fw_size; 1159 1160 switch (ucode_id) { 1161 case AMDGPU_UCODE_ID_CP_PFP: 1162 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1163 adev->gfx.pfp_fw->data; 1164 adev->gfx.pfp_fw_version = 1165 le32_to_cpu(cp_hdr->header.ucode_version); 1166 adev->gfx.pfp_feature_version = 1167 le32_to_cpu(cp_hdr->ucode_feature_version); 1168 ucode_fw = adev->gfx.pfp_fw; 1169 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1170 break; 1171 case AMDGPU_UCODE_ID_CP_RS64_PFP: 1172 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1173 adev->gfx.pfp_fw->data; 1174 adev->gfx.pfp_fw_version = 1175 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1176 adev->gfx.pfp_feature_version = 1177 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1178 ucode_fw = adev->gfx.pfp_fw; 1179 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1180 break; 1181 case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK: 1182 case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK: 1183 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1184 adev->gfx.pfp_fw->data; 1185 ucode_fw = adev->gfx.pfp_fw; 1186 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1187 break; 1188 case AMDGPU_UCODE_ID_CP_ME: 1189 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1190 adev->gfx.me_fw->data; 1191 adev->gfx.me_fw_version = 1192 le32_to_cpu(cp_hdr->header.ucode_version); 1193 adev->gfx.me_feature_version = 1194 le32_to_cpu(cp_hdr->ucode_feature_version); 1195 ucode_fw = adev->gfx.me_fw; 1196 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1197 break; 1198 case AMDGPU_UCODE_ID_CP_RS64_ME: 1199 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1200 adev->gfx.me_fw->data; 1201 adev->gfx.me_fw_version = 1202 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1203 adev->gfx.me_feature_version = 1204 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1205 ucode_fw = adev->gfx.me_fw; 1206 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1207 break; 1208 case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK: 1209 case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK: 1210 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1211 adev->gfx.me_fw->data; 1212 ucode_fw = adev->gfx.me_fw; 1213 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1214 break; 1215 case AMDGPU_UCODE_ID_CP_CE: 1216 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1217 adev->gfx.ce_fw->data; 1218 adev->gfx.ce_fw_version = 1219 le32_to_cpu(cp_hdr->header.ucode_version); 1220 adev->gfx.ce_feature_version = 1221 le32_to_cpu(cp_hdr->ucode_feature_version); 1222 ucode_fw = adev->gfx.ce_fw; 1223 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1224 break; 1225 case AMDGPU_UCODE_ID_CP_MEC1: 1226 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1227 adev->gfx.mec_fw->data; 1228 adev->gfx.mec_fw_version = 1229 le32_to_cpu(cp_hdr->header.ucode_version); 1230 adev->gfx.mec_feature_version = 1231 le32_to_cpu(cp_hdr->ucode_feature_version); 1232 ucode_fw = adev->gfx.mec_fw; 1233 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1234 le32_to_cpu(cp_hdr->jt_size) * 4; 1235 break; 1236 case AMDGPU_UCODE_ID_CP_MEC1_JT: 1237 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1238 adev->gfx.mec_fw->data; 1239 ucode_fw = adev->gfx.mec_fw; 1240 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; 1241 break; 1242 case AMDGPU_UCODE_ID_CP_MEC2: 1243 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1244 adev->gfx.mec2_fw->data; 1245 adev->gfx.mec2_fw_version = 1246 le32_to_cpu(cp_hdr->header.ucode_version); 1247 adev->gfx.mec2_feature_version = 1248 le32_to_cpu(cp_hdr->ucode_feature_version); 1249 ucode_fw = adev->gfx.mec2_fw; 1250 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1251 le32_to_cpu(cp_hdr->jt_size) * 4; 1252 break; 1253 case AMDGPU_UCODE_ID_CP_MEC2_JT: 1254 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1255 adev->gfx.mec2_fw->data; 1256 ucode_fw = adev->gfx.mec2_fw; 1257 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; 1258 break; 1259 case AMDGPU_UCODE_ID_CP_RS64_MEC: 1260 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1261 adev->gfx.mec_fw->data; 1262 adev->gfx.mec_fw_version = 1263 le32_to_cpu(cp_hdr_v2_0->header.ucode_version); 1264 adev->gfx.mec_feature_version = 1265 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); 1266 ucode_fw = adev->gfx.mec_fw; 1267 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); 1268 break; 1269 case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK: 1270 case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK: 1271 case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK: 1272 case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK: 1273 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) 1274 adev->gfx.mec_fw->data; 1275 ucode_fw = adev->gfx.mec_fw; 1276 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); 1277 break; 1278 default: 1279 dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id); 1280 return; 1281 } 1282 1283 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1284 info = &adev->firmware.ucode[ucode_id]; 1285 info->ucode_id = ucode_id; 1286 info->fw = ucode_fw; 1287 adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE); 1288 } 1289 } 1290 1291 bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id) 1292 { 1293 return !(xcc_id % (adev->gfx.num_xcc_per_xcp ? 1294 adev->gfx.num_xcc_per_xcp : 1)); 1295 } 1296 1297 static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev, 1298 struct device_attribute *addr, 1299 char *buf) 1300 { 1301 struct drm_device *ddev = dev_get_drvdata(dev); 1302 struct amdgpu_device *adev = drm_to_adev(ddev); 1303 int mode; 1304 1305 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr, 1306 AMDGPU_XCP_FL_NONE); 1307 1308 return sysfs_emit(buf, "%s\n", amdgpu_gfx_compute_mode_desc(mode)); 1309 } 1310 1311 static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev, 1312 struct device_attribute *addr, 1313 const char *buf, size_t count) 1314 { 1315 struct drm_device *ddev = dev_get_drvdata(dev); 1316 struct amdgpu_device *adev = drm_to_adev(ddev); 1317 enum amdgpu_gfx_partition mode; 1318 int ret = 0, num_xcc; 1319 1320 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1321 if (num_xcc % 2 != 0) 1322 return -EINVAL; 1323 1324 if (!strncasecmp("SPX", buf, strlen("SPX"))) { 1325 mode = AMDGPU_SPX_PARTITION_MODE; 1326 } else if (!strncasecmp("DPX", buf, strlen("DPX"))) { 1327 /* 1328 * DPX mode needs AIDs to be in multiple of 2. 1329 * Each AID connects 2 XCCs. 1330 */ 1331 if (num_xcc%4) 1332 return -EINVAL; 1333 mode = AMDGPU_DPX_PARTITION_MODE; 1334 } else if (!strncasecmp("TPX", buf, strlen("TPX"))) { 1335 if (num_xcc != 6) 1336 return -EINVAL; 1337 mode = AMDGPU_TPX_PARTITION_MODE; 1338 } else if (!strncasecmp("QPX", buf, strlen("QPX"))) { 1339 if (num_xcc != 8) 1340 return -EINVAL; 1341 mode = AMDGPU_QPX_PARTITION_MODE; 1342 } else if (!strncasecmp("CPX", buf, strlen("CPX"))) { 1343 mode = AMDGPU_CPX_PARTITION_MODE; 1344 } else { 1345 return -EINVAL; 1346 } 1347 1348 ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode); 1349 1350 if (ret) 1351 return ret; 1352 1353 return count; 1354 } 1355 1356 static const char *xcp_desc[] = { 1357 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 1358 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 1359 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 1360 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 1361 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 1362 }; 1363 1364 static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev, 1365 struct device_attribute *addr, 1366 char *buf) 1367 { 1368 struct drm_device *ddev = dev_get_drvdata(dev); 1369 struct amdgpu_device *adev = drm_to_adev(ddev); 1370 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1371 int size = 0, mode; 1372 char *sep = ""; 1373 1374 if (!xcp_mgr || !xcp_mgr->avail_xcp_modes) 1375 return sysfs_emit(buf, "Not supported\n"); 1376 1377 for_each_inst(mode, xcp_mgr->avail_xcp_modes) { 1378 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 1379 sep = ", "; 1380 } 1381 1382 size += sysfs_emit_at(buf, size, "\n"); 1383 1384 return size; 1385 } 1386 1387 static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) 1388 { 1389 struct amdgpu_device *adev = ring->adev; 1390 struct drm_gpu_scheduler *sched = &ring->sched; 1391 struct drm_sched_entity entity; 1392 struct dma_fence *f; 1393 struct amdgpu_job *job; 1394 struct amdgpu_ib *ib; 1395 int i, r; 1396 1397 /* Initialize the scheduler entity */ 1398 r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL, 1399 &sched, 1, NULL); 1400 if (r) { 1401 dev_err(adev->dev, "Failed setting up GFX kernel entity.\n"); 1402 goto err; 1403 } 1404 1405 r = amdgpu_job_alloc_with_ib(ring->adev, &entity, NULL, 1406 64, 0, 1407 &job); 1408 if (r) 1409 goto err; 1410 1411 job->enforce_isolation = true; 1412 1413 ib = &job->ibs[0]; 1414 for (i = 0; i <= ring->funcs->align_mask; ++i) 1415 ib->ptr[i] = ring->funcs->nop; 1416 ib->length_dw = ring->funcs->align_mask + 1; 1417 1418 f = amdgpu_job_submit(job); 1419 1420 r = dma_fence_wait(f, false); 1421 if (r) 1422 goto err; 1423 1424 dma_fence_put(f); 1425 1426 /* Clean up the scheduler entity */ 1427 drm_sched_entity_destroy(&entity); 1428 return 0; 1429 1430 err: 1431 return r; 1432 } 1433 1434 static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id) 1435 { 1436 int num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1437 struct amdgpu_ring *ring; 1438 int num_xcc_to_clear; 1439 int i, r, xcc_id; 1440 1441 if (adev->gfx.num_xcc_per_xcp) 1442 num_xcc_to_clear = adev->gfx.num_xcc_per_xcp; 1443 else 1444 num_xcc_to_clear = 1; 1445 1446 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1447 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 1448 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings]; 1449 if ((ring->xcp_id == xcp_id) && ring->sched.ready) { 1450 r = amdgpu_gfx_run_cleaner_shader_job(ring); 1451 if (r) 1452 return r; 1453 num_xcc_to_clear--; 1454 break; 1455 } 1456 } 1457 } 1458 1459 if (num_xcc_to_clear) 1460 return -ENOENT; 1461 1462 return 0; 1463 } 1464 1465 static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev, 1466 struct device_attribute *attr, 1467 const char *buf, 1468 size_t count) 1469 { 1470 struct drm_device *ddev = dev_get_drvdata(dev); 1471 struct amdgpu_device *adev = drm_to_adev(ddev); 1472 int ret; 1473 long value; 1474 1475 if (amdgpu_in_reset(adev)) 1476 return -EPERM; 1477 if (adev->in_suspend && !adev->in_runpm) 1478 return -EPERM; 1479 1480 ret = kstrtol(buf, 0, &value); 1481 1482 if (ret) 1483 return -EINVAL; 1484 1485 if (value < 0) 1486 return -EINVAL; 1487 1488 if (adev->xcp_mgr) { 1489 if (value >= adev->xcp_mgr->num_xcps) 1490 return -EINVAL; 1491 } else { 1492 if (value > 1) 1493 return -EINVAL; 1494 } 1495 1496 ret = pm_runtime_get_sync(ddev->dev); 1497 if (ret < 0) { 1498 pm_runtime_put_autosuspend(ddev->dev); 1499 return ret; 1500 } 1501 1502 ret = amdgpu_gfx_run_cleaner_shader(adev, value); 1503 1504 pm_runtime_mark_last_busy(ddev->dev); 1505 pm_runtime_put_autosuspend(ddev->dev); 1506 1507 if (ret) 1508 return ret; 1509 1510 return count; 1511 } 1512 1513 static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev, 1514 struct device_attribute *attr, 1515 char *buf) 1516 { 1517 struct drm_device *ddev = dev_get_drvdata(dev); 1518 struct amdgpu_device *adev = drm_to_adev(ddev); 1519 int i; 1520 ssize_t size = 0; 1521 1522 if (adev->xcp_mgr) { 1523 for (i = 0; i < adev->xcp_mgr->num_xcps; i++) { 1524 size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]); 1525 if (i < (adev->xcp_mgr->num_xcps - 1)) 1526 size += sysfs_emit_at(buf, size, " "); 1527 } 1528 buf[size++] = '\n'; 1529 } else { 1530 size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]); 1531 } 1532 1533 return size; 1534 } 1535 1536 static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev, 1537 struct device_attribute *attr, 1538 const char *buf, size_t count) 1539 { 1540 struct drm_device *ddev = dev_get_drvdata(dev); 1541 struct amdgpu_device *adev = drm_to_adev(ddev); 1542 long partition_values[MAX_XCP] = {0}; 1543 int ret, i, num_partitions; 1544 const char *input_buf = buf; 1545 1546 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { 1547 ret = sscanf(input_buf, "%ld", &partition_values[i]); 1548 if (ret <= 0) 1549 break; 1550 1551 /* Move the pointer to the next value in the string */ 1552 input_buf = strchr(input_buf, ' '); 1553 if (input_buf) { 1554 input_buf++; 1555 } else { 1556 i++; 1557 break; 1558 } 1559 } 1560 num_partitions = i; 1561 1562 if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps) 1563 return -EINVAL; 1564 1565 if (!adev->xcp_mgr && num_partitions != 1) 1566 return -EINVAL; 1567 1568 for (i = 0; i < num_partitions; i++) { 1569 if (partition_values[i] != 0 && partition_values[i] != 1) 1570 return -EINVAL; 1571 } 1572 1573 mutex_lock(&adev->enforce_isolation_mutex); 1574 1575 for (i = 0; i < num_partitions; i++) { 1576 if (adev->enforce_isolation[i] && !partition_values[i]) { 1577 /* Going from enabled to disabled */ 1578 amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(i)); 1579 } else if (!adev->enforce_isolation[i] && partition_values[i]) { 1580 /* Going from disabled to enabled */ 1581 amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(i)); 1582 } 1583 adev->enforce_isolation[i] = partition_values[i]; 1584 } 1585 1586 mutex_unlock(&adev->enforce_isolation_mutex); 1587 1588 return count; 1589 } 1590 1591 static DEVICE_ATTR(run_cleaner_shader, 0200, 1592 NULL, amdgpu_gfx_set_run_cleaner_shader); 1593 1594 static DEVICE_ATTR(enforce_isolation, 0644, 1595 amdgpu_gfx_get_enforce_isolation, 1596 amdgpu_gfx_set_enforce_isolation); 1597 1598 static DEVICE_ATTR(current_compute_partition, 0644, 1599 amdgpu_gfx_get_current_compute_partition, 1600 amdgpu_gfx_set_compute_partition); 1601 1602 static DEVICE_ATTR(available_compute_partition, 0444, 1603 amdgpu_gfx_get_available_compute_partition, NULL); 1604 1605 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev) 1606 { 1607 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1608 bool xcp_switch_supported; 1609 int r; 1610 1611 if (!xcp_mgr) 1612 return 0; 1613 1614 xcp_switch_supported = 1615 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode); 1616 1617 if (!xcp_switch_supported) 1618 dev_attr_current_compute_partition.attr.mode &= 1619 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1620 1621 r = device_create_file(adev->dev, &dev_attr_current_compute_partition); 1622 if (r) 1623 return r; 1624 1625 if (xcp_switch_supported) 1626 r = device_create_file(adev->dev, 1627 &dev_attr_available_compute_partition); 1628 1629 return r; 1630 } 1631 1632 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev) 1633 { 1634 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 1635 bool xcp_switch_supported; 1636 1637 if (!xcp_mgr) 1638 return; 1639 1640 xcp_switch_supported = 1641 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode); 1642 device_remove_file(adev->dev, &dev_attr_current_compute_partition); 1643 1644 if (xcp_switch_supported) 1645 device_remove_file(adev->dev, 1646 &dev_attr_available_compute_partition); 1647 } 1648 1649 int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev) 1650 { 1651 int r; 1652 1653 r = device_create_file(adev->dev, &dev_attr_enforce_isolation); 1654 if (r) 1655 return r; 1656 1657 r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader); 1658 if (r) 1659 return r; 1660 1661 return 0; 1662 } 1663 1664 void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev) 1665 { 1666 device_remove_file(adev->dev, &dev_attr_enforce_isolation); 1667 device_remove_file(adev->dev, &dev_attr_run_cleaner_shader); 1668 } 1669 1670 int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev, 1671 unsigned int cleaner_shader_size) 1672 { 1673 if (!adev->gfx.enable_cleaner_shader) 1674 return -EOPNOTSUPP; 1675 1676 return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE, 1677 AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT, 1678 &adev->gfx.cleaner_shader_obj, 1679 &adev->gfx.cleaner_shader_gpu_addr, 1680 (void **)&adev->gfx.cleaner_shader_cpu_ptr); 1681 } 1682 1683 void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev) 1684 { 1685 if (!adev->gfx.enable_cleaner_shader) 1686 return; 1687 1688 amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj, 1689 &adev->gfx.cleaner_shader_gpu_addr, 1690 (void **)&adev->gfx.cleaner_shader_cpu_ptr); 1691 } 1692 1693 void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev, 1694 unsigned int cleaner_shader_size, 1695 const void *cleaner_shader_ptr) 1696 { 1697 if (!adev->gfx.enable_cleaner_shader) 1698 return; 1699 1700 if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr) 1701 memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr, 1702 cleaner_shader_size); 1703 } 1704 1705 /** 1706 * amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver) 1707 * @adev: amdgpu_device pointer 1708 * @idx: Index of the scheduler to control 1709 * @enable: Whether to enable or disable the KFD scheduler 1710 * 1711 * This function is used to control the KFD (Kernel Fusion Driver) scheduler 1712 * from the KGD. It is part of the cleaner shader feature. This function plays 1713 * a key role in enforcing process isolation on the GPU. 1714 * 1715 * The function uses a reference count mechanism (kfd_sch_req_count) to keep 1716 * track of the number of requests to enable the KFD scheduler. When a request 1717 * to enable the KFD scheduler is made, the reference count is decremented. 1718 * When the reference count reaches zero, a delayed work is scheduled to 1719 * enforce isolation after a delay of GFX_SLICE_PERIOD. 1720 * 1721 * When a request to disable the KFD scheduler is made, the function first 1722 * checks if the reference count is zero. If it is, it cancels the delayed work 1723 * for enforcing isolation and checks if the KFD scheduler is active. If the 1724 * KFD scheduler is active, it sends a request to stop the KFD scheduler and 1725 * sets the KFD scheduler state to inactive. Then, it increments the reference 1726 * count. 1727 * 1728 * The function is synchronized using the kfd_sch_mutex to ensure that the KFD 1729 * scheduler state and reference count are updated atomically. 1730 * 1731 * Note: If the reference count is already zero when a request to enable the 1732 * KFD scheduler is made, it means there's an imbalance bug somewhere. The 1733 * function triggers a warning in this case. 1734 */ 1735 static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx, 1736 bool enable) 1737 { 1738 mutex_lock(&adev->gfx.kfd_sch_mutex); 1739 1740 if (enable) { 1741 /* If the count is already 0, it means there's an imbalance bug somewhere. 1742 * Note that the bug may be in a different caller than the one which triggers the 1743 * WARN_ON_ONCE. 1744 */ 1745 if (WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx] == 0)) { 1746 dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n"); 1747 goto unlock; 1748 } 1749 1750 adev->gfx.kfd_sch_req_count[idx]--; 1751 1752 if (adev->gfx.kfd_sch_req_count[idx] == 0 && 1753 adev->gfx.kfd_sch_inactive[idx]) { 1754 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, 1755 msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx])); 1756 } 1757 } else { 1758 if (adev->gfx.kfd_sch_req_count[idx] == 0) { 1759 cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work); 1760 if (!adev->gfx.kfd_sch_inactive[idx]) { 1761 amdgpu_amdkfd_stop_sched(adev, idx); 1762 adev->gfx.kfd_sch_inactive[idx] = true; 1763 } 1764 } 1765 1766 adev->gfx.kfd_sch_req_count[idx]++; 1767 } 1768 1769 unlock: 1770 mutex_unlock(&adev->gfx.kfd_sch_mutex); 1771 } 1772 1773 /** 1774 * amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation 1775 * 1776 * @work: work_struct. 1777 * 1778 * This function is the work handler for enforcing shader isolation on AMD GPUs. 1779 * It counts the number of emitted fences for each GFX and compute ring. If there 1780 * are any fences, it schedules the `enforce_isolation_work` to be run after a 1781 * delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion 1782 * Driver (KFD) to resume the runqueue. The function is synchronized using the 1783 * `enforce_isolation_mutex`. 1784 */ 1785 void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work) 1786 { 1787 struct amdgpu_isolation_work *isolation_work = 1788 container_of(work, struct amdgpu_isolation_work, work.work); 1789 struct amdgpu_device *adev = isolation_work->adev; 1790 u32 i, idx, fences = 0; 1791 1792 if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION) 1793 idx = 0; 1794 else 1795 idx = isolation_work->xcp_id; 1796 1797 if (idx >= MAX_XCP) 1798 return; 1799 1800 mutex_lock(&adev->enforce_isolation_mutex); 1801 for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) { 1802 if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id) 1803 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]); 1804 } 1805 for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) { 1806 if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id) 1807 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]); 1808 } 1809 if (fences) { 1810 /* we've already had our timeslice, so let's wrap this up */ 1811 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work, 1812 msecs_to_jiffies(1)); 1813 } else { 1814 /* Tell KFD to resume the runqueue */ 1815 if (adev->kfd.init_complete) { 1816 WARN_ON_ONCE(!adev->gfx.kfd_sch_inactive[idx]); 1817 WARN_ON_ONCE(adev->gfx.kfd_sch_req_count[idx]); 1818 amdgpu_amdkfd_start_sched(adev, idx); 1819 adev->gfx.kfd_sch_inactive[idx] = false; 1820 } 1821 } 1822 mutex_unlock(&adev->enforce_isolation_mutex); 1823 } 1824 1825 static void 1826 amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev, 1827 u32 idx) 1828 { 1829 unsigned long cjiffies; 1830 bool wait = false; 1831 1832 mutex_lock(&adev->enforce_isolation_mutex); 1833 if (adev->enforce_isolation[idx]) { 1834 /* set the initial values if nothing is set */ 1835 if (!adev->gfx.enforce_isolation_jiffies[idx]) { 1836 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 1837 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 1838 } 1839 /* Make sure KFD gets a chance to run */ 1840 if (amdgpu_amdkfd_compute_active(adev, idx)) { 1841 cjiffies = jiffies; 1842 if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) { 1843 cjiffies -= adev->gfx.enforce_isolation_jiffies[idx]; 1844 if ((jiffies_to_msecs(cjiffies) >= GFX_SLICE_PERIOD_MS)) { 1845 /* if our time is up, let KGD work drain before scheduling more */ 1846 wait = true; 1847 /* reset the timer period */ 1848 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 1849 } else { 1850 /* set the timer period to what's left in our time slice */ 1851 adev->gfx.enforce_isolation_time[idx] = 1852 GFX_SLICE_PERIOD_MS - jiffies_to_msecs(cjiffies); 1853 } 1854 } else { 1855 /* if jiffies wrap around we will just wait a little longer */ 1856 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 1857 } 1858 } else { 1859 /* if there is no KFD work, then set the full slice period */ 1860 adev->gfx.enforce_isolation_jiffies[idx] = jiffies; 1861 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS; 1862 } 1863 } 1864 mutex_unlock(&adev->enforce_isolation_mutex); 1865 1866 if (wait) 1867 msleep(GFX_SLICE_PERIOD_MS); 1868 } 1869 1870 void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring) 1871 { 1872 struct amdgpu_device *adev = ring->adev; 1873 u32 idx; 1874 1875 if (!adev->gfx.enable_cleaner_shader) 1876 return; 1877 1878 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION) 1879 idx = 0; 1880 else 1881 idx = ring->xcp_id; 1882 1883 if (idx >= MAX_XCP) 1884 return; 1885 1886 /* Don't submit more work until KFD has had some time */ 1887 amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx); 1888 1889 mutex_lock(&adev->enforce_isolation_mutex); 1890 if (adev->enforce_isolation[idx]) { 1891 if (adev->kfd.init_complete) 1892 amdgpu_gfx_kfd_sch_ctrl(adev, idx, false); 1893 } 1894 mutex_unlock(&adev->enforce_isolation_mutex); 1895 } 1896 1897 void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring) 1898 { 1899 struct amdgpu_device *adev = ring->adev; 1900 u32 idx; 1901 1902 if (!adev->gfx.enable_cleaner_shader) 1903 return; 1904 1905 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION) 1906 idx = 0; 1907 else 1908 idx = ring->xcp_id; 1909 1910 if (idx >= MAX_XCP) 1911 return; 1912 1913 mutex_lock(&adev->enforce_isolation_mutex); 1914 if (adev->enforce_isolation[idx]) { 1915 if (adev->kfd.init_complete) 1916 amdgpu_gfx_kfd_sch_ctrl(adev, idx, true); 1917 } 1918 mutex_unlock(&adev->enforce_isolation_mutex); 1919 } 1920