1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 #include "amdgpu_ip.h" 27 28 #include <drm/drm_drv.h> 29 #include "../amdxcp/amdgpu_xcp_drv.h" 30 31 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr); 32 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr); 33 34 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 35 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 36 { 37 int (*run_func)(void *handle, uint32_t inst_mask); 38 int ret = 0; 39 40 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 41 return 0; 42 43 run_func = NULL; 44 45 switch (xcp_state) { 46 case AMDGPU_XCP_PREPARE_SUSPEND: 47 run_func = xcp_ip->ip_funcs->prepare_suspend; 48 break; 49 case AMDGPU_XCP_SUSPEND: 50 run_func = xcp_ip->ip_funcs->suspend; 51 break; 52 case AMDGPU_XCP_PREPARE_RESUME: 53 run_func = xcp_ip->ip_funcs->prepare_resume; 54 break; 55 case AMDGPU_XCP_RESUME: 56 run_func = xcp_ip->ip_funcs->resume; 57 break; 58 } 59 60 if (run_func) 61 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 62 63 return ret; 64 } 65 66 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 67 int state) 68 { 69 struct amdgpu_xcp_ip *xcp_ip; 70 struct amdgpu_xcp *xcp; 71 int i, ret; 72 73 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 74 return -EINVAL; 75 76 xcp = &xcp_mgr->xcp[xcp_id]; 77 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 78 xcp_ip = &xcp->ip[i]; 79 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 80 if (ret) 81 break; 82 } 83 84 return ret; 85 } 86 87 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 88 { 89 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 90 AMDGPU_XCP_PREPARE_SUSPEND); 91 } 92 93 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 94 { 95 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 96 } 97 98 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 99 { 100 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 101 AMDGPU_XCP_PREPARE_RESUME); 102 } 103 104 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 105 { 106 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 107 } 108 109 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 110 struct amdgpu_xcp_ip *ip) 111 { 112 struct amdgpu_xcp *xcp; 113 114 if (!ip) 115 return; 116 117 xcp = &xcp_mgr->xcp[xcp_id]; 118 xcp->ip[ip->ip_id] = *ip; 119 xcp->ip[ip->ip_id].valid = true; 120 121 xcp->valid = true; 122 } 123 124 static void __amdgpu_xcp_set_unique_id(struct amdgpu_xcp_mgr *xcp_mgr, 125 int xcp_id) 126 { 127 struct amdgpu_xcp *xcp = &xcp_mgr->xcp[xcp_id]; 128 struct amdgpu_device *adev = xcp_mgr->adev; 129 uint32_t inst_mask; 130 uint64_t uid; 131 int i; 132 133 if (!amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask) && 134 inst_mask) { 135 i = GET_INST(GC, (ffs(inst_mask) - 1)); 136 uid = amdgpu_device_get_uid(xcp_mgr->adev->uid_info, 137 AMDGPU_UID_TYPE_XCD, i); 138 if (uid) 139 xcp->unique_id = uid; 140 } 141 } 142 143 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 144 { 145 struct amdgpu_device *adev = xcp_mgr->adev; 146 struct amdgpu_xcp_ip ip; 147 uint8_t mem_id; 148 int i, j, ret; 149 150 if (!num_xcps || num_xcps > MAX_XCP) 151 return -EINVAL; 152 153 xcp_mgr->mode = mode; 154 155 for (i = 0; i < MAX_XCP; ++i) 156 xcp_mgr->xcp[i].valid = false; 157 158 /* This is needed for figuring out memory id of xcp */ 159 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 160 161 for (i = 0; i < num_xcps; ++i) { 162 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 163 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 164 &ip); 165 if (ret) 166 continue; 167 168 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 169 } 170 171 xcp_mgr->xcp[i].id = i; 172 173 if (xcp_mgr->funcs->get_xcp_mem_id) { 174 ret = xcp_mgr->funcs->get_xcp_mem_id( 175 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 176 if (ret) 177 continue; 178 else 179 xcp_mgr->xcp[i].mem_id = mem_id; 180 } 181 __amdgpu_xcp_set_unique_id(xcp_mgr, i); 182 } 183 184 xcp_mgr->num_xcps = num_xcps; 185 xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN; 186 amdgpu_xcp_update_partition_sched_list(adev); 187 188 return 0; 189 } 190 191 static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, 192 int mode) 193 { 194 int ret, curr_mode, num_xcps = 0; 195 196 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 197 return 0; 198 199 mutex_lock(&xcp_mgr->xcp_lock); 200 201 curr_mode = xcp_mgr->mode; 202 /* State set to transient mode */ 203 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 204 205 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 206 207 if (ret) { 208 /* Failed, get whatever mode it's at now */ 209 if (xcp_mgr->funcs->query_partition_mode) 210 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 211 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 212 else 213 xcp_mgr->mode = curr_mode; 214 215 goto out; 216 } 217 amdgpu_xcp_sysfs_entries_update(xcp_mgr); 218 out: 219 mutex_unlock(&xcp_mgr->xcp_lock); 220 221 return ret; 222 } 223 224 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 225 { 226 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 227 return -EINVAL; 228 229 if (xcp_mgr->mode == mode) 230 return 0; 231 232 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode); 233 } 234 235 int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr) 236 { 237 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 238 return 0; 239 240 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode); 241 } 242 243 static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr) 244 { 245 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 246 return true; 247 248 if (!amdgpu_sriov_vf(xcp_mgr->adev) && 249 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 250 return true; 251 252 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE && 253 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS) 254 return true; 255 256 return false; 257 } 258 259 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 260 { 261 int mode; 262 263 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr)) 264 return xcp_mgr->mode; 265 266 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 267 mutex_lock(&xcp_mgr->xcp_lock); 268 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 269 270 /* First time query for VF, set the mode here */ 271 if (amdgpu_sriov_vf(xcp_mgr->adev) && 272 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 273 xcp_mgr->mode = mode; 274 275 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 276 dev_WARN( 277 xcp_mgr->adev->dev, 278 "Cached partition mode %d not matching with device mode %d", 279 xcp_mgr->mode, mode); 280 281 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 282 mutex_unlock(&xcp_mgr->xcp_lock); 283 284 return mode; 285 } 286 287 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 288 { 289 struct drm_device *p_ddev; 290 struct drm_device *ddev; 291 int i, ret; 292 293 ddev = adev_to_drm(adev); 294 295 /* xcp #0 shares drm device setting with adev */ 296 adev->xcp_mgr->xcp->ddev = ddev; 297 298 for (i = 1; i < MAX_XCP; i++) { 299 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 300 if (ret == -ENOSPC) { 301 dev_warn(adev->dev, 302 "Skip xcp node #%d when out of drm node resource.", i); 303 ret = 0; 304 goto out; 305 } else if (ret) { 306 goto out; 307 } 308 309 /* Redirect all IOCTLs to the primary device */ 310 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 311 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 312 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 313 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 314 p_ddev->render->dev = ddev; 315 p_ddev->primary->dev = ddev; 316 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 317 p_ddev->driver = &amdgpu_partition_driver; 318 adev->xcp_mgr->xcp[i].ddev = p_ddev; 319 320 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]); 321 } 322 ret = 0; 323 out: 324 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr); 325 326 return ret; 327 } 328 329 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 330 int init_num_xcps, 331 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 332 { 333 struct amdgpu_xcp_mgr *xcp_mgr; 334 int i; 335 336 if (!xcp_funcs || !xcp_funcs->get_ip_details) 337 return -EINVAL; 338 339 xcp_mgr = kzalloc_obj(*xcp_mgr); 340 341 if (!xcp_mgr) 342 return -ENOMEM; 343 344 xcp_mgr->adev = adev; 345 xcp_mgr->funcs = xcp_funcs; 346 xcp_mgr->mode = init_mode; 347 mutex_init(&xcp_mgr->xcp_lock); 348 349 if (init_mode != AMDGPU_XCP_MODE_NONE) 350 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 351 352 adev->xcp_mgr = xcp_mgr; 353 for (i = 0; i < MAX_XCP; ++i) 354 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr; 355 356 return amdgpu_xcp_dev_alloc(adev); 357 } 358 359 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 360 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 361 { 362 struct amdgpu_xcp *xcp; 363 int i, id_mask = 0; 364 365 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 366 return -EINVAL; 367 368 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 369 xcp = &xcp_mgr->xcp[i]; 370 if ((xcp->valid) && (xcp->ip[ip].valid) && 371 (xcp->ip[ip].inst_mask & BIT(instance))) 372 id_mask |= BIT(i); 373 } 374 375 if (!id_mask) 376 id_mask = -ENXIO; 377 378 return id_mask; 379 } 380 381 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 382 enum AMDGPU_XCP_IP_BLOCK ip, 383 uint32_t *inst_mask) 384 { 385 if (!xcp->valid || !inst_mask || ip >= AMDGPU_XCP_MAX_BLOCKS || 386 !(xcp->ip[ip].valid)) 387 return -EINVAL; 388 389 *inst_mask = xcp->ip[ip].inst_mask; 390 391 return 0; 392 } 393 394 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 395 const struct pci_device_id *ent) 396 { 397 int i, ret; 398 399 if (!adev->xcp_mgr) 400 return 0; 401 402 for (i = 1; i < MAX_XCP; i++) { 403 if (!adev->xcp_mgr->xcp[i].ddev) 404 break; 405 406 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 407 if (ret) 408 return ret; 409 } 410 411 return 0; 412 } 413 414 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 415 { 416 struct drm_device *p_ddev; 417 int i; 418 419 if (!adev->xcp_mgr) 420 return; 421 422 for (i = 1; i < MAX_XCP; i++) { 423 if (!adev->xcp_mgr->xcp[i].ddev) 424 break; 425 426 p_ddev = adev->xcp_mgr->xcp[i].ddev; 427 drm_dev_unplug(p_ddev); 428 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 429 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 430 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 431 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 432 amdgpu_xcp_drm_dev_free(p_ddev); 433 } 434 } 435 436 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 437 struct amdgpu_fpriv *fpriv, 438 struct drm_file *file_priv) 439 { 440 int i; 441 442 if (!adev->xcp_mgr) 443 return 0; 444 445 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION; 446 for (i = 0; i < MAX_XCP; ++i) { 447 if (!adev->xcp_mgr->xcp[i].ddev) 448 break; 449 450 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 451 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 452 dev_err(adev->dev, "renderD%d partition %d not valid!", 453 file_priv->minor->index, i); 454 return -ENOENT; 455 } 456 dev_dbg(adev->dev, "renderD%d partition %d opened!", 457 file_priv->minor->index, i); 458 fpriv->xcp_id = i; 459 break; 460 } 461 } 462 463 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 : 464 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 465 return 0; 466 } 467 468 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 469 struct amdgpu_ctx_entity *entity) 470 { 471 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 472 struct drm_gpu_scheduler *sched; 473 474 if (!xcp_mgr) 475 return; 476 477 sched = entity->entity.rq->sched; 478 if (drm_sched_wqueue_ready(sched)) { 479 struct amdgpu_ring *ring = to_amdgpu_ring(sched); 480 481 mutex_lock(&xcp_mgr->xcp_lock); 482 if (ring->xcp_id < xcp_mgr->num_xcps && xcp_mgr->xcp[ring->xcp_id].valid) 483 atomic_dec(&xcp_mgr->xcp[ring->xcp_id].ref_cnt); 484 mutex_unlock(&xcp_mgr->xcp_lock); 485 } 486 } 487 488 int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, 489 u32 hw_ip, u32 hw_prio, 490 struct amdgpu_fpriv *fpriv, 491 unsigned int *num_scheds, 492 struct drm_gpu_scheduler ***scheds) 493 { 494 u32 sel_xcp_id; 495 int i; 496 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 497 int r = 0; 498 499 mutex_lock(&xcp_mgr->xcp_lock); 500 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { 501 u32 least_ref_cnt = ~0; 502 503 fpriv->xcp_id = 0; 504 for (i = 0; i < xcp_mgr->num_xcps; i++) { 505 u32 total_ref_cnt; 506 507 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt); 508 if (total_ref_cnt < least_ref_cnt) { 509 fpriv->xcp_id = i; 510 least_ref_cnt = total_ref_cnt; 511 } 512 } 513 } 514 sel_xcp_id = fpriv->xcp_id; 515 516 if (sel_xcp_id >= xcp_mgr->num_xcps || !xcp_mgr->xcp[sel_xcp_id].valid) { 517 dev_err(adev->dev, "Selected partition #%d is not valid.", sel_xcp_id); 518 r = -ENODEV; 519 goto out; 520 } 521 522 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { 523 *num_scheds = 524 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; 525 *scheds = 526 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].sched; 527 atomic_inc(&xcp_mgr->xcp[sel_xcp_id].ref_cnt); 528 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); 529 } else { 530 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); 531 r = -ENOENT; 532 } 533 534 out: 535 mutex_unlock(&xcp_mgr->xcp_lock); 536 return r; 537 } 538 539 static void amdgpu_set_xcp_id(struct amdgpu_device *adev, 540 uint32_t inst_idx, 541 struct amdgpu_ring *ring) 542 { 543 int xcp_id; 544 enum AMDGPU_XCP_IP_BLOCK ip_blk; 545 uint32_t inst_mask; 546 547 ring->xcp_id = AMDGPU_XCP_NO_PARTITION; 548 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 549 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id; 550 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) || 551 (ring->funcs->type == AMDGPU_RING_TYPE_CPER)) 552 return; 553 554 inst_mask = 1 << inst_idx; 555 556 switch (ring->funcs->type) { 557 case AMDGPU_HW_IP_GFX: 558 case AMDGPU_RING_TYPE_COMPUTE: 559 case AMDGPU_RING_TYPE_KIQ: 560 case AMDGPU_RING_TYPE_MES: 561 ip_blk = AMDGPU_XCP_GFX; 562 break; 563 case AMDGPU_RING_TYPE_SDMA: 564 ip_blk = AMDGPU_XCP_SDMA; 565 break; 566 case AMDGPU_RING_TYPE_VCN_ENC: 567 case AMDGPU_RING_TYPE_VCN_JPEG: 568 ip_blk = AMDGPU_XCP_VCN; 569 break; 570 default: 571 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type); 572 return; 573 } 574 575 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) { 576 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) { 577 ring->xcp_id = xcp_id; 578 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name, 579 ring->xcp_id); 580 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 581 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id; 582 break; 583 } 584 } 585 } 586 587 static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, 588 struct amdgpu_ring *ring, 589 unsigned int sel_xcp_id) 590 { 591 unsigned int *num_gpu_sched; 592 593 if (sel_xcp_id >= MAX_XCP || sel_xcp_id == AMDGPU_XCP_NO_PARTITION) 594 return; 595 596 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] 597 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; 598 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] 599 .sched[(*num_gpu_sched)++] = &ring->sched; 600 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d", 601 ring->name, sel_xcp_id, ring->funcs->type, 602 ring->hw_prio, *num_gpu_sched); 603 } 604 605 static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev) 606 { 607 struct amdgpu_ring *ring; 608 int i; 609 610 for (i = 0; i < MAX_XCP; i++) { 611 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0); 612 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched)); 613 } 614 615 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 616 return 0; 617 618 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 619 ring = adev->rings[i]; 620 if (!ring || !ring->sched.ready || ring->no_scheduler) 621 continue; 622 623 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id); 624 625 /* VCN may be shared by two partitions under CPX MODE in certain 626 * configs. 627 */ 628 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || 629 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) && 630 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst)) 631 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1); 632 } 633 634 return 0; 635 } 636 637 int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev) 638 { 639 int i; 640 641 for (i = 0; i < adev->num_rings; i++) { 642 struct amdgpu_ring *ring = adev->rings[i]; 643 644 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || 645 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 646 amdgpu_set_xcp_id(adev, ring->xcc_id, ring); 647 else 648 amdgpu_set_xcp_id(adev, ring->me, ring); 649 } 650 651 return amdgpu_xcp_sched_list_update(adev); 652 } 653 654 void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr) 655 { 656 struct amdgpu_device *adev = xcp_mgr->adev; 657 658 xcp_mgr->supp_xcp_modes = 0; 659 660 switch (NUM_XCC(adev->gfx.xcc_mask)) { 661 case 8: 662 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 663 BIT(AMDGPU_DPX_PARTITION_MODE) | 664 BIT(AMDGPU_QPX_PARTITION_MODE) | 665 BIT(AMDGPU_CPX_PARTITION_MODE); 666 break; 667 case 6: 668 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 669 BIT(AMDGPU_TPX_PARTITION_MODE) | 670 BIT(AMDGPU_CPX_PARTITION_MODE); 671 672 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3) && 673 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4) && 674 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 5, 0)) 675 xcp_mgr->supp_xcp_modes |= BIT(AMDGPU_DPX_PARTITION_MODE); 676 break; 677 case 4: 678 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 679 BIT(AMDGPU_DPX_PARTITION_MODE) | 680 BIT(AMDGPU_CPX_PARTITION_MODE); 681 break; 682 case 2: 683 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 684 BIT(AMDGPU_CPX_PARTITION_MODE); 685 break; 686 case 1: 687 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 688 BIT(AMDGPU_CPX_PARTITION_MODE); 689 break; 690 691 default: 692 break; 693 } 694 } 695 696 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 697 { 698 int ret = 0; 699 700 /* TODO: 701 * Stop user queues and threads, and make sure GPU is empty of work. 702 */ 703 704 if (flags & AMDGPU_XCP_OPS_KFD) 705 ret = amdgpu_amdkfd_prepare_partition_switch(xcp_mgr->adev); 706 707 return ret; 708 } 709 710 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 711 { 712 int ret = 0; 713 714 if (flags & AMDGPU_XCP_OPS_KFD) { 715 amdgpu_amdkfd_device_probe(xcp_mgr->adev); 716 amdgpu_amdkfd_device_init(xcp_mgr->adev); 717 /* If KFD init failed, return failure */ 718 if (!xcp_mgr->adev->kfd.init_complete) 719 ret = -EIO; 720 } 721 722 return ret; 723 } 724 725 /*====================== xcp sysfs - configuration ======================*/ 726 #define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \ 727 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \ 728 struct amdgpu_xcp_res_details *xcp_res, char *buf) \ 729 { \ 730 return sysfs_emit(buf, "%d\n", xcp_res->_name); \ 731 } 732 733 struct amdgpu_xcp_res_sysfs_attribute { 734 struct attribute attr; 735 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf); 736 }; 737 738 #define XCP_CFG_SYSFS_RES_ATTR(_name) \ 739 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \ 740 .attr = { .name = __stringify(_name), .mode = 0400 }, \ 741 .show = amdgpu_xcp_res_sysfs_##_name##_show, \ 742 } 743 744 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst) 745 XCP_CFG_SYSFS_RES_ATTR(num_inst); 746 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared) 747 XCP_CFG_SYSFS_RES_ATTR(num_shared); 748 749 #define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr 750 751 static struct attribute *xcp_cfg_res_sysfs_attrs[] = { 752 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst), 753 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL 754 }; 755 756 static const char *xcp_desc[] = { 757 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 758 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 759 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 760 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 761 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 762 }; 763 764 static const char *nps_desc[] = { 765 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN", 766 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 767 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 768 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 769 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 770 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 771 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 772 }; 773 774 ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs); 775 776 #define to_xcp_attr(x) \ 777 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr) 778 #define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj) 779 780 static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj, 781 struct attribute *attr, char *buf) 782 { 783 struct amdgpu_xcp_res_sysfs_attribute *attribute; 784 struct amdgpu_xcp_res_details *xcp_res; 785 786 attribute = to_xcp_attr(attr); 787 xcp_res = to_xcp_res(kobj); 788 789 if (!attribute->show) 790 return -EIO; 791 792 return attribute->show(xcp_res, buf); 793 } 794 795 static const struct sysfs_ops xcp_cfg_res_sysfs_ops = { 796 .show = xcp_cfg_res_sysfs_attr_show, 797 }; 798 799 static const struct kobj_type xcp_cfg_res_sysfs_ktype = { 800 .sysfs_ops = &xcp_cfg_res_sysfs_ops, 801 .default_groups = xcp_cfg_res_sysfs_groups, 802 }; 803 804 const char *xcp_res_names[] = { 805 [AMDGPU_XCP_RES_XCC] = "xcc", 806 [AMDGPU_XCP_RES_DMA] = "dma", 807 [AMDGPU_XCP_RES_DEC] = "dec", 808 [AMDGPU_XCP_RES_JPEG] = "jpeg", 809 }; 810 811 static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr, 812 int mode, 813 struct amdgpu_xcp_cfg *xcp_cfg) 814 { 815 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info) 816 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg); 817 818 return -EOPNOTSUPP; 819 } 820 821 #define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj) 822 static ssize_t supported_xcp_configs_show(struct kobject *kobj, 823 struct kobj_attribute *attr, char *buf) 824 { 825 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 826 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr; 827 int size = 0, mode; 828 char *sep = ""; 829 830 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes) 831 return sysfs_emit(buf, "Not supported\n"); 832 833 for_each_inst(mode, xcp_mgr->supp_xcp_modes) { 834 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 835 sep = ", "; 836 } 837 838 size += sysfs_emit_at(buf, size, "\n"); 839 840 return size; 841 } 842 843 static ssize_t supported_nps_configs_show(struct kobject *kobj, 844 struct kobj_attribute *attr, char *buf) 845 { 846 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 847 int size = 0, mode; 848 char *sep = ""; 849 850 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes) 851 return sysfs_emit(buf, "Not supported\n"); 852 853 for_each_inst(mode, xcp_cfg->compatible_nps_modes) { 854 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 855 sep = ", "; 856 } 857 858 size += sysfs_emit_at(buf, size, "\n"); 859 860 return size; 861 } 862 863 static ssize_t xcp_config_show(struct kobject *kobj, 864 struct kobj_attribute *attr, char *buf) 865 { 866 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 867 868 return sysfs_emit(buf, "%s\n", 869 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode)); 870 } 871 872 static ssize_t xcp_config_store(struct kobject *kobj, 873 struct kobj_attribute *attr, 874 const char *buf, size_t size) 875 { 876 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 877 int mode, r; 878 879 if (!strncasecmp("SPX", buf, strlen("SPX"))) 880 mode = AMDGPU_SPX_PARTITION_MODE; 881 else if (!strncasecmp("DPX", buf, strlen("DPX"))) 882 mode = AMDGPU_DPX_PARTITION_MODE; 883 else if (!strncasecmp("TPX", buf, strlen("TPX"))) 884 mode = AMDGPU_TPX_PARTITION_MODE; 885 else if (!strncasecmp("QPX", buf, strlen("QPX"))) 886 mode = AMDGPU_QPX_PARTITION_MODE; 887 else if (!strncasecmp("CPX", buf, strlen("CPX"))) 888 mode = AMDGPU_CPX_PARTITION_MODE; 889 else 890 return -EINVAL; 891 892 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 893 894 if (r) 895 return r; 896 897 xcp_cfg->mode = mode; 898 return size; 899 } 900 901 static struct kobj_attribute xcp_cfg_sysfs_mode = 902 __ATTR_RW_MODE(xcp_config, 0644); 903 904 static void xcp_cfg_sysfs_release(struct kobject *kobj) 905 { 906 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 907 908 kfree(xcp_cfg); 909 } 910 911 static const struct kobj_type xcp_cfg_sysfs_ktype = { 912 .release = xcp_cfg_sysfs_release, 913 .sysfs_ops = &kobj_sysfs_ops, 914 }; 915 916 static struct kobj_attribute supp_part_sysfs_mode = 917 __ATTR_RO(supported_xcp_configs); 918 919 static struct kobj_attribute supp_nps_sysfs_mode = 920 __ATTR_RO(supported_nps_configs); 921 922 static const struct attribute *xcp_attrs[] = { 923 &supp_part_sysfs_mode.attr, 924 &xcp_cfg_sysfs_mode.attr, 925 NULL, 926 }; 927 928 static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) 929 { 930 struct amdgpu_xcp_res_details *xcp_res; 931 struct amdgpu_xcp_cfg *xcp_cfg; 932 int i, r, rid, mode; 933 934 if (!adev->xcp_mgr) 935 return; 936 937 xcp_cfg = kzalloc_obj(*xcp_cfg); 938 if (!xcp_cfg) 939 return; 940 xcp_cfg->xcp_mgr = adev->xcp_mgr; 941 942 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype, 943 &adev->dev->kobj, "compute_partition_config"); 944 if (r) 945 goto err1; 946 947 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs); 948 if (r) 949 goto err1; 950 951 if (adev->gmc.supported_nps_modes != 0) { 952 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 953 if (r) { 954 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 955 goto err1; 956 } 957 } 958 959 mode = (xcp_cfg->xcp_mgr->mode == 960 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ? 961 AMDGPU_SPX_PARTITION_MODE : 962 xcp_cfg->xcp_mgr->mode; 963 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 964 if (r) { 965 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 966 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 967 goto err1; 968 } 969 970 xcp_cfg->mode = mode; 971 for (i = 0; i < xcp_cfg->num_res; i++) { 972 xcp_res = &xcp_cfg->xcp_res[i]; 973 rid = xcp_res->id; 974 r = kobject_init_and_add(&xcp_res->kobj, 975 &xcp_cfg_res_sysfs_ktype, 976 &xcp_cfg->kobj, "%s", 977 xcp_res_names[rid]); 978 if (r) { 979 kobject_put(&xcp_res->kobj); 980 goto err; 981 } 982 } 983 984 adev->xcp_mgr->xcp_cfg = xcp_cfg; 985 return; 986 err: 987 while (i--) { 988 xcp_res = &xcp_cfg->xcp_res[i]; 989 kobject_put(&xcp_res->kobj); 990 } 991 992 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 993 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 994 err1: 995 kobject_put(&xcp_cfg->kobj); 996 } 997 998 static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev) 999 { 1000 struct amdgpu_xcp_res_details *xcp_res; 1001 struct amdgpu_xcp_cfg *xcp_cfg; 1002 int i; 1003 1004 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg) 1005 return; 1006 1007 xcp_cfg = adev->xcp_mgr->xcp_cfg; 1008 for (i = 0; i < xcp_cfg->num_res; i++) { 1009 xcp_res = &xcp_cfg->xcp_res[i]; 1010 kobject_put(&xcp_res->kobj); 1011 } 1012 1013 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 1014 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 1015 kobject_put(&xcp_cfg->kobj); 1016 } 1017 1018 /*====================== xcp sysfs - data entries ======================*/ 1019 1020 #define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj) 1021 1022 static ssize_t xcp_metrics_show(struct kobject *kobj, 1023 struct kobj_attribute *attr, char *buf) 1024 { 1025 struct amdgpu_xcp *xcp = to_xcp(kobj); 1026 struct amdgpu_xcp_mgr *xcp_mgr; 1027 ssize_t size; 1028 1029 xcp_mgr = xcp->xcp_mgr; 1030 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL); 1031 if (size <= 0) 1032 return size; 1033 1034 if (size > PAGE_SIZE) 1035 return -ENOSPC; 1036 1037 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf); 1038 } 1039 1040 static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj, 1041 struct attribute *attr, int n) 1042 { 1043 struct amdgpu_xcp *xcp = to_xcp(kobj); 1044 1045 if (!xcp || !xcp->valid) 1046 return 0; 1047 1048 return attr->mode; 1049 } 1050 1051 static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics); 1052 1053 static struct attribute *amdgpu_xcp_attrs[] = { 1054 &xcp_sysfs_metrics.attr, 1055 NULL, 1056 }; 1057 1058 static const struct attribute_group amdgpu_xcp_attrs_group = { 1059 .attrs = amdgpu_xcp_attrs, 1060 .is_visible = amdgpu_xcp_attrs_is_visible 1061 }; 1062 1063 static const struct kobj_type xcp_sysfs_ktype = { 1064 .sysfs_ops = &kobj_sysfs_ops, 1065 }; 1066 1067 bool amdgpu_xcp_is_primary(struct amdgpu_xcp *xcp) 1068 { 1069 return xcp->ddev == adev_to_drm(xcp->xcp_mgr->adev); 1070 } 1071 1072 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n) 1073 { 1074 struct amdgpu_xcp *xcp; 1075 1076 for (n--; n >= 0; n--) { 1077 xcp = &xcp_mgr->xcp[n]; 1078 if (!xcp->ddev || !xcp->valid) 1079 continue; 1080 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1081 kobject_put(&xcp->kobj); 1082 } 1083 } 1084 1085 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr) 1086 { 1087 struct amdgpu_xcp *xcp; 1088 int i, r; 1089 1090 for (i = 0; i < MAX_XCP; i++) { 1091 /* Redirect all IOCTLs to the primary device */ 1092 xcp = &xcp_mgr->xcp[i]; 1093 if (!xcp->ddev) 1094 break; 1095 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype, 1096 &xcp->ddev->dev->kobj, "xcp"); 1097 if (r) 1098 goto out; 1099 1100 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1101 if (r) 1102 goto out; 1103 } 1104 1105 return; 1106 out: 1107 kobject_put(&xcp->kobj); 1108 } 1109 1110 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr) 1111 { 1112 struct amdgpu_xcp *xcp; 1113 int i; 1114 1115 for (i = 0; i < MAX_XCP; i++) { 1116 /* Redirect all IOCTLs to the primary device */ 1117 xcp = &xcp_mgr->xcp[i]; 1118 if (!xcp->ddev) 1119 continue; 1120 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1121 amdgpu_ualink_xcp_sysfs_update(xcp); 1122 } 1123 1124 return; 1125 } 1126 1127 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev) 1128 { 1129 if (!adev->xcp_mgr) 1130 return; 1131 1132 amdgpu_xcp_cfg_sysfs_init(adev); 1133 1134 return; 1135 } 1136 1137 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev) 1138 { 1139 if (!adev->xcp_mgr) 1140 return; 1141 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP); 1142 amdgpu_xcp_cfg_sysfs_fini(adev); 1143 } 1144