1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 27 #include <drm/drm_drv.h> 28 #include "../amdxcp/amdgpu_xcp_drv.h" 29 30 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr); 31 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr); 32 33 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 34 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 35 { 36 int (*run_func)(void *handle, uint32_t inst_mask); 37 int ret = 0; 38 39 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 40 return 0; 41 42 run_func = NULL; 43 44 switch (xcp_state) { 45 case AMDGPU_XCP_PREPARE_SUSPEND: 46 run_func = xcp_ip->ip_funcs->prepare_suspend; 47 break; 48 case AMDGPU_XCP_SUSPEND: 49 run_func = xcp_ip->ip_funcs->suspend; 50 break; 51 case AMDGPU_XCP_PREPARE_RESUME: 52 run_func = xcp_ip->ip_funcs->prepare_resume; 53 break; 54 case AMDGPU_XCP_RESUME: 55 run_func = xcp_ip->ip_funcs->resume; 56 break; 57 } 58 59 if (run_func) 60 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 61 62 return ret; 63 } 64 65 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 66 int state) 67 { 68 struct amdgpu_xcp_ip *xcp_ip; 69 struct amdgpu_xcp *xcp; 70 int i, ret; 71 72 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 73 return -EINVAL; 74 75 xcp = &xcp_mgr->xcp[xcp_id]; 76 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 77 xcp_ip = &xcp->ip[i]; 78 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 79 if (ret) 80 break; 81 } 82 83 return ret; 84 } 85 86 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 87 { 88 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 89 AMDGPU_XCP_PREPARE_SUSPEND); 90 } 91 92 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 93 { 94 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 95 } 96 97 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 98 { 99 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 100 AMDGPU_XCP_PREPARE_RESUME); 101 } 102 103 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 104 { 105 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 106 } 107 108 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 109 struct amdgpu_xcp_ip *ip) 110 { 111 struct amdgpu_xcp *xcp; 112 113 if (!ip) 114 return; 115 116 xcp = &xcp_mgr->xcp[xcp_id]; 117 xcp->ip[ip->ip_id] = *ip; 118 xcp->ip[ip->ip_id].valid = true; 119 120 xcp->valid = true; 121 } 122 123 static void __amdgpu_xcp_set_unique_id(struct amdgpu_xcp_mgr *xcp_mgr, 124 int xcp_id) 125 { 126 struct amdgpu_xcp *xcp = &xcp_mgr->xcp[xcp_id]; 127 struct amdgpu_device *adev = xcp_mgr->adev; 128 uint32_t inst_mask; 129 uint64_t uid; 130 int i; 131 132 if (!amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask) && 133 inst_mask) { 134 i = GET_INST(GC, (ffs(inst_mask) - 1)); 135 uid = amdgpu_device_get_uid(xcp_mgr->adev->uid_info, 136 AMDGPU_UID_TYPE_XCD, i); 137 if (uid) 138 xcp->unique_id = uid; 139 } 140 } 141 142 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 143 { 144 struct amdgpu_device *adev = xcp_mgr->adev; 145 struct amdgpu_xcp_ip ip; 146 uint8_t mem_id; 147 int i, j, ret; 148 149 if (!num_xcps || num_xcps > MAX_XCP) 150 return -EINVAL; 151 152 xcp_mgr->mode = mode; 153 154 for (i = 0; i < MAX_XCP; ++i) 155 xcp_mgr->xcp[i].valid = false; 156 157 /* This is needed for figuring out memory id of xcp */ 158 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 159 160 for (i = 0; i < num_xcps; ++i) { 161 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 162 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 163 &ip); 164 if (ret) 165 continue; 166 167 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 168 } 169 170 xcp_mgr->xcp[i].id = i; 171 172 if (xcp_mgr->funcs->get_xcp_mem_id) { 173 ret = xcp_mgr->funcs->get_xcp_mem_id( 174 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 175 if (ret) 176 continue; 177 else 178 xcp_mgr->xcp[i].mem_id = mem_id; 179 } 180 __amdgpu_xcp_set_unique_id(xcp_mgr, i); 181 } 182 183 xcp_mgr->num_xcps = num_xcps; 184 xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN; 185 amdgpu_xcp_update_partition_sched_list(adev); 186 187 return 0; 188 } 189 190 static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, 191 int mode) 192 { 193 int ret, curr_mode, num_xcps = 0; 194 195 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 196 return 0; 197 198 mutex_lock(&xcp_mgr->xcp_lock); 199 200 curr_mode = xcp_mgr->mode; 201 /* State set to transient mode */ 202 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 203 204 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 205 206 if (ret) { 207 /* Failed, get whatever mode it's at now */ 208 if (xcp_mgr->funcs->query_partition_mode) 209 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 210 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 211 else 212 xcp_mgr->mode = curr_mode; 213 214 goto out; 215 } 216 amdgpu_xcp_sysfs_entries_update(xcp_mgr); 217 out: 218 mutex_unlock(&xcp_mgr->xcp_lock); 219 220 return ret; 221 } 222 223 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 224 { 225 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 226 return -EINVAL; 227 228 if (xcp_mgr->mode == mode) 229 return 0; 230 231 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode); 232 } 233 234 int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr) 235 { 236 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 237 return 0; 238 239 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode); 240 } 241 242 static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr) 243 { 244 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 245 return true; 246 247 if (!amdgpu_sriov_vf(xcp_mgr->adev) && 248 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 249 return true; 250 251 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE && 252 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS) 253 return true; 254 255 return false; 256 } 257 258 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 259 { 260 int mode; 261 262 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr)) 263 return xcp_mgr->mode; 264 265 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 266 mutex_lock(&xcp_mgr->xcp_lock); 267 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 268 269 /* First time query for VF, set the mode here */ 270 if (amdgpu_sriov_vf(xcp_mgr->adev) && 271 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 272 xcp_mgr->mode = mode; 273 274 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 275 dev_WARN( 276 xcp_mgr->adev->dev, 277 "Cached partition mode %d not matching with device mode %d", 278 xcp_mgr->mode, mode); 279 280 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 281 mutex_unlock(&xcp_mgr->xcp_lock); 282 283 return mode; 284 } 285 286 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 287 { 288 struct drm_device *p_ddev; 289 struct drm_device *ddev; 290 int i, ret; 291 292 ddev = adev_to_drm(adev); 293 294 /* xcp #0 shares drm device setting with adev */ 295 adev->xcp_mgr->xcp->ddev = ddev; 296 297 for (i = 1; i < MAX_XCP; i++) { 298 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 299 if (ret == -ENOSPC) { 300 dev_warn(adev->dev, 301 "Skip xcp node #%d when out of drm node resource.", i); 302 ret = 0; 303 goto out; 304 } else if (ret) { 305 goto out; 306 } 307 308 /* Redirect all IOCTLs to the primary device */ 309 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 310 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 311 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 312 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 313 p_ddev->render->dev = ddev; 314 p_ddev->primary->dev = ddev; 315 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 316 p_ddev->driver = &amdgpu_partition_driver; 317 adev->xcp_mgr->xcp[i].ddev = p_ddev; 318 319 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]); 320 } 321 ret = 0; 322 out: 323 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr); 324 325 return ret; 326 } 327 328 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 329 int init_num_xcps, 330 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 331 { 332 struct amdgpu_xcp_mgr *xcp_mgr; 333 int i; 334 335 if (!xcp_funcs || !xcp_funcs->get_ip_details) 336 return -EINVAL; 337 338 xcp_mgr = kzalloc_obj(*xcp_mgr); 339 340 if (!xcp_mgr) 341 return -ENOMEM; 342 343 xcp_mgr->adev = adev; 344 xcp_mgr->funcs = xcp_funcs; 345 xcp_mgr->mode = init_mode; 346 mutex_init(&xcp_mgr->xcp_lock); 347 348 if (init_mode != AMDGPU_XCP_MODE_NONE) 349 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 350 351 adev->xcp_mgr = xcp_mgr; 352 for (i = 0; i < MAX_XCP; ++i) 353 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr; 354 355 return amdgpu_xcp_dev_alloc(adev); 356 } 357 358 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 359 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 360 { 361 struct amdgpu_xcp *xcp; 362 int i, id_mask = 0; 363 364 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 365 return -EINVAL; 366 367 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 368 xcp = &xcp_mgr->xcp[i]; 369 if ((xcp->valid) && (xcp->ip[ip].valid) && 370 (xcp->ip[ip].inst_mask & BIT(instance))) 371 id_mask |= BIT(i); 372 } 373 374 if (!id_mask) 375 id_mask = -ENXIO; 376 377 return id_mask; 378 } 379 380 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 381 enum AMDGPU_XCP_IP_BLOCK ip, 382 uint32_t *inst_mask) 383 { 384 if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid)) 385 return -EINVAL; 386 387 *inst_mask = xcp->ip[ip].inst_mask; 388 389 return 0; 390 } 391 392 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 393 const struct pci_device_id *ent) 394 { 395 int i, ret; 396 397 if (!adev->xcp_mgr) 398 return 0; 399 400 for (i = 1; i < MAX_XCP; i++) { 401 if (!adev->xcp_mgr->xcp[i].ddev) 402 break; 403 404 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 405 if (ret) 406 return ret; 407 } 408 409 return 0; 410 } 411 412 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 413 { 414 struct drm_device *p_ddev; 415 int i; 416 417 if (!adev->xcp_mgr) 418 return; 419 420 for (i = 1; i < MAX_XCP; i++) { 421 if (!adev->xcp_mgr->xcp[i].ddev) 422 break; 423 424 p_ddev = adev->xcp_mgr->xcp[i].ddev; 425 drm_dev_unplug(p_ddev); 426 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 427 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 428 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 429 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 430 amdgpu_xcp_drm_dev_free(p_ddev); 431 } 432 } 433 434 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 435 struct amdgpu_fpriv *fpriv, 436 struct drm_file *file_priv) 437 { 438 int i; 439 440 if (!adev->xcp_mgr) 441 return 0; 442 443 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION; 444 for (i = 0; i < MAX_XCP; ++i) { 445 if (!adev->xcp_mgr->xcp[i].ddev) 446 break; 447 448 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 449 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 450 dev_err(adev->dev, "renderD%d partition %d not valid!", 451 file_priv->minor->index, i); 452 return -ENOENT; 453 } 454 dev_dbg(adev->dev, "renderD%d partition %d opened!", 455 file_priv->minor->index, i); 456 fpriv->xcp_id = i; 457 break; 458 } 459 } 460 461 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 : 462 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 463 return 0; 464 } 465 466 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 467 struct amdgpu_ctx_entity *entity) 468 { 469 struct drm_gpu_scheduler *sched; 470 struct amdgpu_ring *ring; 471 472 if (!adev->xcp_mgr) 473 return; 474 475 sched = entity->entity.rq->sched; 476 if (drm_sched_wqueue_ready(sched)) { 477 ring = to_amdgpu_ring(entity->entity.rq->sched); 478 atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); 479 } 480 } 481 482 int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, 483 u32 hw_ip, u32 hw_prio, 484 struct amdgpu_fpriv *fpriv, 485 unsigned int *num_scheds, 486 struct drm_gpu_scheduler ***scheds) 487 { 488 u32 sel_xcp_id; 489 int i; 490 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 491 492 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { 493 u32 least_ref_cnt = ~0; 494 495 fpriv->xcp_id = 0; 496 for (i = 0; i < xcp_mgr->num_xcps; i++) { 497 u32 total_ref_cnt; 498 499 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt); 500 if (total_ref_cnt < least_ref_cnt) { 501 fpriv->xcp_id = i; 502 least_ref_cnt = total_ref_cnt; 503 } 504 } 505 } 506 sel_xcp_id = fpriv->xcp_id; 507 508 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { 509 *num_scheds = 510 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; 511 *scheds = 512 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched; 513 atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt); 514 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); 515 } else { 516 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); 517 return -ENOENT; 518 } 519 520 return 0; 521 } 522 523 static void amdgpu_set_xcp_id(struct amdgpu_device *adev, 524 uint32_t inst_idx, 525 struct amdgpu_ring *ring) 526 { 527 int xcp_id; 528 enum AMDGPU_XCP_IP_BLOCK ip_blk; 529 uint32_t inst_mask; 530 531 ring->xcp_id = AMDGPU_XCP_NO_PARTITION; 532 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 533 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id; 534 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) || 535 (ring->funcs->type == AMDGPU_RING_TYPE_CPER)) 536 return; 537 538 inst_mask = 1 << inst_idx; 539 540 switch (ring->funcs->type) { 541 case AMDGPU_HW_IP_GFX: 542 case AMDGPU_RING_TYPE_COMPUTE: 543 case AMDGPU_RING_TYPE_KIQ: 544 case AMDGPU_RING_TYPE_MES: 545 ip_blk = AMDGPU_XCP_GFX; 546 break; 547 case AMDGPU_RING_TYPE_SDMA: 548 ip_blk = AMDGPU_XCP_SDMA; 549 break; 550 case AMDGPU_RING_TYPE_VCN_ENC: 551 case AMDGPU_RING_TYPE_VCN_JPEG: 552 ip_blk = AMDGPU_XCP_VCN; 553 break; 554 default: 555 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type); 556 return; 557 } 558 559 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) { 560 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) { 561 ring->xcp_id = xcp_id; 562 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name, 563 ring->xcp_id); 564 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 565 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id; 566 break; 567 } 568 } 569 } 570 571 static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, 572 struct amdgpu_ring *ring, 573 unsigned int sel_xcp_id) 574 { 575 unsigned int *num_gpu_sched; 576 577 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] 578 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; 579 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] 580 .sched[(*num_gpu_sched)++] = &ring->sched; 581 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d", 582 ring->name, sel_xcp_id, ring->funcs->type, 583 ring->hw_prio, *num_gpu_sched); 584 } 585 586 static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev) 587 { 588 struct amdgpu_ring *ring; 589 int i; 590 591 for (i = 0; i < MAX_XCP; i++) { 592 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0); 593 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched)); 594 } 595 596 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 597 return 0; 598 599 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 600 ring = adev->rings[i]; 601 if (!ring || !ring->sched.ready || ring->no_scheduler) 602 continue; 603 604 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id); 605 606 /* VCN may be shared by two partitions under CPX MODE in certain 607 * configs. 608 */ 609 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || 610 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) && 611 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst)) 612 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1); 613 } 614 615 return 0; 616 } 617 618 int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev) 619 { 620 int i; 621 622 for (i = 0; i < adev->num_rings; i++) { 623 struct amdgpu_ring *ring = adev->rings[i]; 624 625 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || 626 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 627 amdgpu_set_xcp_id(adev, ring->xcc_id, ring); 628 else 629 amdgpu_set_xcp_id(adev, ring->me, ring); 630 } 631 632 return amdgpu_xcp_sched_list_update(adev); 633 } 634 635 void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr) 636 { 637 struct amdgpu_device *adev = xcp_mgr->adev; 638 639 xcp_mgr->supp_xcp_modes = 0; 640 641 switch (NUM_XCC(adev->gfx.xcc_mask)) { 642 case 8: 643 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 644 BIT(AMDGPU_DPX_PARTITION_MODE) | 645 BIT(AMDGPU_QPX_PARTITION_MODE) | 646 BIT(AMDGPU_CPX_PARTITION_MODE); 647 break; 648 case 6: 649 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 650 BIT(AMDGPU_TPX_PARTITION_MODE) | 651 BIT(AMDGPU_CPX_PARTITION_MODE); 652 break; 653 case 4: 654 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 655 BIT(AMDGPU_DPX_PARTITION_MODE) | 656 BIT(AMDGPU_CPX_PARTITION_MODE); 657 break; 658 case 2: 659 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 660 BIT(AMDGPU_CPX_PARTITION_MODE); 661 break; 662 case 1: 663 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 664 BIT(AMDGPU_CPX_PARTITION_MODE); 665 break; 666 667 default: 668 break; 669 } 670 } 671 672 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 673 { 674 /* TODO: 675 * Stop user queues and threads, and make sure GPU is empty of work. 676 */ 677 678 if (flags & AMDGPU_XCP_OPS_KFD) 679 amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev); 680 681 return 0; 682 } 683 684 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 685 { 686 int ret = 0; 687 688 if (flags & AMDGPU_XCP_OPS_KFD) { 689 amdgpu_amdkfd_device_probe(xcp_mgr->adev); 690 amdgpu_amdkfd_device_init(xcp_mgr->adev); 691 /* If KFD init failed, return failure */ 692 if (!xcp_mgr->adev->kfd.init_complete) 693 ret = -EIO; 694 } 695 696 return ret; 697 } 698 699 /*====================== xcp sysfs - configuration ======================*/ 700 #define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \ 701 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \ 702 struct amdgpu_xcp_res_details *xcp_res, char *buf) \ 703 { \ 704 return sysfs_emit(buf, "%d\n", xcp_res->_name); \ 705 } 706 707 struct amdgpu_xcp_res_sysfs_attribute { 708 struct attribute attr; 709 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf); 710 }; 711 712 #define XCP_CFG_SYSFS_RES_ATTR(_name) \ 713 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \ 714 .attr = { .name = __stringify(_name), .mode = 0400 }, \ 715 .show = amdgpu_xcp_res_sysfs_##_name##_show, \ 716 } 717 718 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst) 719 XCP_CFG_SYSFS_RES_ATTR(num_inst); 720 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared) 721 XCP_CFG_SYSFS_RES_ATTR(num_shared); 722 723 #define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr 724 725 static struct attribute *xcp_cfg_res_sysfs_attrs[] = { 726 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst), 727 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL 728 }; 729 730 static const char *xcp_desc[] = { 731 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 732 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 733 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 734 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 735 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 736 }; 737 738 static const char *nps_desc[] = { 739 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN", 740 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 741 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 742 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 743 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 744 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 745 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 746 }; 747 748 ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs); 749 750 #define to_xcp_attr(x) \ 751 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr) 752 #define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj) 753 754 static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj, 755 struct attribute *attr, char *buf) 756 { 757 struct amdgpu_xcp_res_sysfs_attribute *attribute; 758 struct amdgpu_xcp_res_details *xcp_res; 759 760 attribute = to_xcp_attr(attr); 761 xcp_res = to_xcp_res(kobj); 762 763 if (!attribute->show) 764 return -EIO; 765 766 return attribute->show(xcp_res, buf); 767 } 768 769 static const struct sysfs_ops xcp_cfg_res_sysfs_ops = { 770 .show = xcp_cfg_res_sysfs_attr_show, 771 }; 772 773 static const struct kobj_type xcp_cfg_res_sysfs_ktype = { 774 .sysfs_ops = &xcp_cfg_res_sysfs_ops, 775 .default_groups = xcp_cfg_res_sysfs_groups, 776 }; 777 778 const char *xcp_res_names[] = { 779 [AMDGPU_XCP_RES_XCC] = "xcc", 780 [AMDGPU_XCP_RES_DMA] = "dma", 781 [AMDGPU_XCP_RES_DEC] = "dec", 782 [AMDGPU_XCP_RES_JPEG] = "jpeg", 783 }; 784 785 static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr, 786 int mode, 787 struct amdgpu_xcp_cfg *xcp_cfg) 788 { 789 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info) 790 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg); 791 792 return -EOPNOTSUPP; 793 } 794 795 #define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj) 796 static ssize_t supported_xcp_configs_show(struct kobject *kobj, 797 struct kobj_attribute *attr, char *buf) 798 { 799 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 800 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr; 801 int size = 0, mode; 802 char *sep = ""; 803 804 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes) 805 return sysfs_emit(buf, "Not supported\n"); 806 807 for_each_inst(mode, xcp_mgr->supp_xcp_modes) { 808 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 809 sep = ", "; 810 } 811 812 size += sysfs_emit_at(buf, size, "\n"); 813 814 return size; 815 } 816 817 static ssize_t supported_nps_configs_show(struct kobject *kobj, 818 struct kobj_attribute *attr, char *buf) 819 { 820 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 821 int size = 0, mode; 822 char *sep = ""; 823 824 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes) 825 return sysfs_emit(buf, "Not supported\n"); 826 827 for_each_inst(mode, xcp_cfg->compatible_nps_modes) { 828 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 829 sep = ", "; 830 } 831 832 size += sysfs_emit_at(buf, size, "\n"); 833 834 return size; 835 } 836 837 static ssize_t xcp_config_show(struct kobject *kobj, 838 struct kobj_attribute *attr, char *buf) 839 { 840 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 841 842 return sysfs_emit(buf, "%s\n", 843 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode)); 844 } 845 846 static ssize_t xcp_config_store(struct kobject *kobj, 847 struct kobj_attribute *attr, 848 const char *buf, size_t size) 849 { 850 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 851 int mode, r; 852 853 if (!strncasecmp("SPX", buf, strlen("SPX"))) 854 mode = AMDGPU_SPX_PARTITION_MODE; 855 else if (!strncasecmp("DPX", buf, strlen("DPX"))) 856 mode = AMDGPU_DPX_PARTITION_MODE; 857 else if (!strncasecmp("TPX", buf, strlen("TPX"))) 858 mode = AMDGPU_TPX_PARTITION_MODE; 859 else if (!strncasecmp("QPX", buf, strlen("QPX"))) 860 mode = AMDGPU_QPX_PARTITION_MODE; 861 else if (!strncasecmp("CPX", buf, strlen("CPX"))) 862 mode = AMDGPU_CPX_PARTITION_MODE; 863 else 864 return -EINVAL; 865 866 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 867 868 if (r) 869 return r; 870 871 xcp_cfg->mode = mode; 872 return size; 873 } 874 875 static struct kobj_attribute xcp_cfg_sysfs_mode = 876 __ATTR_RW_MODE(xcp_config, 0644); 877 878 static void xcp_cfg_sysfs_release(struct kobject *kobj) 879 { 880 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 881 882 kfree(xcp_cfg); 883 } 884 885 static const struct kobj_type xcp_cfg_sysfs_ktype = { 886 .release = xcp_cfg_sysfs_release, 887 .sysfs_ops = &kobj_sysfs_ops, 888 }; 889 890 static struct kobj_attribute supp_part_sysfs_mode = 891 __ATTR_RO(supported_xcp_configs); 892 893 static struct kobj_attribute supp_nps_sysfs_mode = 894 __ATTR_RO(supported_nps_configs); 895 896 static const struct attribute *xcp_attrs[] = { 897 &supp_part_sysfs_mode.attr, 898 &xcp_cfg_sysfs_mode.attr, 899 NULL, 900 }; 901 902 static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) 903 { 904 struct amdgpu_xcp_res_details *xcp_res; 905 struct amdgpu_xcp_cfg *xcp_cfg; 906 int i, r, j, rid, mode; 907 908 if (!adev->xcp_mgr) 909 return; 910 911 xcp_cfg = kzalloc_obj(*xcp_cfg); 912 if (!xcp_cfg) 913 return; 914 xcp_cfg->xcp_mgr = adev->xcp_mgr; 915 916 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype, 917 &adev->dev->kobj, "compute_partition_config"); 918 if (r) 919 goto err1; 920 921 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs); 922 if (r) 923 goto err1; 924 925 if (adev->gmc.supported_nps_modes != 0) { 926 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 927 if (r) { 928 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 929 goto err1; 930 } 931 } 932 933 mode = (xcp_cfg->xcp_mgr->mode == 934 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ? 935 AMDGPU_SPX_PARTITION_MODE : 936 xcp_cfg->xcp_mgr->mode; 937 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 938 if (r) { 939 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 940 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 941 goto err1; 942 } 943 944 xcp_cfg->mode = mode; 945 for (i = 0; i < xcp_cfg->num_res; i++) { 946 xcp_res = &xcp_cfg->xcp_res[i]; 947 rid = xcp_res->id; 948 r = kobject_init_and_add(&xcp_res->kobj, 949 &xcp_cfg_res_sysfs_ktype, 950 &xcp_cfg->kobj, "%s", 951 xcp_res_names[rid]); 952 if (r) 953 goto err; 954 } 955 956 adev->xcp_mgr->xcp_cfg = xcp_cfg; 957 return; 958 err: 959 for (j = 0; j < i; j++) { 960 xcp_res = &xcp_cfg->xcp_res[i]; 961 kobject_put(&xcp_res->kobj); 962 } 963 964 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 965 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 966 err1: 967 kobject_put(&xcp_cfg->kobj); 968 } 969 970 static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev) 971 { 972 struct amdgpu_xcp_res_details *xcp_res; 973 struct amdgpu_xcp_cfg *xcp_cfg; 974 int i; 975 976 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg) 977 return; 978 979 xcp_cfg = adev->xcp_mgr->xcp_cfg; 980 for (i = 0; i < xcp_cfg->num_res; i++) { 981 xcp_res = &xcp_cfg->xcp_res[i]; 982 kobject_put(&xcp_res->kobj); 983 } 984 985 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 986 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 987 kobject_put(&xcp_cfg->kobj); 988 } 989 990 /*====================== xcp sysfs - data entries ======================*/ 991 992 #define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj) 993 994 static ssize_t xcp_metrics_show(struct kobject *kobj, 995 struct kobj_attribute *attr, char *buf) 996 { 997 struct amdgpu_xcp *xcp = to_xcp(kobj); 998 struct amdgpu_xcp_mgr *xcp_mgr; 999 ssize_t size; 1000 1001 xcp_mgr = xcp->xcp_mgr; 1002 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL); 1003 if (size <= 0) 1004 return size; 1005 1006 if (size > PAGE_SIZE) 1007 return -ENOSPC; 1008 1009 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf); 1010 } 1011 1012 static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj, 1013 struct attribute *attr, int n) 1014 { 1015 struct amdgpu_xcp *xcp = to_xcp(kobj); 1016 1017 if (!xcp || !xcp->valid) 1018 return 0; 1019 1020 return attr->mode; 1021 } 1022 1023 static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics); 1024 1025 static struct attribute *amdgpu_xcp_attrs[] = { 1026 &xcp_sysfs_metrics.attr, 1027 NULL, 1028 }; 1029 1030 static const struct attribute_group amdgpu_xcp_attrs_group = { 1031 .attrs = amdgpu_xcp_attrs, 1032 .is_visible = amdgpu_xcp_attrs_is_visible 1033 }; 1034 1035 static const struct kobj_type xcp_sysfs_ktype = { 1036 .sysfs_ops = &kobj_sysfs_ops, 1037 }; 1038 1039 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n) 1040 { 1041 struct amdgpu_xcp *xcp; 1042 1043 for (n--; n >= 0; n--) { 1044 xcp = &xcp_mgr->xcp[n]; 1045 if (!xcp->ddev || !xcp->valid) 1046 continue; 1047 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1048 kobject_put(&xcp->kobj); 1049 } 1050 } 1051 1052 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr) 1053 { 1054 struct amdgpu_xcp *xcp; 1055 int i, r; 1056 1057 for (i = 0; i < MAX_XCP; i++) { 1058 /* Redirect all IOCTLs to the primary device */ 1059 xcp = &xcp_mgr->xcp[i]; 1060 if (!xcp->ddev) 1061 break; 1062 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype, 1063 &xcp->ddev->dev->kobj, "xcp"); 1064 if (r) 1065 goto out; 1066 1067 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1068 if (r) 1069 goto out; 1070 } 1071 1072 return; 1073 out: 1074 kobject_put(&xcp->kobj); 1075 } 1076 1077 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr) 1078 { 1079 struct amdgpu_xcp *xcp; 1080 int i; 1081 1082 for (i = 0; i < MAX_XCP; i++) { 1083 /* Redirect all IOCTLs to the primary device */ 1084 xcp = &xcp_mgr->xcp[i]; 1085 if (!xcp->ddev) 1086 continue; 1087 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1088 } 1089 1090 return; 1091 } 1092 1093 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev) 1094 { 1095 if (!adev->xcp_mgr) 1096 return; 1097 1098 amdgpu_xcp_cfg_sysfs_init(adev); 1099 1100 return; 1101 } 1102 1103 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev) 1104 { 1105 if (!adev->xcp_mgr) 1106 return; 1107 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP); 1108 amdgpu_xcp_cfg_sysfs_fini(adev); 1109 } 1110