1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 27 #include <drm/drm_drv.h> 28 #include "../amdxcp/amdgpu_xcp_drv.h" 29 30 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr); 31 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr); 32 33 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 34 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 35 { 36 int (*run_func)(void *handle, uint32_t inst_mask); 37 int ret = 0; 38 39 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 40 return 0; 41 42 run_func = NULL; 43 44 switch (xcp_state) { 45 case AMDGPU_XCP_PREPARE_SUSPEND: 46 run_func = xcp_ip->ip_funcs->prepare_suspend; 47 break; 48 case AMDGPU_XCP_SUSPEND: 49 run_func = xcp_ip->ip_funcs->suspend; 50 break; 51 case AMDGPU_XCP_PREPARE_RESUME: 52 run_func = xcp_ip->ip_funcs->prepare_resume; 53 break; 54 case AMDGPU_XCP_RESUME: 55 run_func = xcp_ip->ip_funcs->resume; 56 break; 57 } 58 59 if (run_func) 60 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 61 62 return ret; 63 } 64 65 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 66 int state) 67 { 68 struct amdgpu_xcp_ip *xcp_ip; 69 struct amdgpu_xcp *xcp; 70 int i, ret; 71 72 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 73 return -EINVAL; 74 75 xcp = &xcp_mgr->xcp[xcp_id]; 76 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 77 xcp_ip = &xcp->ip[i]; 78 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 79 if (ret) 80 break; 81 } 82 83 return ret; 84 } 85 86 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 87 { 88 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 89 AMDGPU_XCP_PREPARE_SUSPEND); 90 } 91 92 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 93 { 94 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 95 } 96 97 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 98 { 99 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 100 AMDGPU_XCP_PREPARE_RESUME); 101 } 102 103 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 104 { 105 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 106 } 107 108 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 109 struct amdgpu_xcp_ip *ip) 110 { 111 struct amdgpu_xcp *xcp; 112 113 if (!ip) 114 return; 115 116 xcp = &xcp_mgr->xcp[xcp_id]; 117 xcp->ip[ip->ip_id] = *ip; 118 xcp->ip[ip->ip_id].valid = true; 119 120 xcp->valid = true; 121 } 122 123 static void __amdgpu_xcp_set_unique_id(struct amdgpu_xcp_mgr *xcp_mgr, 124 int xcp_id) 125 { 126 struct amdgpu_xcp *xcp = &xcp_mgr->xcp[xcp_id]; 127 struct amdgpu_device *adev = xcp_mgr->adev; 128 uint32_t inst_mask; 129 uint64_t uid; 130 int i; 131 132 if (!amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask) && 133 inst_mask) { 134 i = GET_INST(GC, (ffs(inst_mask) - 1)); 135 uid = amdgpu_device_get_uid(xcp_mgr->adev->uid_info, 136 AMDGPU_UID_TYPE_XCD, i); 137 if (uid) 138 xcp->unique_id = uid; 139 } 140 } 141 142 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 143 { 144 struct amdgpu_device *adev = xcp_mgr->adev; 145 struct amdgpu_xcp_ip ip; 146 uint8_t mem_id; 147 int i, j, ret; 148 149 if (!num_xcps || num_xcps > MAX_XCP) 150 return -EINVAL; 151 152 xcp_mgr->mode = mode; 153 154 for (i = 0; i < MAX_XCP; ++i) 155 xcp_mgr->xcp[i].valid = false; 156 157 /* This is needed for figuring out memory id of xcp */ 158 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 159 160 for (i = 0; i < num_xcps; ++i) { 161 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 162 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 163 &ip); 164 if (ret) 165 continue; 166 167 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 168 } 169 170 xcp_mgr->xcp[i].id = i; 171 172 if (xcp_mgr->funcs->get_xcp_mem_id) { 173 ret = xcp_mgr->funcs->get_xcp_mem_id( 174 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 175 if (ret) 176 continue; 177 else 178 xcp_mgr->xcp[i].mem_id = mem_id; 179 } 180 __amdgpu_xcp_set_unique_id(xcp_mgr, i); 181 } 182 183 xcp_mgr->num_xcps = num_xcps; 184 amdgpu_xcp_update_partition_sched_list(adev); 185 186 return 0; 187 } 188 189 static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, 190 int mode) 191 { 192 int ret, curr_mode, num_xcps = 0; 193 194 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 195 return 0; 196 197 mutex_lock(&xcp_mgr->xcp_lock); 198 199 curr_mode = xcp_mgr->mode; 200 /* State set to transient mode */ 201 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 202 203 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 204 205 if (ret) { 206 /* Failed, get whatever mode it's at now */ 207 if (xcp_mgr->funcs->query_partition_mode) 208 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 209 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 210 else 211 xcp_mgr->mode = curr_mode; 212 213 goto out; 214 } 215 amdgpu_xcp_sysfs_entries_update(xcp_mgr); 216 out: 217 mutex_unlock(&xcp_mgr->xcp_lock); 218 219 return ret; 220 } 221 222 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 223 { 224 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 225 return -EINVAL; 226 227 if (xcp_mgr->mode == mode) 228 return 0; 229 230 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode); 231 } 232 233 int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr) 234 { 235 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 236 return 0; 237 238 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode); 239 } 240 241 static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr) 242 { 243 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 244 return true; 245 246 if (!amdgpu_sriov_vf(xcp_mgr->adev) && 247 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 248 return true; 249 250 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE && 251 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS) 252 return true; 253 254 return false; 255 } 256 257 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 258 { 259 int mode; 260 261 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr)) 262 return xcp_mgr->mode; 263 264 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 265 mutex_lock(&xcp_mgr->xcp_lock); 266 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 267 268 /* First time query for VF, set the mode here */ 269 if (amdgpu_sriov_vf(xcp_mgr->adev) && 270 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 271 xcp_mgr->mode = mode; 272 273 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 274 dev_WARN( 275 xcp_mgr->adev->dev, 276 "Cached partition mode %d not matching with device mode %d", 277 xcp_mgr->mode, mode); 278 279 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 280 mutex_unlock(&xcp_mgr->xcp_lock); 281 282 return mode; 283 } 284 285 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 286 { 287 struct drm_device *p_ddev; 288 struct drm_device *ddev; 289 int i, ret; 290 291 ddev = adev_to_drm(adev); 292 293 /* xcp #0 shares drm device setting with adev */ 294 adev->xcp_mgr->xcp->ddev = ddev; 295 296 for (i = 1; i < MAX_XCP; i++) { 297 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 298 if (ret == -ENOSPC) { 299 dev_warn(adev->dev, 300 "Skip xcp node #%d when out of drm node resource.", i); 301 ret = 0; 302 goto out; 303 } else if (ret) { 304 goto out; 305 } 306 307 /* Redirect all IOCTLs to the primary device */ 308 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 309 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 310 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 311 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 312 p_ddev->render->dev = ddev; 313 p_ddev->primary->dev = ddev; 314 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 315 p_ddev->driver = &amdgpu_partition_driver; 316 adev->xcp_mgr->xcp[i].ddev = p_ddev; 317 318 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]); 319 } 320 ret = 0; 321 out: 322 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr); 323 324 return ret; 325 } 326 327 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 328 int init_num_xcps, 329 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 330 { 331 struct amdgpu_xcp_mgr *xcp_mgr; 332 int i; 333 334 if (!xcp_funcs || !xcp_funcs->get_ip_details) 335 return -EINVAL; 336 337 xcp_mgr = kzalloc(sizeof(*xcp_mgr), GFP_KERNEL); 338 339 if (!xcp_mgr) 340 return -ENOMEM; 341 342 xcp_mgr->adev = adev; 343 xcp_mgr->funcs = xcp_funcs; 344 xcp_mgr->mode = init_mode; 345 mutex_init(&xcp_mgr->xcp_lock); 346 347 if (init_mode != AMDGPU_XCP_MODE_NONE) 348 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 349 350 adev->xcp_mgr = xcp_mgr; 351 for (i = 0; i < MAX_XCP; ++i) 352 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr; 353 354 return amdgpu_xcp_dev_alloc(adev); 355 } 356 357 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 358 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 359 { 360 struct amdgpu_xcp *xcp; 361 int i, id_mask = 0; 362 363 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 364 return -EINVAL; 365 366 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 367 xcp = &xcp_mgr->xcp[i]; 368 if ((xcp->valid) && (xcp->ip[ip].valid) && 369 (xcp->ip[ip].inst_mask & BIT(instance))) 370 id_mask |= BIT(i); 371 } 372 373 if (!id_mask) 374 id_mask = -ENXIO; 375 376 return id_mask; 377 } 378 379 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 380 enum AMDGPU_XCP_IP_BLOCK ip, 381 uint32_t *inst_mask) 382 { 383 if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid)) 384 return -EINVAL; 385 386 *inst_mask = xcp->ip[ip].inst_mask; 387 388 return 0; 389 } 390 391 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 392 const struct pci_device_id *ent) 393 { 394 int i, ret; 395 396 if (!adev->xcp_mgr) 397 return 0; 398 399 for (i = 1; i < MAX_XCP; i++) { 400 if (!adev->xcp_mgr->xcp[i].ddev) 401 break; 402 403 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 404 if (ret) 405 return ret; 406 } 407 408 return 0; 409 } 410 411 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 412 { 413 struct drm_device *p_ddev; 414 int i; 415 416 if (!adev->xcp_mgr) 417 return; 418 419 for (i = 1; i < MAX_XCP; i++) { 420 if (!adev->xcp_mgr->xcp[i].ddev) 421 break; 422 423 p_ddev = adev->xcp_mgr->xcp[i].ddev; 424 drm_dev_unplug(p_ddev); 425 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 426 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 427 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 428 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 429 amdgpu_xcp_drm_dev_free(p_ddev); 430 } 431 } 432 433 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 434 struct amdgpu_fpriv *fpriv, 435 struct drm_file *file_priv) 436 { 437 int i; 438 439 if (!adev->xcp_mgr) 440 return 0; 441 442 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION; 443 for (i = 0; i < MAX_XCP; ++i) { 444 if (!adev->xcp_mgr->xcp[i].ddev) 445 break; 446 447 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 448 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 449 dev_err(adev->dev, "renderD%d partition %d not valid!", 450 file_priv->minor->index, i); 451 return -ENOENT; 452 } 453 dev_dbg(adev->dev, "renderD%d partition %d opened!", 454 file_priv->minor->index, i); 455 fpriv->xcp_id = i; 456 break; 457 } 458 } 459 460 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 : 461 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 462 return 0; 463 } 464 465 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 466 struct amdgpu_ctx_entity *entity) 467 { 468 struct drm_gpu_scheduler *sched; 469 struct amdgpu_ring *ring; 470 471 if (!adev->xcp_mgr) 472 return; 473 474 sched = entity->entity.rq->sched; 475 if (drm_sched_wqueue_ready(sched)) { 476 ring = to_amdgpu_ring(entity->entity.rq->sched); 477 atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); 478 } 479 } 480 481 int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, 482 u32 hw_ip, u32 hw_prio, 483 struct amdgpu_fpriv *fpriv, 484 unsigned int *num_scheds, 485 struct drm_gpu_scheduler ***scheds) 486 { 487 u32 sel_xcp_id; 488 int i; 489 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 490 491 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { 492 u32 least_ref_cnt = ~0; 493 494 fpriv->xcp_id = 0; 495 for (i = 0; i < xcp_mgr->num_xcps; i++) { 496 u32 total_ref_cnt; 497 498 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt); 499 if (total_ref_cnt < least_ref_cnt) { 500 fpriv->xcp_id = i; 501 least_ref_cnt = total_ref_cnt; 502 } 503 } 504 } 505 sel_xcp_id = fpriv->xcp_id; 506 507 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { 508 *num_scheds = 509 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; 510 *scheds = 511 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched; 512 atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt); 513 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); 514 } else { 515 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); 516 return -ENOENT; 517 } 518 519 return 0; 520 } 521 522 static void amdgpu_set_xcp_id(struct amdgpu_device *adev, 523 uint32_t inst_idx, 524 struct amdgpu_ring *ring) 525 { 526 int xcp_id; 527 enum AMDGPU_XCP_IP_BLOCK ip_blk; 528 uint32_t inst_mask; 529 530 ring->xcp_id = AMDGPU_XCP_NO_PARTITION; 531 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 532 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id; 533 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) || 534 (ring->funcs->type == AMDGPU_RING_TYPE_CPER)) 535 return; 536 537 inst_mask = 1 << inst_idx; 538 539 switch (ring->funcs->type) { 540 case AMDGPU_HW_IP_GFX: 541 case AMDGPU_RING_TYPE_COMPUTE: 542 case AMDGPU_RING_TYPE_KIQ: 543 ip_blk = AMDGPU_XCP_GFX; 544 break; 545 case AMDGPU_RING_TYPE_SDMA: 546 ip_blk = AMDGPU_XCP_SDMA; 547 break; 548 case AMDGPU_RING_TYPE_VCN_ENC: 549 case AMDGPU_RING_TYPE_VCN_JPEG: 550 ip_blk = AMDGPU_XCP_VCN; 551 break; 552 default: 553 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type); 554 return; 555 } 556 557 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) { 558 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) { 559 ring->xcp_id = xcp_id; 560 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name, 561 ring->xcp_id); 562 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 563 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id; 564 break; 565 } 566 } 567 } 568 569 static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, 570 struct amdgpu_ring *ring, 571 unsigned int sel_xcp_id) 572 { 573 unsigned int *num_gpu_sched; 574 575 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] 576 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; 577 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] 578 .sched[(*num_gpu_sched)++] = &ring->sched; 579 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d", 580 ring->name, sel_xcp_id, ring->funcs->type, 581 ring->hw_prio, *num_gpu_sched); 582 } 583 584 static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev) 585 { 586 struct amdgpu_ring *ring; 587 int i; 588 589 for (i = 0; i < MAX_XCP; i++) { 590 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0); 591 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched)); 592 } 593 594 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 595 return 0; 596 597 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 598 ring = adev->rings[i]; 599 if (!ring || !ring->sched.ready || ring->no_scheduler) 600 continue; 601 602 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id); 603 604 /* VCN may be shared by two partitions under CPX MODE in certain 605 * configs. 606 */ 607 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || 608 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) && 609 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst)) 610 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1); 611 } 612 613 return 0; 614 } 615 616 int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev) 617 { 618 int i; 619 620 for (i = 0; i < adev->num_rings; i++) { 621 struct amdgpu_ring *ring = adev->rings[i]; 622 623 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || 624 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 625 amdgpu_set_xcp_id(adev, ring->xcc_id, ring); 626 else 627 amdgpu_set_xcp_id(adev, ring->me, ring); 628 } 629 630 return amdgpu_xcp_sched_list_update(adev); 631 } 632 633 void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr) 634 { 635 struct amdgpu_device *adev = xcp_mgr->adev; 636 637 xcp_mgr->supp_xcp_modes = 0; 638 639 switch (NUM_XCC(adev->gfx.xcc_mask)) { 640 case 8: 641 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 642 BIT(AMDGPU_DPX_PARTITION_MODE) | 643 BIT(AMDGPU_QPX_PARTITION_MODE) | 644 BIT(AMDGPU_CPX_PARTITION_MODE); 645 break; 646 case 6: 647 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 648 BIT(AMDGPU_TPX_PARTITION_MODE) | 649 BIT(AMDGPU_CPX_PARTITION_MODE); 650 break; 651 case 4: 652 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 653 BIT(AMDGPU_DPX_PARTITION_MODE) | 654 BIT(AMDGPU_CPX_PARTITION_MODE); 655 break; 656 case 2: 657 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 658 BIT(AMDGPU_CPX_PARTITION_MODE); 659 break; 660 case 1: 661 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 662 BIT(AMDGPU_CPX_PARTITION_MODE); 663 break; 664 665 default: 666 break; 667 } 668 } 669 670 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 671 { 672 /* TODO: 673 * Stop user queues and threads, and make sure GPU is empty of work. 674 */ 675 676 if (flags & AMDGPU_XCP_OPS_KFD) 677 amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev); 678 679 return 0; 680 } 681 682 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 683 { 684 int ret = 0; 685 686 if (flags & AMDGPU_XCP_OPS_KFD) { 687 amdgpu_amdkfd_device_probe(xcp_mgr->adev); 688 amdgpu_amdkfd_device_init(xcp_mgr->adev); 689 /* If KFD init failed, return failure */ 690 if (!xcp_mgr->adev->kfd.init_complete) 691 ret = -EIO; 692 } 693 694 return ret; 695 } 696 697 /*====================== xcp sysfs - configuration ======================*/ 698 #define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \ 699 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \ 700 struct amdgpu_xcp_res_details *xcp_res, char *buf) \ 701 { \ 702 return sysfs_emit(buf, "%d\n", xcp_res->_name); \ 703 } 704 705 struct amdgpu_xcp_res_sysfs_attribute { 706 struct attribute attr; 707 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf); 708 }; 709 710 #define XCP_CFG_SYSFS_RES_ATTR(_name) \ 711 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \ 712 .attr = { .name = __stringify(_name), .mode = 0400 }, \ 713 .show = amdgpu_xcp_res_sysfs_##_name##_show, \ 714 } 715 716 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst) 717 XCP_CFG_SYSFS_RES_ATTR(num_inst); 718 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared) 719 XCP_CFG_SYSFS_RES_ATTR(num_shared); 720 721 #define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr 722 723 static struct attribute *xcp_cfg_res_sysfs_attrs[] = { 724 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst), 725 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL 726 }; 727 728 static const char *xcp_desc[] = { 729 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 730 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 731 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 732 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 733 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 734 }; 735 736 static const char *nps_desc[] = { 737 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN", 738 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 739 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 740 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 741 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 742 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 743 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 744 }; 745 746 ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs); 747 748 #define to_xcp_attr(x) \ 749 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr) 750 #define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj) 751 752 static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj, 753 struct attribute *attr, char *buf) 754 { 755 struct amdgpu_xcp_res_sysfs_attribute *attribute; 756 struct amdgpu_xcp_res_details *xcp_res; 757 758 attribute = to_xcp_attr(attr); 759 xcp_res = to_xcp_res(kobj); 760 761 if (!attribute->show) 762 return -EIO; 763 764 return attribute->show(xcp_res, buf); 765 } 766 767 static const struct sysfs_ops xcp_cfg_res_sysfs_ops = { 768 .show = xcp_cfg_res_sysfs_attr_show, 769 }; 770 771 static const struct kobj_type xcp_cfg_res_sysfs_ktype = { 772 .sysfs_ops = &xcp_cfg_res_sysfs_ops, 773 .default_groups = xcp_cfg_res_sysfs_groups, 774 }; 775 776 const char *xcp_res_names[] = { 777 [AMDGPU_XCP_RES_XCC] = "xcc", 778 [AMDGPU_XCP_RES_DMA] = "dma", 779 [AMDGPU_XCP_RES_DEC] = "dec", 780 [AMDGPU_XCP_RES_JPEG] = "jpeg", 781 }; 782 783 static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr, 784 int mode, 785 struct amdgpu_xcp_cfg *xcp_cfg) 786 { 787 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info) 788 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg); 789 790 return -EOPNOTSUPP; 791 } 792 793 #define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj) 794 static ssize_t supported_xcp_configs_show(struct kobject *kobj, 795 struct kobj_attribute *attr, char *buf) 796 { 797 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 798 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr; 799 int size = 0, mode; 800 char *sep = ""; 801 802 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes) 803 return sysfs_emit(buf, "Not supported\n"); 804 805 for_each_inst(mode, xcp_mgr->supp_xcp_modes) { 806 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 807 sep = ", "; 808 } 809 810 size += sysfs_emit_at(buf, size, "\n"); 811 812 return size; 813 } 814 815 static ssize_t supported_nps_configs_show(struct kobject *kobj, 816 struct kobj_attribute *attr, char *buf) 817 { 818 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 819 int size = 0, mode; 820 char *sep = ""; 821 822 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes) 823 return sysfs_emit(buf, "Not supported\n"); 824 825 for_each_inst(mode, xcp_cfg->compatible_nps_modes) { 826 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 827 sep = ", "; 828 } 829 830 size += sysfs_emit_at(buf, size, "\n"); 831 832 return size; 833 } 834 835 static ssize_t xcp_config_show(struct kobject *kobj, 836 struct kobj_attribute *attr, char *buf) 837 { 838 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 839 840 return sysfs_emit(buf, "%s\n", 841 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode)); 842 } 843 844 static ssize_t xcp_config_store(struct kobject *kobj, 845 struct kobj_attribute *attr, 846 const char *buf, size_t size) 847 { 848 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 849 int mode, r; 850 851 if (!strncasecmp("SPX", buf, strlen("SPX"))) 852 mode = AMDGPU_SPX_PARTITION_MODE; 853 else if (!strncasecmp("DPX", buf, strlen("DPX"))) 854 mode = AMDGPU_DPX_PARTITION_MODE; 855 else if (!strncasecmp("TPX", buf, strlen("TPX"))) 856 mode = AMDGPU_TPX_PARTITION_MODE; 857 else if (!strncasecmp("QPX", buf, strlen("QPX"))) 858 mode = AMDGPU_QPX_PARTITION_MODE; 859 else if (!strncasecmp("CPX", buf, strlen("CPX"))) 860 mode = AMDGPU_CPX_PARTITION_MODE; 861 else 862 return -EINVAL; 863 864 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 865 866 if (r) 867 return r; 868 869 xcp_cfg->mode = mode; 870 return size; 871 } 872 873 static struct kobj_attribute xcp_cfg_sysfs_mode = 874 __ATTR_RW_MODE(xcp_config, 0644); 875 876 static void xcp_cfg_sysfs_release(struct kobject *kobj) 877 { 878 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 879 880 kfree(xcp_cfg); 881 } 882 883 static const struct kobj_type xcp_cfg_sysfs_ktype = { 884 .release = xcp_cfg_sysfs_release, 885 .sysfs_ops = &kobj_sysfs_ops, 886 }; 887 888 static struct kobj_attribute supp_part_sysfs_mode = 889 __ATTR_RO(supported_xcp_configs); 890 891 static struct kobj_attribute supp_nps_sysfs_mode = 892 __ATTR_RO(supported_nps_configs); 893 894 static const struct attribute *xcp_attrs[] = { 895 &supp_part_sysfs_mode.attr, 896 &xcp_cfg_sysfs_mode.attr, 897 NULL, 898 }; 899 900 static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) 901 { 902 struct amdgpu_xcp_res_details *xcp_res; 903 struct amdgpu_xcp_cfg *xcp_cfg; 904 int i, r, j, rid, mode; 905 906 if (!adev->xcp_mgr) 907 return; 908 909 xcp_cfg = kzalloc(sizeof(*xcp_cfg), GFP_KERNEL); 910 if (!xcp_cfg) 911 return; 912 xcp_cfg->xcp_mgr = adev->xcp_mgr; 913 914 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype, 915 &adev->dev->kobj, "compute_partition_config"); 916 if (r) 917 goto err1; 918 919 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs); 920 if (r) 921 goto err1; 922 923 if (adev->gmc.supported_nps_modes != 0) { 924 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 925 if (r) { 926 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 927 goto err1; 928 } 929 } 930 931 mode = (xcp_cfg->xcp_mgr->mode == 932 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ? 933 AMDGPU_SPX_PARTITION_MODE : 934 xcp_cfg->xcp_mgr->mode; 935 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 936 if (r) { 937 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 938 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 939 goto err1; 940 } 941 942 xcp_cfg->mode = mode; 943 for (i = 0; i < xcp_cfg->num_res; i++) { 944 xcp_res = &xcp_cfg->xcp_res[i]; 945 rid = xcp_res->id; 946 r = kobject_init_and_add(&xcp_res->kobj, 947 &xcp_cfg_res_sysfs_ktype, 948 &xcp_cfg->kobj, "%s", 949 xcp_res_names[rid]); 950 if (r) 951 goto err; 952 } 953 954 adev->xcp_mgr->xcp_cfg = xcp_cfg; 955 return; 956 err: 957 for (j = 0; j < i; j++) { 958 xcp_res = &xcp_cfg->xcp_res[i]; 959 kobject_put(&xcp_res->kobj); 960 } 961 962 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 963 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 964 err1: 965 kobject_put(&xcp_cfg->kobj); 966 } 967 968 static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev) 969 { 970 struct amdgpu_xcp_res_details *xcp_res; 971 struct amdgpu_xcp_cfg *xcp_cfg; 972 int i; 973 974 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg) 975 return; 976 977 xcp_cfg = adev->xcp_mgr->xcp_cfg; 978 for (i = 0; i < xcp_cfg->num_res; i++) { 979 xcp_res = &xcp_cfg->xcp_res[i]; 980 kobject_put(&xcp_res->kobj); 981 } 982 983 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 984 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 985 kobject_put(&xcp_cfg->kobj); 986 } 987 988 /*====================== xcp sysfs - data entries ======================*/ 989 990 #define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj) 991 992 static ssize_t xcp_metrics_show(struct kobject *kobj, 993 struct kobj_attribute *attr, char *buf) 994 { 995 struct amdgpu_xcp *xcp = to_xcp(kobj); 996 struct amdgpu_xcp_mgr *xcp_mgr; 997 ssize_t size; 998 999 xcp_mgr = xcp->xcp_mgr; 1000 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL); 1001 if (size <= 0) 1002 return size; 1003 1004 if (size > PAGE_SIZE) 1005 return -ENOSPC; 1006 1007 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf); 1008 } 1009 1010 static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj, 1011 struct attribute *attr, int n) 1012 { 1013 struct amdgpu_xcp *xcp = to_xcp(kobj); 1014 1015 if (!xcp || !xcp->valid) 1016 return 0; 1017 1018 return attr->mode; 1019 } 1020 1021 static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics); 1022 1023 static struct attribute *amdgpu_xcp_attrs[] = { 1024 &xcp_sysfs_metrics.attr, 1025 NULL, 1026 }; 1027 1028 static const struct attribute_group amdgpu_xcp_attrs_group = { 1029 .attrs = amdgpu_xcp_attrs, 1030 .is_visible = amdgpu_xcp_attrs_is_visible 1031 }; 1032 1033 static const struct kobj_type xcp_sysfs_ktype = { 1034 .sysfs_ops = &kobj_sysfs_ops, 1035 }; 1036 1037 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n) 1038 { 1039 struct amdgpu_xcp *xcp; 1040 1041 for (n--; n >= 0; n--) { 1042 xcp = &xcp_mgr->xcp[n]; 1043 if (!xcp->ddev || !xcp->valid) 1044 continue; 1045 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1046 kobject_put(&xcp->kobj); 1047 } 1048 } 1049 1050 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr) 1051 { 1052 struct amdgpu_xcp *xcp; 1053 int i, r; 1054 1055 for (i = 0; i < MAX_XCP; i++) { 1056 /* Redirect all IOCTLs to the primary device */ 1057 xcp = &xcp_mgr->xcp[i]; 1058 if (!xcp->ddev) 1059 break; 1060 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype, 1061 &xcp->ddev->dev->kobj, "xcp"); 1062 if (r) 1063 goto out; 1064 1065 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1066 if (r) 1067 goto out; 1068 } 1069 1070 return; 1071 out: 1072 kobject_put(&xcp->kobj); 1073 } 1074 1075 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr) 1076 { 1077 struct amdgpu_xcp *xcp; 1078 int i; 1079 1080 for (i = 0; i < MAX_XCP; i++) { 1081 /* Redirect all IOCTLs to the primary device */ 1082 xcp = &xcp_mgr->xcp[i]; 1083 if (!xcp->ddev) 1084 continue; 1085 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1086 } 1087 1088 return; 1089 } 1090 1091 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev) 1092 { 1093 if (!adev->xcp_mgr) 1094 return; 1095 1096 amdgpu_xcp_cfg_sysfs_init(adev); 1097 1098 return; 1099 } 1100 1101 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev) 1102 { 1103 if (!adev->xcp_mgr) 1104 return; 1105 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP); 1106 amdgpu_xcp_cfg_sysfs_fini(adev); 1107 } 1108