1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 27 #include <drm/drm_drv.h> 28 #include "../amdxcp/amdgpu_xcp_drv.h" 29 30 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr); 31 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr); 32 33 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 34 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 35 { 36 int (*run_func)(void *handle, uint32_t inst_mask); 37 int ret = 0; 38 39 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 40 return 0; 41 42 run_func = NULL; 43 44 switch (xcp_state) { 45 case AMDGPU_XCP_PREPARE_SUSPEND: 46 run_func = xcp_ip->ip_funcs->prepare_suspend; 47 break; 48 case AMDGPU_XCP_SUSPEND: 49 run_func = xcp_ip->ip_funcs->suspend; 50 break; 51 case AMDGPU_XCP_PREPARE_RESUME: 52 run_func = xcp_ip->ip_funcs->prepare_resume; 53 break; 54 case AMDGPU_XCP_RESUME: 55 run_func = xcp_ip->ip_funcs->resume; 56 break; 57 } 58 59 if (run_func) 60 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 61 62 return ret; 63 } 64 65 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 66 int state) 67 { 68 struct amdgpu_xcp_ip *xcp_ip; 69 struct amdgpu_xcp *xcp; 70 int i, ret; 71 72 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 73 return -EINVAL; 74 75 xcp = &xcp_mgr->xcp[xcp_id]; 76 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 77 xcp_ip = &xcp->ip[i]; 78 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 79 if (ret) 80 break; 81 } 82 83 return ret; 84 } 85 86 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 87 { 88 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 89 AMDGPU_XCP_PREPARE_SUSPEND); 90 } 91 92 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 93 { 94 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 95 } 96 97 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 98 { 99 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 100 AMDGPU_XCP_PREPARE_RESUME); 101 } 102 103 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 104 { 105 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 106 } 107 108 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 109 struct amdgpu_xcp_ip *ip) 110 { 111 struct amdgpu_xcp *xcp; 112 113 if (!ip) 114 return; 115 116 xcp = &xcp_mgr->xcp[xcp_id]; 117 xcp->ip[ip->ip_id] = *ip; 118 xcp->ip[ip->ip_id].valid = true; 119 120 xcp->valid = true; 121 } 122 123 static void __amdgpu_xcp_set_unique_id(struct amdgpu_xcp_mgr *xcp_mgr, 124 int xcp_id) 125 { 126 struct amdgpu_xcp *xcp = &xcp_mgr->xcp[xcp_id]; 127 struct amdgpu_device *adev = xcp_mgr->adev; 128 uint32_t inst_mask; 129 uint64_t uid; 130 int i; 131 132 if (!amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask) && 133 inst_mask) { 134 i = GET_INST(GC, (ffs(inst_mask) - 1)); 135 uid = amdgpu_device_get_uid(xcp_mgr->adev->uid_info, 136 AMDGPU_UID_TYPE_XCD, i); 137 if (uid) 138 xcp->unique_id = uid; 139 } 140 } 141 142 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 143 { 144 struct amdgpu_device *adev = xcp_mgr->adev; 145 struct amdgpu_xcp_ip ip; 146 uint8_t mem_id; 147 int i, j, ret; 148 149 if (!num_xcps || num_xcps > MAX_XCP) 150 return -EINVAL; 151 152 xcp_mgr->mode = mode; 153 154 for (i = 0; i < MAX_XCP; ++i) 155 xcp_mgr->xcp[i].valid = false; 156 157 /* This is needed for figuring out memory id of xcp */ 158 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 159 160 for (i = 0; i < num_xcps; ++i) { 161 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 162 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 163 &ip); 164 if (ret) 165 continue; 166 167 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 168 } 169 170 xcp_mgr->xcp[i].id = i; 171 172 if (xcp_mgr->funcs->get_xcp_mem_id) { 173 ret = xcp_mgr->funcs->get_xcp_mem_id( 174 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 175 if (ret) 176 continue; 177 else 178 xcp_mgr->xcp[i].mem_id = mem_id; 179 } 180 __amdgpu_xcp_set_unique_id(xcp_mgr, i); 181 } 182 183 xcp_mgr->num_xcps = num_xcps; 184 xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN; 185 amdgpu_xcp_update_partition_sched_list(adev); 186 187 return 0; 188 } 189 190 static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, 191 int mode) 192 { 193 int ret, curr_mode, num_xcps = 0; 194 195 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 196 return 0; 197 198 mutex_lock(&xcp_mgr->xcp_lock); 199 200 curr_mode = xcp_mgr->mode; 201 /* State set to transient mode */ 202 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 203 204 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 205 206 if (ret) { 207 /* Failed, get whatever mode it's at now */ 208 if (xcp_mgr->funcs->query_partition_mode) 209 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 210 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 211 else 212 xcp_mgr->mode = curr_mode; 213 214 goto out; 215 } 216 amdgpu_xcp_sysfs_entries_update(xcp_mgr); 217 out: 218 mutex_unlock(&xcp_mgr->xcp_lock); 219 220 return ret; 221 } 222 223 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 224 { 225 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 226 return -EINVAL; 227 228 if (xcp_mgr->mode == mode) 229 return 0; 230 231 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode); 232 } 233 234 int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr) 235 { 236 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 237 return 0; 238 239 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode); 240 } 241 242 static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr) 243 { 244 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 245 return true; 246 247 if (!amdgpu_sriov_vf(xcp_mgr->adev) && 248 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 249 return true; 250 251 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE && 252 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS) 253 return true; 254 255 return false; 256 } 257 258 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 259 { 260 int mode; 261 262 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr)) 263 return xcp_mgr->mode; 264 265 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 266 mutex_lock(&xcp_mgr->xcp_lock); 267 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 268 269 /* First time query for VF, set the mode here */ 270 if (amdgpu_sriov_vf(xcp_mgr->adev) && 271 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 272 xcp_mgr->mode = mode; 273 274 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 275 dev_WARN( 276 xcp_mgr->adev->dev, 277 "Cached partition mode %d not matching with device mode %d", 278 xcp_mgr->mode, mode); 279 280 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 281 mutex_unlock(&xcp_mgr->xcp_lock); 282 283 return mode; 284 } 285 286 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 287 { 288 struct drm_device *p_ddev; 289 struct drm_device *ddev; 290 int i, ret; 291 292 ddev = adev_to_drm(adev); 293 294 /* xcp #0 shares drm device setting with adev */ 295 adev->xcp_mgr->xcp->ddev = ddev; 296 297 for (i = 1; i < MAX_XCP; i++) { 298 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 299 if (ret == -ENOSPC) { 300 dev_warn(adev->dev, 301 "Skip xcp node #%d when out of drm node resource.", i); 302 ret = 0; 303 goto out; 304 } else if (ret) { 305 goto out; 306 } 307 308 /* Redirect all IOCTLs to the primary device */ 309 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 310 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 311 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 312 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 313 p_ddev->render->dev = ddev; 314 p_ddev->primary->dev = ddev; 315 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 316 p_ddev->driver = &amdgpu_partition_driver; 317 adev->xcp_mgr->xcp[i].ddev = p_ddev; 318 319 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]); 320 } 321 ret = 0; 322 out: 323 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr); 324 325 return ret; 326 } 327 328 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 329 int init_num_xcps, 330 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 331 { 332 struct amdgpu_xcp_mgr *xcp_mgr; 333 int i; 334 335 if (!xcp_funcs || !xcp_funcs->get_ip_details) 336 return -EINVAL; 337 338 xcp_mgr = kzalloc_obj(*xcp_mgr); 339 340 if (!xcp_mgr) 341 return -ENOMEM; 342 343 xcp_mgr->adev = adev; 344 xcp_mgr->funcs = xcp_funcs; 345 xcp_mgr->mode = init_mode; 346 mutex_init(&xcp_mgr->xcp_lock); 347 348 if (init_mode != AMDGPU_XCP_MODE_NONE) 349 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 350 351 adev->xcp_mgr = xcp_mgr; 352 for (i = 0; i < MAX_XCP; ++i) 353 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr; 354 355 return amdgpu_xcp_dev_alloc(adev); 356 } 357 358 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 359 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 360 { 361 struct amdgpu_xcp *xcp; 362 int i, id_mask = 0; 363 364 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 365 return -EINVAL; 366 367 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 368 xcp = &xcp_mgr->xcp[i]; 369 if ((xcp->valid) && (xcp->ip[ip].valid) && 370 (xcp->ip[ip].inst_mask & BIT(instance))) 371 id_mask |= BIT(i); 372 } 373 374 if (!id_mask) 375 id_mask = -ENXIO; 376 377 return id_mask; 378 } 379 380 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 381 enum AMDGPU_XCP_IP_BLOCK ip, 382 uint32_t *inst_mask) 383 { 384 if (!xcp->valid || !inst_mask || ip >= AMDGPU_XCP_MAX_BLOCKS || 385 !(xcp->ip[ip].valid)) 386 return -EINVAL; 387 388 *inst_mask = xcp->ip[ip].inst_mask; 389 390 return 0; 391 } 392 393 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 394 const struct pci_device_id *ent) 395 { 396 int i, ret; 397 398 if (!adev->xcp_mgr) 399 return 0; 400 401 for (i = 1; i < MAX_XCP; i++) { 402 if (!adev->xcp_mgr->xcp[i].ddev) 403 break; 404 405 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 406 if (ret) 407 return ret; 408 } 409 410 return 0; 411 } 412 413 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 414 { 415 struct drm_device *p_ddev; 416 int i; 417 418 if (!adev->xcp_mgr) 419 return; 420 421 for (i = 1; i < MAX_XCP; i++) { 422 if (!adev->xcp_mgr->xcp[i].ddev) 423 break; 424 425 p_ddev = adev->xcp_mgr->xcp[i].ddev; 426 drm_dev_unplug(p_ddev); 427 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 428 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 429 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 430 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 431 amdgpu_xcp_drm_dev_free(p_ddev); 432 } 433 } 434 435 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 436 struct amdgpu_fpriv *fpriv, 437 struct drm_file *file_priv) 438 { 439 int i; 440 441 if (!adev->xcp_mgr) 442 return 0; 443 444 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION; 445 for (i = 0; i < MAX_XCP; ++i) { 446 if (!adev->xcp_mgr->xcp[i].ddev) 447 break; 448 449 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 450 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 451 dev_err(adev->dev, "renderD%d partition %d not valid!", 452 file_priv->minor->index, i); 453 return -ENOENT; 454 } 455 dev_dbg(adev->dev, "renderD%d partition %d opened!", 456 file_priv->minor->index, i); 457 fpriv->xcp_id = i; 458 break; 459 } 460 } 461 462 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 : 463 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 464 return 0; 465 } 466 467 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 468 struct amdgpu_ctx_entity *entity) 469 { 470 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 471 struct drm_gpu_scheduler *sched; 472 473 if (!xcp_mgr) 474 return; 475 476 sched = entity->entity.rq->sched; 477 if (drm_sched_wqueue_ready(sched)) { 478 struct amdgpu_ring *ring = to_amdgpu_ring(sched); 479 480 mutex_lock(&xcp_mgr->xcp_lock); 481 if (ring->xcp_id < xcp_mgr->num_xcps && xcp_mgr->xcp[ring->xcp_id].valid) 482 atomic_dec(&xcp_mgr->xcp[ring->xcp_id].ref_cnt); 483 mutex_unlock(&xcp_mgr->xcp_lock); 484 } 485 } 486 487 int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, 488 u32 hw_ip, u32 hw_prio, 489 struct amdgpu_fpriv *fpriv, 490 unsigned int *num_scheds, 491 struct drm_gpu_scheduler ***scheds) 492 { 493 u32 sel_xcp_id; 494 int i; 495 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 496 int r = 0; 497 498 mutex_lock(&xcp_mgr->xcp_lock); 499 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { 500 u32 least_ref_cnt = ~0; 501 502 fpriv->xcp_id = 0; 503 for (i = 0; i < xcp_mgr->num_xcps; i++) { 504 u32 total_ref_cnt; 505 506 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt); 507 if (total_ref_cnt < least_ref_cnt) { 508 fpriv->xcp_id = i; 509 least_ref_cnt = total_ref_cnt; 510 } 511 } 512 } 513 sel_xcp_id = fpriv->xcp_id; 514 515 if (sel_xcp_id >= xcp_mgr->num_xcps || !xcp_mgr->xcp[sel_xcp_id].valid) { 516 dev_err(adev->dev, "Selected partition #%d is not valid.", sel_xcp_id); 517 r = -ENODEV; 518 goto out; 519 } 520 521 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { 522 *num_scheds = 523 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; 524 *scheds = 525 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].sched; 526 atomic_inc(&xcp_mgr->xcp[sel_xcp_id].ref_cnt); 527 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); 528 } else { 529 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); 530 r = -ENOENT; 531 } 532 533 out: 534 mutex_unlock(&xcp_mgr->xcp_lock); 535 return r; 536 } 537 538 static void amdgpu_set_xcp_id(struct amdgpu_device *adev, 539 uint32_t inst_idx, 540 struct amdgpu_ring *ring) 541 { 542 int xcp_id; 543 enum AMDGPU_XCP_IP_BLOCK ip_blk; 544 uint32_t inst_mask; 545 546 ring->xcp_id = AMDGPU_XCP_NO_PARTITION; 547 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 548 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id; 549 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) || 550 (ring->funcs->type == AMDGPU_RING_TYPE_CPER)) 551 return; 552 553 inst_mask = 1 << inst_idx; 554 555 switch (ring->funcs->type) { 556 case AMDGPU_HW_IP_GFX: 557 case AMDGPU_RING_TYPE_COMPUTE: 558 case AMDGPU_RING_TYPE_KIQ: 559 case AMDGPU_RING_TYPE_MES: 560 ip_blk = AMDGPU_XCP_GFX; 561 break; 562 case AMDGPU_RING_TYPE_SDMA: 563 ip_blk = AMDGPU_XCP_SDMA; 564 break; 565 case AMDGPU_RING_TYPE_VCN_ENC: 566 case AMDGPU_RING_TYPE_VCN_JPEG: 567 ip_blk = AMDGPU_XCP_VCN; 568 break; 569 default: 570 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type); 571 return; 572 } 573 574 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) { 575 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) { 576 ring->xcp_id = xcp_id; 577 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name, 578 ring->xcp_id); 579 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 580 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id; 581 break; 582 } 583 } 584 } 585 586 static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, 587 struct amdgpu_ring *ring, 588 unsigned int sel_xcp_id) 589 { 590 unsigned int *num_gpu_sched; 591 592 if (sel_xcp_id >= MAX_XCP || sel_xcp_id == AMDGPU_XCP_NO_PARTITION) 593 return; 594 595 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] 596 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; 597 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] 598 .sched[(*num_gpu_sched)++] = &ring->sched; 599 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d", 600 ring->name, sel_xcp_id, ring->funcs->type, 601 ring->hw_prio, *num_gpu_sched); 602 } 603 604 static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev) 605 { 606 struct amdgpu_ring *ring; 607 int i; 608 609 for (i = 0; i < MAX_XCP; i++) { 610 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0); 611 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched)); 612 } 613 614 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 615 return 0; 616 617 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 618 ring = adev->rings[i]; 619 if (!ring || !ring->sched.ready || ring->no_scheduler) 620 continue; 621 622 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id); 623 624 /* VCN may be shared by two partitions under CPX MODE in certain 625 * configs. 626 */ 627 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || 628 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) && 629 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst)) 630 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1); 631 } 632 633 return 0; 634 } 635 636 int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev) 637 { 638 int i; 639 640 for (i = 0; i < adev->num_rings; i++) { 641 struct amdgpu_ring *ring = adev->rings[i]; 642 643 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || 644 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 645 amdgpu_set_xcp_id(adev, ring->xcc_id, ring); 646 else 647 amdgpu_set_xcp_id(adev, ring->me, ring); 648 } 649 650 return amdgpu_xcp_sched_list_update(adev); 651 } 652 653 void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr) 654 { 655 struct amdgpu_device *adev = xcp_mgr->adev; 656 657 xcp_mgr->supp_xcp_modes = 0; 658 659 switch (NUM_XCC(adev->gfx.xcc_mask)) { 660 case 8: 661 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 662 BIT(AMDGPU_DPX_PARTITION_MODE) | 663 BIT(AMDGPU_QPX_PARTITION_MODE) | 664 BIT(AMDGPU_CPX_PARTITION_MODE); 665 break; 666 case 6: 667 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 668 BIT(AMDGPU_TPX_PARTITION_MODE) | 669 BIT(AMDGPU_CPX_PARTITION_MODE); 670 break; 671 case 4: 672 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 673 BIT(AMDGPU_DPX_PARTITION_MODE) | 674 BIT(AMDGPU_CPX_PARTITION_MODE); 675 break; 676 case 2: 677 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 678 BIT(AMDGPU_CPX_PARTITION_MODE); 679 break; 680 case 1: 681 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 682 BIT(AMDGPU_CPX_PARTITION_MODE); 683 break; 684 685 default: 686 break; 687 } 688 } 689 690 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 691 { 692 /* TODO: 693 * Stop user queues and threads, and make sure GPU is empty of work. 694 */ 695 696 if (flags & AMDGPU_XCP_OPS_KFD) 697 amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev); 698 699 return 0; 700 } 701 702 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 703 { 704 int ret = 0; 705 706 if (flags & AMDGPU_XCP_OPS_KFD) { 707 amdgpu_amdkfd_device_probe(xcp_mgr->adev); 708 amdgpu_amdkfd_device_init(xcp_mgr->adev); 709 /* If KFD init failed, return failure */ 710 if (!xcp_mgr->adev->kfd.init_complete) 711 ret = -EIO; 712 } 713 714 return ret; 715 } 716 717 /*====================== xcp sysfs - configuration ======================*/ 718 #define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \ 719 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \ 720 struct amdgpu_xcp_res_details *xcp_res, char *buf) \ 721 { \ 722 return sysfs_emit(buf, "%d\n", xcp_res->_name); \ 723 } 724 725 struct amdgpu_xcp_res_sysfs_attribute { 726 struct attribute attr; 727 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf); 728 }; 729 730 #define XCP_CFG_SYSFS_RES_ATTR(_name) \ 731 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \ 732 .attr = { .name = __stringify(_name), .mode = 0400 }, \ 733 .show = amdgpu_xcp_res_sysfs_##_name##_show, \ 734 } 735 736 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst) 737 XCP_CFG_SYSFS_RES_ATTR(num_inst); 738 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared) 739 XCP_CFG_SYSFS_RES_ATTR(num_shared); 740 741 #define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr 742 743 static struct attribute *xcp_cfg_res_sysfs_attrs[] = { 744 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst), 745 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL 746 }; 747 748 static const char *xcp_desc[] = { 749 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 750 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 751 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 752 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 753 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 754 }; 755 756 static const char *nps_desc[] = { 757 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN", 758 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 759 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 760 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 761 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 762 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 763 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 764 }; 765 766 ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs); 767 768 #define to_xcp_attr(x) \ 769 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr) 770 #define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj) 771 772 static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj, 773 struct attribute *attr, char *buf) 774 { 775 struct amdgpu_xcp_res_sysfs_attribute *attribute; 776 struct amdgpu_xcp_res_details *xcp_res; 777 778 attribute = to_xcp_attr(attr); 779 xcp_res = to_xcp_res(kobj); 780 781 if (!attribute->show) 782 return -EIO; 783 784 return attribute->show(xcp_res, buf); 785 } 786 787 static const struct sysfs_ops xcp_cfg_res_sysfs_ops = { 788 .show = xcp_cfg_res_sysfs_attr_show, 789 }; 790 791 static const struct kobj_type xcp_cfg_res_sysfs_ktype = { 792 .sysfs_ops = &xcp_cfg_res_sysfs_ops, 793 .default_groups = xcp_cfg_res_sysfs_groups, 794 }; 795 796 const char *xcp_res_names[] = { 797 [AMDGPU_XCP_RES_XCC] = "xcc", 798 [AMDGPU_XCP_RES_DMA] = "dma", 799 [AMDGPU_XCP_RES_DEC] = "dec", 800 [AMDGPU_XCP_RES_JPEG] = "jpeg", 801 }; 802 803 static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr, 804 int mode, 805 struct amdgpu_xcp_cfg *xcp_cfg) 806 { 807 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info) 808 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg); 809 810 return -EOPNOTSUPP; 811 } 812 813 #define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj) 814 static ssize_t supported_xcp_configs_show(struct kobject *kobj, 815 struct kobj_attribute *attr, char *buf) 816 { 817 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 818 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr; 819 int size = 0, mode; 820 char *sep = ""; 821 822 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes) 823 return sysfs_emit(buf, "Not supported\n"); 824 825 for_each_inst(mode, xcp_mgr->supp_xcp_modes) { 826 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 827 sep = ", "; 828 } 829 830 size += sysfs_emit_at(buf, size, "\n"); 831 832 return size; 833 } 834 835 static ssize_t supported_nps_configs_show(struct kobject *kobj, 836 struct kobj_attribute *attr, char *buf) 837 { 838 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 839 int size = 0, mode; 840 char *sep = ""; 841 842 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes) 843 return sysfs_emit(buf, "Not supported\n"); 844 845 for_each_inst(mode, xcp_cfg->compatible_nps_modes) { 846 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 847 sep = ", "; 848 } 849 850 size += sysfs_emit_at(buf, size, "\n"); 851 852 return size; 853 } 854 855 static ssize_t xcp_config_show(struct kobject *kobj, 856 struct kobj_attribute *attr, char *buf) 857 { 858 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 859 860 return sysfs_emit(buf, "%s\n", 861 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode)); 862 } 863 864 static ssize_t xcp_config_store(struct kobject *kobj, 865 struct kobj_attribute *attr, 866 const char *buf, size_t size) 867 { 868 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 869 int mode, r; 870 871 if (!strncasecmp("SPX", buf, strlen("SPX"))) 872 mode = AMDGPU_SPX_PARTITION_MODE; 873 else if (!strncasecmp("DPX", buf, strlen("DPX"))) 874 mode = AMDGPU_DPX_PARTITION_MODE; 875 else if (!strncasecmp("TPX", buf, strlen("TPX"))) 876 mode = AMDGPU_TPX_PARTITION_MODE; 877 else if (!strncasecmp("QPX", buf, strlen("QPX"))) 878 mode = AMDGPU_QPX_PARTITION_MODE; 879 else if (!strncasecmp("CPX", buf, strlen("CPX"))) 880 mode = AMDGPU_CPX_PARTITION_MODE; 881 else 882 return -EINVAL; 883 884 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 885 886 if (r) 887 return r; 888 889 xcp_cfg->mode = mode; 890 return size; 891 } 892 893 static struct kobj_attribute xcp_cfg_sysfs_mode = 894 __ATTR_RW_MODE(xcp_config, 0644); 895 896 static void xcp_cfg_sysfs_release(struct kobject *kobj) 897 { 898 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 899 900 kfree(xcp_cfg); 901 } 902 903 static const struct kobj_type xcp_cfg_sysfs_ktype = { 904 .release = xcp_cfg_sysfs_release, 905 .sysfs_ops = &kobj_sysfs_ops, 906 }; 907 908 static struct kobj_attribute supp_part_sysfs_mode = 909 __ATTR_RO(supported_xcp_configs); 910 911 static struct kobj_attribute supp_nps_sysfs_mode = 912 __ATTR_RO(supported_nps_configs); 913 914 static const struct attribute *xcp_attrs[] = { 915 &supp_part_sysfs_mode.attr, 916 &xcp_cfg_sysfs_mode.attr, 917 NULL, 918 }; 919 920 static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) 921 { 922 struct amdgpu_xcp_res_details *xcp_res; 923 struct amdgpu_xcp_cfg *xcp_cfg; 924 int i, r, rid, mode; 925 926 if (!adev->xcp_mgr) 927 return; 928 929 xcp_cfg = kzalloc_obj(*xcp_cfg); 930 if (!xcp_cfg) 931 return; 932 xcp_cfg->xcp_mgr = adev->xcp_mgr; 933 934 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype, 935 &adev->dev->kobj, "compute_partition_config"); 936 if (r) 937 goto err1; 938 939 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs); 940 if (r) 941 goto err1; 942 943 if (adev->gmc.supported_nps_modes != 0) { 944 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 945 if (r) { 946 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 947 goto err1; 948 } 949 } 950 951 mode = (xcp_cfg->xcp_mgr->mode == 952 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ? 953 AMDGPU_SPX_PARTITION_MODE : 954 xcp_cfg->xcp_mgr->mode; 955 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 956 if (r) { 957 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 958 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 959 goto err1; 960 } 961 962 xcp_cfg->mode = mode; 963 for (i = 0; i < xcp_cfg->num_res; i++) { 964 xcp_res = &xcp_cfg->xcp_res[i]; 965 rid = xcp_res->id; 966 r = kobject_init_and_add(&xcp_res->kobj, 967 &xcp_cfg_res_sysfs_ktype, 968 &xcp_cfg->kobj, "%s", 969 xcp_res_names[rid]); 970 if (r) { 971 kobject_put(&xcp_res->kobj); 972 goto err; 973 } 974 } 975 976 adev->xcp_mgr->xcp_cfg = xcp_cfg; 977 return; 978 err: 979 while (i--) { 980 xcp_res = &xcp_cfg->xcp_res[i]; 981 kobject_put(&xcp_res->kobj); 982 } 983 984 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 985 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 986 err1: 987 kobject_put(&xcp_cfg->kobj); 988 } 989 990 static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev) 991 { 992 struct amdgpu_xcp_res_details *xcp_res; 993 struct amdgpu_xcp_cfg *xcp_cfg; 994 int i; 995 996 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg) 997 return; 998 999 xcp_cfg = adev->xcp_mgr->xcp_cfg; 1000 for (i = 0; i < xcp_cfg->num_res; i++) { 1001 xcp_res = &xcp_cfg->xcp_res[i]; 1002 kobject_put(&xcp_res->kobj); 1003 } 1004 1005 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 1006 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 1007 kobject_put(&xcp_cfg->kobj); 1008 } 1009 1010 /*====================== xcp sysfs - data entries ======================*/ 1011 1012 #define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj) 1013 1014 static ssize_t xcp_metrics_show(struct kobject *kobj, 1015 struct kobj_attribute *attr, char *buf) 1016 { 1017 struct amdgpu_xcp *xcp = to_xcp(kobj); 1018 struct amdgpu_xcp_mgr *xcp_mgr; 1019 ssize_t size; 1020 1021 xcp_mgr = xcp->xcp_mgr; 1022 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL); 1023 if (size <= 0) 1024 return size; 1025 1026 if (size > PAGE_SIZE) 1027 return -ENOSPC; 1028 1029 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf); 1030 } 1031 1032 static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj, 1033 struct attribute *attr, int n) 1034 { 1035 struct amdgpu_xcp *xcp = to_xcp(kobj); 1036 1037 if (!xcp || !xcp->valid) 1038 return 0; 1039 1040 return attr->mode; 1041 } 1042 1043 static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics); 1044 1045 static struct attribute *amdgpu_xcp_attrs[] = { 1046 &xcp_sysfs_metrics.attr, 1047 NULL, 1048 }; 1049 1050 static const struct attribute_group amdgpu_xcp_attrs_group = { 1051 .attrs = amdgpu_xcp_attrs, 1052 .is_visible = amdgpu_xcp_attrs_is_visible 1053 }; 1054 1055 static const struct kobj_type xcp_sysfs_ktype = { 1056 .sysfs_ops = &kobj_sysfs_ops, 1057 }; 1058 1059 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n) 1060 { 1061 struct amdgpu_xcp *xcp; 1062 1063 for (n--; n >= 0; n--) { 1064 xcp = &xcp_mgr->xcp[n]; 1065 if (!xcp->ddev || !xcp->valid) 1066 continue; 1067 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1068 kobject_put(&xcp->kobj); 1069 } 1070 } 1071 1072 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr) 1073 { 1074 struct amdgpu_xcp *xcp; 1075 int i, r; 1076 1077 for (i = 0; i < MAX_XCP; i++) { 1078 /* Redirect all IOCTLs to the primary device */ 1079 xcp = &xcp_mgr->xcp[i]; 1080 if (!xcp->ddev) 1081 break; 1082 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype, 1083 &xcp->ddev->dev->kobj, "xcp"); 1084 if (r) 1085 goto out; 1086 1087 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1088 if (r) 1089 goto out; 1090 } 1091 1092 return; 1093 out: 1094 kobject_put(&xcp->kobj); 1095 } 1096 1097 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr) 1098 { 1099 struct amdgpu_xcp *xcp; 1100 int i; 1101 1102 for (i = 0; i < MAX_XCP; i++) { 1103 /* Redirect all IOCTLs to the primary device */ 1104 xcp = &xcp_mgr->xcp[i]; 1105 if (!xcp->ddev) 1106 continue; 1107 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1108 } 1109 1110 return; 1111 } 1112 1113 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev) 1114 { 1115 if (!adev->xcp_mgr) 1116 return; 1117 1118 amdgpu_xcp_cfg_sysfs_init(adev); 1119 1120 return; 1121 } 1122 1123 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev) 1124 { 1125 if (!adev->xcp_mgr) 1126 return; 1127 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP); 1128 amdgpu_xcp_cfg_sysfs_fini(adev); 1129 } 1130