1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 27 #include <drm/drm_drv.h> 28 #include "../amdxcp/amdgpu_xcp_drv.h" 29 30 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr); 31 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr); 32 33 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 34 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 35 { 36 int (*run_func)(void *handle, uint32_t inst_mask); 37 int ret = 0; 38 39 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 40 return 0; 41 42 run_func = NULL; 43 44 switch (xcp_state) { 45 case AMDGPU_XCP_PREPARE_SUSPEND: 46 run_func = xcp_ip->ip_funcs->prepare_suspend; 47 break; 48 case AMDGPU_XCP_SUSPEND: 49 run_func = xcp_ip->ip_funcs->suspend; 50 break; 51 case AMDGPU_XCP_PREPARE_RESUME: 52 run_func = xcp_ip->ip_funcs->prepare_resume; 53 break; 54 case AMDGPU_XCP_RESUME: 55 run_func = xcp_ip->ip_funcs->resume; 56 break; 57 } 58 59 if (run_func) 60 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 61 62 return ret; 63 } 64 65 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 66 int state) 67 { 68 struct amdgpu_xcp_ip *xcp_ip; 69 struct amdgpu_xcp *xcp; 70 int i, ret; 71 72 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 73 return -EINVAL; 74 75 xcp = &xcp_mgr->xcp[xcp_id]; 76 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 77 xcp_ip = &xcp->ip[i]; 78 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 79 if (ret) 80 break; 81 } 82 83 return ret; 84 } 85 86 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 87 { 88 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 89 AMDGPU_XCP_PREPARE_SUSPEND); 90 } 91 92 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 93 { 94 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 95 } 96 97 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 98 { 99 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 100 AMDGPU_XCP_PREPARE_RESUME); 101 } 102 103 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 104 { 105 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 106 } 107 108 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 109 struct amdgpu_xcp_ip *ip) 110 { 111 struct amdgpu_xcp *xcp; 112 113 if (!ip) 114 return; 115 116 xcp = &xcp_mgr->xcp[xcp_id]; 117 xcp->ip[ip->ip_id] = *ip; 118 xcp->ip[ip->ip_id].valid = true; 119 120 xcp->valid = true; 121 } 122 123 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 124 { 125 struct amdgpu_device *adev = xcp_mgr->adev; 126 struct amdgpu_xcp_ip ip; 127 uint8_t mem_id; 128 int i, j, ret; 129 130 if (!num_xcps || num_xcps > MAX_XCP) 131 return -EINVAL; 132 133 xcp_mgr->mode = mode; 134 135 for (i = 0; i < MAX_XCP; ++i) 136 xcp_mgr->xcp[i].valid = false; 137 138 /* This is needed for figuring out memory id of xcp */ 139 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 140 141 for (i = 0; i < num_xcps; ++i) { 142 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 143 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 144 &ip); 145 if (ret) 146 continue; 147 148 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 149 } 150 151 xcp_mgr->xcp[i].id = i; 152 153 if (xcp_mgr->funcs->get_xcp_mem_id) { 154 ret = xcp_mgr->funcs->get_xcp_mem_id( 155 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 156 if (ret) 157 continue; 158 else 159 xcp_mgr->xcp[i].mem_id = mem_id; 160 } 161 } 162 163 xcp_mgr->num_xcps = num_xcps; 164 amdgpu_xcp_update_partition_sched_list(adev); 165 166 return 0; 167 } 168 169 static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, 170 int mode) 171 { 172 int ret, curr_mode, num_xcps = 0; 173 174 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 175 return 0; 176 177 mutex_lock(&xcp_mgr->xcp_lock); 178 179 curr_mode = xcp_mgr->mode; 180 /* State set to transient mode */ 181 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 182 183 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 184 185 if (ret) { 186 /* Failed, get whatever mode it's at now */ 187 if (xcp_mgr->funcs->query_partition_mode) 188 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 189 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 190 else 191 xcp_mgr->mode = curr_mode; 192 193 goto out; 194 } 195 amdgpu_xcp_sysfs_entries_update(xcp_mgr); 196 out: 197 mutex_unlock(&xcp_mgr->xcp_lock); 198 199 return ret; 200 } 201 202 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 203 { 204 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 205 return -EINVAL; 206 207 if (xcp_mgr->mode == mode) 208 return 0; 209 210 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode); 211 } 212 213 int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr) 214 { 215 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 216 return 0; 217 218 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode); 219 } 220 221 static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr) 222 { 223 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 224 return true; 225 226 if (!amdgpu_sriov_vf(xcp_mgr->adev) && 227 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 228 return true; 229 230 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE && 231 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS) 232 return true; 233 234 return false; 235 } 236 237 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 238 { 239 int mode; 240 241 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr)) 242 return xcp_mgr->mode; 243 244 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 245 mutex_lock(&xcp_mgr->xcp_lock); 246 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 247 248 /* First time query for VF, set the mode here */ 249 if (amdgpu_sriov_vf(xcp_mgr->adev) && 250 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 251 xcp_mgr->mode = mode; 252 253 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 254 dev_WARN( 255 xcp_mgr->adev->dev, 256 "Cached partition mode %d not matching with device mode %d", 257 xcp_mgr->mode, mode); 258 259 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 260 mutex_unlock(&xcp_mgr->xcp_lock); 261 262 return mode; 263 } 264 265 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 266 { 267 struct drm_device *p_ddev; 268 struct drm_device *ddev; 269 int i, ret; 270 271 ddev = adev_to_drm(adev); 272 273 /* xcp #0 shares drm device setting with adev */ 274 adev->xcp_mgr->xcp->ddev = ddev; 275 276 for (i = 1; i < MAX_XCP; i++) { 277 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 278 if (ret == -ENOSPC) { 279 dev_warn(adev->dev, 280 "Skip xcp node #%d when out of drm node resource.", i); 281 ret = 0; 282 goto out; 283 } else if (ret) { 284 goto out; 285 } 286 287 /* Redirect all IOCTLs to the primary device */ 288 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 289 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 290 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 291 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 292 p_ddev->render->dev = ddev; 293 p_ddev->primary->dev = ddev; 294 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 295 p_ddev->driver = &amdgpu_partition_driver; 296 adev->xcp_mgr->xcp[i].ddev = p_ddev; 297 298 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]); 299 } 300 ret = 0; 301 out: 302 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr); 303 304 return ret; 305 } 306 307 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 308 int init_num_xcps, 309 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 310 { 311 struct amdgpu_xcp_mgr *xcp_mgr; 312 int i; 313 314 if (!xcp_funcs || !xcp_funcs->get_ip_details) 315 return -EINVAL; 316 317 xcp_mgr = kzalloc(sizeof(*xcp_mgr), GFP_KERNEL); 318 319 if (!xcp_mgr) 320 return -ENOMEM; 321 322 xcp_mgr->adev = adev; 323 xcp_mgr->funcs = xcp_funcs; 324 xcp_mgr->mode = init_mode; 325 mutex_init(&xcp_mgr->xcp_lock); 326 327 if (init_mode != AMDGPU_XCP_MODE_NONE) 328 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 329 330 adev->xcp_mgr = xcp_mgr; 331 for (i = 0; i < MAX_XCP; ++i) 332 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr; 333 334 return amdgpu_xcp_dev_alloc(adev); 335 } 336 337 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 338 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 339 { 340 struct amdgpu_xcp *xcp; 341 int i, id_mask = 0; 342 343 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 344 return -EINVAL; 345 346 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 347 xcp = &xcp_mgr->xcp[i]; 348 if ((xcp->valid) && (xcp->ip[ip].valid) && 349 (xcp->ip[ip].inst_mask & BIT(instance))) 350 id_mask |= BIT(i); 351 } 352 353 if (!id_mask) 354 id_mask = -ENXIO; 355 356 return id_mask; 357 } 358 359 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 360 enum AMDGPU_XCP_IP_BLOCK ip, 361 uint32_t *inst_mask) 362 { 363 if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid)) 364 return -EINVAL; 365 366 *inst_mask = xcp->ip[ip].inst_mask; 367 368 return 0; 369 } 370 371 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 372 const struct pci_device_id *ent) 373 { 374 int i, ret; 375 376 if (!adev->xcp_mgr) 377 return 0; 378 379 for (i = 1; i < MAX_XCP; i++) { 380 if (!adev->xcp_mgr->xcp[i].ddev) 381 break; 382 383 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 384 if (ret) 385 return ret; 386 } 387 388 return 0; 389 } 390 391 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 392 { 393 struct drm_device *p_ddev; 394 int i; 395 396 if (!adev->xcp_mgr) 397 return; 398 399 for (i = 1; i < MAX_XCP; i++) { 400 if (!adev->xcp_mgr->xcp[i].ddev) 401 break; 402 403 p_ddev = adev->xcp_mgr->xcp[i].ddev; 404 drm_dev_unplug(p_ddev); 405 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 406 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 407 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 408 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 409 amdgpu_xcp_drm_dev_free(p_ddev); 410 } 411 } 412 413 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 414 struct amdgpu_fpriv *fpriv, 415 struct drm_file *file_priv) 416 { 417 int i; 418 419 if (!adev->xcp_mgr) 420 return 0; 421 422 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION; 423 for (i = 0; i < MAX_XCP; ++i) { 424 if (!adev->xcp_mgr->xcp[i].ddev) 425 break; 426 427 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 428 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 429 dev_err(adev->dev, "renderD%d partition %d not valid!", 430 file_priv->minor->index, i); 431 return -ENOENT; 432 } 433 dev_dbg(adev->dev, "renderD%d partition %d opened!", 434 file_priv->minor->index, i); 435 fpriv->xcp_id = i; 436 break; 437 } 438 } 439 440 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 : 441 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 442 return 0; 443 } 444 445 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 446 struct amdgpu_ctx_entity *entity) 447 { 448 struct drm_gpu_scheduler *sched; 449 struct amdgpu_ring *ring; 450 451 if (!adev->xcp_mgr) 452 return; 453 454 sched = entity->entity.rq->sched; 455 if (drm_sched_wqueue_ready(sched)) { 456 ring = to_amdgpu_ring(entity->entity.rq->sched); 457 atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); 458 } 459 } 460 461 int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, 462 u32 hw_ip, u32 hw_prio, 463 struct amdgpu_fpriv *fpriv, 464 unsigned int *num_scheds, 465 struct drm_gpu_scheduler ***scheds) 466 { 467 u32 sel_xcp_id; 468 int i; 469 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; 470 471 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { 472 u32 least_ref_cnt = ~0; 473 474 fpriv->xcp_id = 0; 475 for (i = 0; i < xcp_mgr->num_xcps; i++) { 476 u32 total_ref_cnt; 477 478 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt); 479 if (total_ref_cnt < least_ref_cnt) { 480 fpriv->xcp_id = i; 481 least_ref_cnt = total_ref_cnt; 482 } 483 } 484 } 485 sel_xcp_id = fpriv->xcp_id; 486 487 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { 488 *num_scheds = 489 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; 490 *scheds = 491 xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched; 492 atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt); 493 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); 494 } else { 495 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); 496 return -ENOENT; 497 } 498 499 return 0; 500 } 501 502 static void amdgpu_set_xcp_id(struct amdgpu_device *adev, 503 uint32_t inst_idx, 504 struct amdgpu_ring *ring) 505 { 506 int xcp_id; 507 enum AMDGPU_XCP_IP_BLOCK ip_blk; 508 uint32_t inst_mask; 509 510 ring->xcp_id = AMDGPU_XCP_NO_PARTITION; 511 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 512 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id; 513 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) || 514 (ring->funcs->type == AMDGPU_RING_TYPE_CPER)) 515 return; 516 517 inst_mask = 1 << inst_idx; 518 519 switch (ring->funcs->type) { 520 case AMDGPU_HW_IP_GFX: 521 case AMDGPU_RING_TYPE_COMPUTE: 522 case AMDGPU_RING_TYPE_KIQ: 523 ip_blk = AMDGPU_XCP_GFX; 524 break; 525 case AMDGPU_RING_TYPE_SDMA: 526 ip_blk = AMDGPU_XCP_SDMA; 527 break; 528 case AMDGPU_RING_TYPE_VCN_ENC: 529 case AMDGPU_RING_TYPE_VCN_JPEG: 530 ip_blk = AMDGPU_XCP_VCN; 531 break; 532 default: 533 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type); 534 return; 535 } 536 537 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) { 538 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) { 539 ring->xcp_id = xcp_id; 540 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name, 541 ring->xcp_id); 542 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 543 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id; 544 break; 545 } 546 } 547 } 548 549 static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, 550 struct amdgpu_ring *ring, 551 unsigned int sel_xcp_id) 552 { 553 unsigned int *num_gpu_sched; 554 555 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] 556 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; 557 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] 558 .sched[(*num_gpu_sched)++] = &ring->sched; 559 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d", 560 ring->name, sel_xcp_id, ring->funcs->type, 561 ring->hw_prio, *num_gpu_sched); 562 } 563 564 static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev) 565 { 566 struct amdgpu_ring *ring; 567 int i; 568 569 for (i = 0; i < MAX_XCP; i++) { 570 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0); 571 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched)); 572 } 573 574 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 575 return 0; 576 577 for (i = 0; i < AMDGPU_MAX_RINGS; i++) { 578 ring = adev->rings[i]; 579 if (!ring || !ring->sched.ready || ring->no_scheduler) 580 continue; 581 582 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id); 583 584 /* VCN may be shared by two partitions under CPX MODE in certain 585 * configs. 586 */ 587 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC || 588 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) && 589 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst)) 590 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1); 591 } 592 593 return 0; 594 } 595 596 int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev) 597 { 598 int i; 599 600 for (i = 0; i < adev->num_rings; i++) { 601 struct amdgpu_ring *ring = adev->rings[i]; 602 603 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE || 604 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 605 amdgpu_set_xcp_id(adev, ring->xcc_id, ring); 606 else 607 amdgpu_set_xcp_id(adev, ring->me, ring); 608 } 609 610 return amdgpu_xcp_sched_list_update(adev); 611 } 612 613 void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr) 614 { 615 struct amdgpu_device *adev = xcp_mgr->adev; 616 617 xcp_mgr->supp_xcp_modes = 0; 618 619 switch (NUM_XCC(adev->gfx.xcc_mask)) { 620 case 8: 621 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 622 BIT(AMDGPU_DPX_PARTITION_MODE) | 623 BIT(AMDGPU_QPX_PARTITION_MODE) | 624 BIT(AMDGPU_CPX_PARTITION_MODE); 625 break; 626 case 6: 627 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 628 BIT(AMDGPU_TPX_PARTITION_MODE) | 629 BIT(AMDGPU_CPX_PARTITION_MODE); 630 break; 631 case 4: 632 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 633 BIT(AMDGPU_DPX_PARTITION_MODE) | 634 BIT(AMDGPU_CPX_PARTITION_MODE); 635 break; 636 case 2: 637 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 638 BIT(AMDGPU_CPX_PARTITION_MODE); 639 break; 640 case 1: 641 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) | 642 BIT(AMDGPU_CPX_PARTITION_MODE); 643 break; 644 645 default: 646 break; 647 } 648 } 649 650 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 651 { 652 /* TODO: 653 * Stop user queues and threads, and make sure GPU is empty of work. 654 */ 655 656 if (flags & AMDGPU_XCP_OPS_KFD) 657 amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev); 658 659 return 0; 660 } 661 662 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 663 { 664 int ret = 0; 665 666 if (flags & AMDGPU_XCP_OPS_KFD) { 667 amdgpu_amdkfd_device_probe(xcp_mgr->adev); 668 amdgpu_amdkfd_device_init(xcp_mgr->adev); 669 /* If KFD init failed, return failure */ 670 if (!xcp_mgr->adev->kfd.init_complete) 671 ret = -EIO; 672 } 673 674 return ret; 675 } 676 677 /*====================== xcp sysfs - configuration ======================*/ 678 #define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \ 679 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \ 680 struct amdgpu_xcp_res_details *xcp_res, char *buf) \ 681 { \ 682 return sysfs_emit(buf, "%d\n", xcp_res->_name); \ 683 } 684 685 struct amdgpu_xcp_res_sysfs_attribute { 686 struct attribute attr; 687 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf); 688 }; 689 690 #define XCP_CFG_SYSFS_RES_ATTR(_name) \ 691 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \ 692 .attr = { .name = __stringify(_name), .mode = 0400 }, \ 693 .show = amdgpu_xcp_res_sysfs_##_name##_show, \ 694 } 695 696 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst) 697 XCP_CFG_SYSFS_RES_ATTR(num_inst); 698 XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared) 699 XCP_CFG_SYSFS_RES_ATTR(num_shared); 700 701 #define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr 702 703 static struct attribute *xcp_cfg_res_sysfs_attrs[] = { 704 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst), 705 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL 706 }; 707 708 static const char *xcp_desc[] = { 709 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 710 [AMDGPU_DPX_PARTITION_MODE] = "DPX", 711 [AMDGPU_TPX_PARTITION_MODE] = "TPX", 712 [AMDGPU_QPX_PARTITION_MODE] = "QPX", 713 [AMDGPU_CPX_PARTITION_MODE] = "CPX", 714 }; 715 716 static const char *nps_desc[] = { 717 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN", 718 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 719 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 720 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 721 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 722 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 723 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 724 }; 725 726 ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs); 727 728 #define to_xcp_attr(x) \ 729 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr) 730 #define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj) 731 732 static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj, 733 struct attribute *attr, char *buf) 734 { 735 struct amdgpu_xcp_res_sysfs_attribute *attribute; 736 struct amdgpu_xcp_res_details *xcp_res; 737 738 attribute = to_xcp_attr(attr); 739 xcp_res = to_xcp_res(kobj); 740 741 if (!attribute->show) 742 return -EIO; 743 744 return attribute->show(xcp_res, buf); 745 } 746 747 static const struct sysfs_ops xcp_cfg_res_sysfs_ops = { 748 .show = xcp_cfg_res_sysfs_attr_show, 749 }; 750 751 static const struct kobj_type xcp_cfg_res_sysfs_ktype = { 752 .sysfs_ops = &xcp_cfg_res_sysfs_ops, 753 .default_groups = xcp_cfg_res_sysfs_groups, 754 }; 755 756 const char *xcp_res_names[] = { 757 [AMDGPU_XCP_RES_XCC] = "xcc", 758 [AMDGPU_XCP_RES_DMA] = "dma", 759 [AMDGPU_XCP_RES_DEC] = "dec", 760 [AMDGPU_XCP_RES_JPEG] = "jpeg", 761 }; 762 763 static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr, 764 int mode, 765 struct amdgpu_xcp_cfg *xcp_cfg) 766 { 767 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info) 768 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg); 769 770 return -EOPNOTSUPP; 771 } 772 773 #define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj) 774 static ssize_t supported_xcp_configs_show(struct kobject *kobj, 775 struct kobj_attribute *attr, char *buf) 776 { 777 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 778 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr; 779 int size = 0, mode; 780 char *sep = ""; 781 782 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes) 783 return sysfs_emit(buf, "Not supported\n"); 784 785 for_each_inst(mode, xcp_mgr->supp_xcp_modes) { 786 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]); 787 sep = ", "; 788 } 789 790 size += sysfs_emit_at(buf, size, "\n"); 791 792 return size; 793 } 794 795 static ssize_t supported_nps_configs_show(struct kobject *kobj, 796 struct kobj_attribute *attr, char *buf) 797 { 798 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 799 int size = 0, mode; 800 char *sep = ""; 801 802 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes) 803 return sysfs_emit(buf, "Not supported\n"); 804 805 for_each_inst(mode, xcp_cfg->compatible_nps_modes) { 806 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 807 sep = ", "; 808 } 809 810 size += sysfs_emit_at(buf, size, "\n"); 811 812 return size; 813 } 814 815 static ssize_t xcp_config_show(struct kobject *kobj, 816 struct kobj_attribute *attr, char *buf) 817 { 818 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 819 820 return sysfs_emit(buf, "%s\n", 821 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode)); 822 } 823 824 static ssize_t xcp_config_store(struct kobject *kobj, 825 struct kobj_attribute *attr, 826 const char *buf, size_t size) 827 { 828 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 829 int mode, r; 830 831 if (!strncasecmp("SPX", buf, strlen("SPX"))) 832 mode = AMDGPU_SPX_PARTITION_MODE; 833 else if (!strncasecmp("DPX", buf, strlen("DPX"))) 834 mode = AMDGPU_DPX_PARTITION_MODE; 835 else if (!strncasecmp("TPX", buf, strlen("TPX"))) 836 mode = AMDGPU_TPX_PARTITION_MODE; 837 else if (!strncasecmp("QPX", buf, strlen("QPX"))) 838 mode = AMDGPU_QPX_PARTITION_MODE; 839 else if (!strncasecmp("CPX", buf, strlen("CPX"))) 840 mode = AMDGPU_CPX_PARTITION_MODE; 841 else 842 return -EINVAL; 843 844 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 845 846 if (r) 847 return r; 848 849 xcp_cfg->mode = mode; 850 return size; 851 } 852 853 static struct kobj_attribute xcp_cfg_sysfs_mode = 854 __ATTR_RW_MODE(xcp_config, 0644); 855 856 static void xcp_cfg_sysfs_release(struct kobject *kobj) 857 { 858 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj); 859 860 kfree(xcp_cfg); 861 } 862 863 static const struct kobj_type xcp_cfg_sysfs_ktype = { 864 .release = xcp_cfg_sysfs_release, 865 .sysfs_ops = &kobj_sysfs_ops, 866 }; 867 868 static struct kobj_attribute supp_part_sysfs_mode = 869 __ATTR_RO(supported_xcp_configs); 870 871 static struct kobj_attribute supp_nps_sysfs_mode = 872 __ATTR_RO(supported_nps_configs); 873 874 static const struct attribute *xcp_attrs[] = { 875 &supp_part_sysfs_mode.attr, 876 &xcp_cfg_sysfs_mode.attr, 877 NULL, 878 }; 879 880 static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) 881 { 882 struct amdgpu_xcp_res_details *xcp_res; 883 struct amdgpu_xcp_cfg *xcp_cfg; 884 int i, r, j, rid, mode; 885 886 if (!adev->xcp_mgr) 887 return; 888 889 xcp_cfg = kzalloc(sizeof(*xcp_cfg), GFP_KERNEL); 890 if (!xcp_cfg) 891 return; 892 xcp_cfg->xcp_mgr = adev->xcp_mgr; 893 894 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype, 895 &adev->dev->kobj, "compute_partition_config"); 896 if (r) 897 goto err1; 898 899 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs); 900 if (r) 901 goto err1; 902 903 if (adev->gmc.supported_nps_modes != 0) { 904 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 905 if (r) { 906 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 907 goto err1; 908 } 909 } 910 911 mode = (xcp_cfg->xcp_mgr->mode == 912 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ? 913 AMDGPU_SPX_PARTITION_MODE : 914 xcp_cfg->xcp_mgr->mode; 915 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg); 916 if (r) { 917 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 918 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 919 goto err1; 920 } 921 922 xcp_cfg->mode = mode; 923 for (i = 0; i < xcp_cfg->num_res; i++) { 924 xcp_res = &xcp_cfg->xcp_res[i]; 925 rid = xcp_res->id; 926 r = kobject_init_and_add(&xcp_res->kobj, 927 &xcp_cfg_res_sysfs_ktype, 928 &xcp_cfg->kobj, "%s", 929 xcp_res_names[rid]); 930 if (r) 931 goto err; 932 } 933 934 adev->xcp_mgr->xcp_cfg = xcp_cfg; 935 return; 936 err: 937 for (j = 0; j < i; j++) { 938 xcp_res = &xcp_cfg->xcp_res[i]; 939 kobject_put(&xcp_res->kobj); 940 } 941 942 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 943 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 944 err1: 945 kobject_put(&xcp_cfg->kobj); 946 } 947 948 static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev) 949 { 950 struct amdgpu_xcp_res_details *xcp_res; 951 struct amdgpu_xcp_cfg *xcp_cfg; 952 int i; 953 954 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg) 955 return; 956 957 xcp_cfg = adev->xcp_mgr->xcp_cfg; 958 for (i = 0; i < xcp_cfg->num_res; i++) { 959 xcp_res = &xcp_cfg->xcp_res[i]; 960 kobject_put(&xcp_res->kobj); 961 } 962 963 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr); 964 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs); 965 kobject_put(&xcp_cfg->kobj); 966 } 967 968 /*====================== xcp sysfs - data entries ======================*/ 969 970 #define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj) 971 972 static ssize_t xcp_metrics_show(struct kobject *kobj, 973 struct kobj_attribute *attr, char *buf) 974 { 975 struct amdgpu_xcp *xcp = to_xcp(kobj); 976 struct amdgpu_xcp_mgr *xcp_mgr; 977 ssize_t size; 978 979 xcp_mgr = xcp->xcp_mgr; 980 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL); 981 if (size <= 0) 982 return size; 983 984 if (size > PAGE_SIZE) 985 return -ENOSPC; 986 987 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf); 988 } 989 990 static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj, 991 struct attribute *attr, int n) 992 { 993 struct amdgpu_xcp *xcp = to_xcp(kobj); 994 995 if (!xcp || !xcp->valid) 996 return 0; 997 998 return attr->mode; 999 } 1000 1001 static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics); 1002 1003 static struct attribute *amdgpu_xcp_attrs[] = { 1004 &xcp_sysfs_metrics.attr, 1005 NULL, 1006 }; 1007 1008 static const struct attribute_group amdgpu_xcp_attrs_group = { 1009 .attrs = amdgpu_xcp_attrs, 1010 .is_visible = amdgpu_xcp_attrs_is_visible 1011 }; 1012 1013 static const struct kobj_type xcp_sysfs_ktype = { 1014 .sysfs_ops = &kobj_sysfs_ops, 1015 }; 1016 1017 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n) 1018 { 1019 struct amdgpu_xcp *xcp; 1020 1021 for (n--; n >= 0; n--) { 1022 xcp = &xcp_mgr->xcp[n]; 1023 if (!xcp->ddev || !xcp->valid) 1024 continue; 1025 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1026 kobject_put(&xcp->kobj); 1027 } 1028 } 1029 1030 static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr) 1031 { 1032 struct amdgpu_xcp *xcp; 1033 int i, r; 1034 1035 for (i = 0; i < MAX_XCP; i++) { 1036 /* Redirect all IOCTLs to the primary device */ 1037 xcp = &xcp_mgr->xcp[i]; 1038 if (!xcp->ddev) 1039 break; 1040 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype, 1041 &xcp->ddev->dev->kobj, "xcp"); 1042 if (r) 1043 goto out; 1044 1045 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1046 if (r) 1047 goto out; 1048 } 1049 1050 return; 1051 out: 1052 kobject_put(&xcp->kobj); 1053 } 1054 1055 static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr) 1056 { 1057 struct amdgpu_xcp *xcp; 1058 int i; 1059 1060 for (i = 0; i < MAX_XCP; i++) { 1061 /* Redirect all IOCTLs to the primary device */ 1062 xcp = &xcp_mgr->xcp[i]; 1063 if (!xcp->ddev) 1064 continue; 1065 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group); 1066 } 1067 1068 return; 1069 } 1070 1071 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev) 1072 { 1073 if (!adev->xcp_mgr) 1074 return; 1075 1076 amdgpu_xcp_cfg_sysfs_init(adev); 1077 1078 return; 1079 } 1080 1081 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev) 1082 { 1083 if (!adev->xcp_mgr) 1084 return; 1085 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP); 1086 amdgpu_xcp_cfg_sysfs_fini(adev); 1087 } 1088