1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/module.h> 25 26 #ifdef CONFIG_X86 27 #include <asm/hypervisor.h> 28 #endif 29 30 #include <drm/drm_drv.h> 31 #include <xen/xen.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_dpm.h" 37 #include "amdgpu_video_codecs.h" 38 #include "vi.h" 39 #include "soc15.h" 40 #include "nv.h" 41 #include "amdgpu_virt_ras_cmd.h" 42 43 #define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \ 44 do { \ 45 vf2pf_info->ucode_info[ucode].id = ucode; \ 46 vf2pf_info->ucode_info[ucode].version = ver; \ 47 } while (0) 48 49 #define mmRCC_CONFIG_MEMSIZE 0xde3 50 51 const char *amdgpu_virt_dynamic_crit_table_name[] = { 52 "IP DISCOVERY", 53 "VBIOS IMG", 54 "RAS TELEMETRY", 55 "DATA EXCHANGE", 56 "BAD PAGE INFO", 57 "INIT HEADER", 58 "LAST", 59 }; 60 61 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) 62 { 63 /* By now all MMIO pages except mailbox are blocked */ 64 /* if blocking is enabled in hypervisor. Choose the */ 65 /* SCRATCH_REG0 to test. */ 66 return RREG32_NO_KIQ(0xc040) == 0xffffffff; 67 } 68 69 void amdgpu_virt_init_setting(struct amdgpu_device *adev) 70 { 71 struct drm_device *ddev = adev_to_drm(adev); 72 73 /* enable virtual display */ 74 if (adev->asic_type != CHIP_ALDEBARAN && 75 adev->asic_type != CHIP_ARCTURUS && 76 ((adev->pdev->class >> 8) != PCI_CLASS_ACCELERATOR_PROCESSING)) { 77 if (adev->mode_info.num_crtc == 0) 78 adev->mode_info.num_crtc = 1; 79 adev->enable_virtual_display = true; 80 } 81 ddev->driver_features &= ~DRIVER_ATOMIC; 82 adev->cg_flags = 0; 83 adev->pg_flags = 0; 84 85 /* Reduce kcq number to 2 to reduce latency */ 86 if (amdgpu_num_kcq == -1) 87 amdgpu_num_kcq = 2; 88 } 89 90 /** 91 * amdgpu_virt_request_full_gpu() - request full gpu access 92 * @adev: amdgpu device. 93 * @init: is driver init time. 94 * When start to init/fini driver, first need to request full gpu access. 95 * Return: Zero if request success, otherwise will return error. 96 */ 97 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 98 { 99 struct amdgpu_virt *virt = &adev->virt; 100 int r; 101 102 if (virt->ops && virt->ops->req_full_gpu) { 103 r = virt->ops->req_full_gpu(adev, init); 104 if (r) { 105 adev->no_hw_access = true; 106 return r; 107 } 108 109 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 110 } 111 112 return 0; 113 } 114 115 /** 116 * amdgpu_virt_release_full_gpu() - release full gpu access 117 * @adev: amdgpu device. 118 * @init: is driver init time. 119 * When finishing driver init/fini, need to release full gpu access. 120 * Return: Zero if release success, otherwise will returen error. 121 */ 122 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 123 { 124 struct amdgpu_virt *virt = &adev->virt; 125 int r; 126 127 if (virt->ops && virt->ops->rel_full_gpu) { 128 r = virt->ops->rel_full_gpu(adev, init); 129 if (r) 130 return r; 131 132 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 133 } 134 return 0; 135 } 136 137 /** 138 * amdgpu_virt_reset_gpu() - reset gpu 139 * @adev: amdgpu device. 140 * Send reset command to GPU hypervisor to reset GPU that VM is using 141 * Return: Zero if reset success, otherwise will return error. 142 */ 143 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) 144 { 145 struct amdgpu_virt *virt = &adev->virt; 146 int r; 147 148 if (virt->ops && virt->ops->reset_gpu) { 149 r = virt->ops->reset_gpu(adev); 150 if (r) 151 return r; 152 153 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 154 } 155 156 return 0; 157 } 158 159 void amdgpu_virt_request_init_data(struct amdgpu_device *adev) 160 { 161 struct amdgpu_virt *virt = &adev->virt; 162 163 if (virt->ops && virt->ops->req_init_data) 164 virt->ops->req_init_data(adev); 165 166 if (adev->virt.req_init_data_ver > 0) 167 dev_info(adev->dev, "host supports REQ_INIT_DATA handshake of critical_region_version %d\n", 168 adev->virt.req_init_data_ver); 169 else 170 dev_warn(adev->dev, "host doesn't support REQ_INIT_DATA handshake\n"); 171 } 172 173 /** 174 * amdgpu_virt_ready_to_reset() - send ready to reset to host 175 * @adev: amdgpu device. 176 * Send ready to reset message to GPU hypervisor to signal we have stopped GPU 177 * activity and is ready for host FLR 178 */ 179 void amdgpu_virt_ready_to_reset(struct amdgpu_device *adev) 180 { 181 struct amdgpu_virt *virt = &adev->virt; 182 183 if (virt->ops && virt->ops->reset_gpu) 184 virt->ops->ready_to_reset(adev); 185 } 186 187 /** 188 * amdgpu_virt_wait_reset() - wait for reset gpu completed 189 * @adev: amdgpu device. 190 * Wait for GPU reset completed. 191 * Return: Zero if reset success, otherwise will return error. 192 */ 193 int amdgpu_virt_wait_reset(struct amdgpu_device *adev) 194 { 195 struct amdgpu_virt *virt = &adev->virt; 196 197 if (!virt->ops || !virt->ops->wait_reset) 198 return -EINVAL; 199 200 return virt->ops->wait_reset(adev); 201 } 202 203 /** 204 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table 205 * @adev: amdgpu device. 206 * MM table is used by UVD and VCE for its initialization 207 * Return: Zero if allocate success. 208 */ 209 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) 210 { 211 int r; 212 213 if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) 214 return 0; 215 216 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 217 AMDGPU_GEM_DOMAIN_VRAM | 218 AMDGPU_GEM_DOMAIN_GTT, 219 &adev->virt.mm_table.bo, 220 &adev->virt.mm_table.gpu_addr, 221 (void *)&adev->virt.mm_table.cpu_addr); 222 if (r) { 223 dev_err(adev->dev, "failed to alloc mm table and error = %d.\n", r); 224 return r; 225 } 226 227 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); 228 dev_info(adev->dev, "MM table gpu addr = 0x%llx, cpu addr = %p.\n", 229 adev->virt.mm_table.gpu_addr, 230 adev->virt.mm_table.cpu_addr); 231 return 0; 232 } 233 234 /** 235 * amdgpu_virt_free_mm_table() - free mm table memory 236 * @adev: amdgpu device. 237 * Free MM table memory 238 */ 239 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) 240 { 241 if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) 242 return; 243 244 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo, 245 &adev->virt.mm_table.gpu_addr, 246 (void *)&adev->virt.mm_table.cpu_addr); 247 adev->virt.mm_table.gpu_addr = 0; 248 } 249 250 /** 251 * amdgpu_virt_rcvd_ras_interrupt() - receive ras interrupt 252 * @adev: amdgpu device. 253 * Check whether host sent RAS error message 254 * Return: true if found, otherwise false 255 */ 256 bool amdgpu_virt_rcvd_ras_interrupt(struct amdgpu_device *adev) 257 { 258 struct amdgpu_virt *virt = &adev->virt; 259 260 if (!virt->ops || !virt->ops->rcvd_ras_intr) 261 return false; 262 263 return virt->ops->rcvd_ras_intr(adev); 264 } 265 266 267 unsigned int amd_sriov_msg_checksum(void *obj, 268 unsigned long obj_size, 269 unsigned int key, 270 unsigned int checksum) 271 { 272 unsigned int ret = key; 273 unsigned long i = 0; 274 unsigned char *pos; 275 276 pos = (char *)obj; 277 /* calculate checksum */ 278 for (i = 0; i < obj_size; ++i) 279 ret += *(pos + i); 280 /* minus the checksum itself */ 281 pos = (char *)&checksum; 282 for (i = 0; i < sizeof(checksum); ++i) 283 ret -= *(pos + i); 284 return ret; 285 } 286 287 #define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY 512 288 /* Max bad page slots allowed for SRIOV*/ 289 #define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY 10665U 290 291 /** 292 * amdgpu_virt_ras_realloc_eh_data_space - alloc/realloc VF bad-page @data->bps and @data->bps_bo 293 * @adev: amdgpu device 294 * @data: VF RAS error-handler data 295 * @pages: minimum number of new slots to add beyond @data->capacity 296 * 297 * Return: 0 on success, %-ENOMEM on failure. 298 */ 299 static int amdgpu_virt_ras_realloc_eh_data_space(struct amdgpu_device *adev, 300 struct amdgpu_virt_ras_err_handler_data *data, 301 int pages) 302 { 303 struct eeprom_table_record *new_bps; 304 struct amdgpu_bo **new_bo; 305 unsigned int old_space; 306 unsigned int new_space; 307 unsigned int align_space; 308 309 old_space = (unsigned int)data->capacity; 310 new_space = old_space + max_t(unsigned int, (unsigned int)pages, 311 (unsigned int)AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); 312 if (new_space < old_space || new_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) 313 return -ENOMEM; 314 315 align_space = ALIGN(new_space, AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); 316 if (align_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) 317 return -ENOMEM; 318 319 new_bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 320 new_bo = kcalloc(align_space, sizeof(*data->bps_bo), GFP_KERNEL); 321 if (!new_bps || !new_bo) { 322 kfree(new_bps); 323 kfree(new_bo); 324 dev_warn_ratelimited(adev->dev, 325 "RAS WARN: failed to grow bad page table to %u slots\n", 326 align_space); 327 return -ENOMEM; 328 } 329 330 memcpy(new_bps, data->bps, data->count * sizeof(*data->bps)); 331 memcpy(new_bo, data->bps_bo, data->count * sizeof(*data->bps_bo)); 332 333 kfree(data->bps); 334 kfree(data->bps_bo); 335 data->bps = new_bps; 336 data->bps_bo = new_bo; 337 data->capacity = (int)align_space; 338 339 return 0; 340 } 341 342 static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev) 343 { 344 struct amdgpu_virt *virt = &adev->virt; 345 struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data; 346 unsigned int align_space = AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY; 347 void *bps = NULL; 348 struct amdgpu_bo **bps_bo = NULL; 349 350 *data = kmalloc_obj(struct amdgpu_virt_ras_err_handler_data); 351 if (!*data) 352 goto data_failure; 353 354 bps = kmalloc_objs(*(*data)->bps, align_space); 355 if (!bps) 356 goto bps_failure; 357 358 bps_bo = kcalloc(align_space, sizeof(*(*data)->bps_bo), GFP_KERNEL); 359 if (!bps_bo) 360 goto bps_bo_failure; 361 362 (*data)->bps = bps; 363 (*data)->bps_bo = bps_bo; 364 (*data)->capacity = align_space; 365 (*data)->count = 0; 366 (*data)->last_reserved = 0; 367 368 virt->ras_init_done = true; 369 370 return 0; 371 372 bps_bo_failure: 373 kfree(bps); 374 bps_failure: 375 kfree(*data); 376 data_failure: 377 return -ENOMEM; 378 } 379 380 static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev) 381 { 382 struct amdgpu_virt *virt = &adev->virt; 383 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 384 struct amdgpu_bo *bo; 385 int i; 386 387 if (!data) 388 return; 389 390 for (i = data->last_reserved - 1; i >= 0; i--) { 391 bo = data->bps_bo[i]; 392 if (bo) { 393 amdgpu_bo_free_kernel(&bo, NULL, NULL); 394 data->bps_bo[i] = bo; 395 } 396 data->last_reserved = i; 397 } 398 } 399 400 void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev) 401 { 402 struct amdgpu_virt *virt = &adev->virt; 403 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 404 405 virt->ras_init_done = false; 406 407 if (!data) 408 return; 409 410 amdgpu_virt_ras_release_bp(adev); 411 412 kfree(data->bps); 413 kfree(data->bps_bo); 414 kfree(data); 415 virt->virt_eh_data = NULL; 416 } 417 418 static bool amdgpu_virt_ras_add_bps(struct amdgpu_device *adev, 419 const struct eeprom_table_record *bps, int pages) 420 { 421 struct amdgpu_virt *virt = &adev->virt; 422 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 423 int need; 424 425 if (!data || pages <= 0) 426 return false; 427 428 if (pages > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY - data->count) { 429 dev_warn_ratelimited(adev->dev, 430 "RAS WARN: bad page table at capacity (count=%d pages=%d max=%u)\n", 431 data->count, pages, 432 AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY); 433 return false; 434 } 435 436 need = data->count + pages; 437 if (need > data->capacity && 438 amdgpu_virt_ras_realloc_eh_data_space(adev, data, need - data->capacity)) 439 return false; 440 441 memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps)); 442 data->count += pages; 443 444 return true; 445 } 446 447 static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) 448 { 449 struct amdgpu_virt *virt = &adev->virt; 450 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 451 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 452 struct ttm_resource_manager *man = &mgr->manager; 453 struct amdgpu_bo *bo = NULL; 454 uint64_t bp; 455 int i; 456 457 if (!data) 458 return; 459 460 for (i = data->last_reserved; i < data->count; i++) { 461 bp = data->bps[i].retired_page; 462 463 /* There are two cases of reserve error should be ignored: 464 * 1) a ras bad page has been allocated (used by someone); 465 * 2) a ras bad page has been reserved (duplicate error injection 466 * for one page); 467 */ 468 if (ttm_resource_manager_used(man)) { 469 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr, 470 bp << AMDGPU_GPU_PAGE_SHIFT, 471 AMDGPU_GPU_PAGE_SIZE); 472 data->bps_bo[i] = NULL; 473 } else { 474 if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT, 475 AMDGPU_GPU_PAGE_SIZE, 476 &bo, NULL)) 477 dev_dbg(adev->dev, 478 "RAS WARN: reserve vram for retired page %llx fail\n", 479 bp); 480 data->bps_bo[i] = bo; 481 } 482 data->last_reserved = i + 1; 483 bo = NULL; 484 } 485 } 486 487 static bool amdgpu_virt_ras_check_bad_page(struct amdgpu_device *adev, 488 uint64_t retired_page) 489 { 490 struct amdgpu_virt *virt = &adev->virt; 491 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 492 int i; 493 494 if (!data) 495 return true; 496 497 for (i = 0; i < data->count; i++) 498 if (retired_page == data->bps[i].retired_page) 499 return true; 500 501 return false; 502 } 503 504 static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, 505 uint64_t bp_block_offset, uint32_t bp_block_size) 506 { 507 struct eeprom_table_record bp; 508 uint64_t retired_page; 509 uint32_t bp_idx, bp_cnt; 510 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 511 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 512 void *vram_usage_va = fw_va ? fw_va : drv_va; 513 514 memset(&bp, 0, sizeof(bp)); 515 516 if (!bp_block_size) 517 return; 518 519 bp_cnt = bp_block_size / sizeof(uint64_t); 520 for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { 521 retired_page = *(uint64_t *)(vram_usage_va + 522 bp_block_offset + bp_idx * sizeof(uint64_t)); 523 bp.retired_page = retired_page; 524 525 if (amdgpu_virt_ras_check_bad_page(adev, retired_page)) 526 continue; 527 528 if (!amdgpu_virt_ras_add_bps(adev, &bp, 1)) 529 break; 530 531 amdgpu_virt_ras_reserve_bps(adev); 532 } 533 } 534 535 static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) 536 { 537 struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf; 538 struct amdgim_pf2vf_info_v1 *pf2vf_v1; 539 struct amd_sriov_msg_pf2vf_info *pf2vf; 540 541 uint32_t checksum; 542 uint32_t checkval; 543 544 uint32_t i; 545 uint32_t tmp; 546 547 if (adev->virt.fw_reserve.p_pf2vf == NULL) 548 return -EINVAL; 549 550 if (pf2vf_info->size > 1024) { 551 dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size); 552 return -EINVAL; 553 } 554 555 switch (pf2vf_info->version) { 556 case 1: 557 pf2vf_v1 = (struct amdgim_pf2vf_info_v1 *)pf2vf_info; 558 checksum = pf2vf_v1->checksum; 559 checkval = amd_sriov_msg_checksum( 560 adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 561 adev->virt.fw_reserve.checksum_key, checksum); 562 if (checksum != checkval) { 563 dev_err(adev->dev, 564 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", 565 checksum, checkval); 566 return -EINVAL; 567 } 568 569 adev->virt.gim_feature = pf2vf_v1->feature_flags; 570 break; 571 case 2: 572 /* TODO: missing key, need to add it later */ 573 pf2vf = (struct amd_sriov_msg_pf2vf_info *)pf2vf_info; 574 checksum = pf2vf->checksum; 575 checkval = amd_sriov_msg_checksum( 576 adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 577 0, checksum); 578 if (checksum != checkval) { 579 dev_err(adev->dev, 580 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", 581 checksum, checkval); 582 return -EINVAL; 583 } 584 585 adev->virt.vf2pf_update_interval_ms = 586 pf2vf->vf2pf_update_interval_ms; 587 adev->virt.gim_feature = pf2vf->feature_flags.all; 588 adev->virt.reg_access = pf2vf->reg_access_flags.all; 589 590 adev->virt.decode_max_dimension_pixels = 0; 591 adev->virt.decode_max_frame_pixels = 0; 592 adev->virt.encode_max_dimension_pixels = 0; 593 adev->virt.encode_max_frame_pixels = 0; 594 adev->virt.is_mm_bw_enabled = false; 595 for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) { 596 tmp = pf2vf->mm_bw_management[i].decode_max_dimension_pixels; 597 adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels); 598 599 tmp = pf2vf->mm_bw_management[i].decode_max_frame_pixels; 600 adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels); 601 602 tmp = pf2vf->mm_bw_management[i].encode_max_dimension_pixels; 603 adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels); 604 605 tmp = pf2vf->mm_bw_management[i].encode_max_frame_pixels; 606 adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels); 607 } 608 if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0)) 609 adev->virt.is_mm_bw_enabled = true; 610 611 adev->unique_id = pf2vf->uuid; 612 613 adev->unitid = 0; 614 if (amdgpu_sriov_is_unitid_support(adev)) 615 adev->unitid = pf2vf->unitid; 616 617 adev->virt.ras_en_caps.all = pf2vf->ras_en_caps.all; 618 adev->virt.ras_telemetry_en_caps.all = 619 pf2vf->ras_telemetry_en_caps.all; 620 break; 621 default: 622 dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version); 623 return -EINVAL; 624 } 625 626 /* correct too large or too little interval value */ 627 if (adev->virt.vf2pf_update_interval_ms < 200 || adev->virt.vf2pf_update_interval_ms > 10000) 628 adev->virt.vf2pf_update_interval_ms = 2000; 629 630 return 0; 631 } 632 633 static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) 634 { 635 struct amd_sriov_msg_vf2pf_info *vf2pf_info; 636 vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; 637 638 if (adev->virt.fw_reserve.p_vf2pf == NULL) 639 return; 640 641 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCE, adev->vce.fw_version); 642 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_UVD, adev->uvd.fw_version); 643 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MC, adev->gmc.fw_version); 644 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ME, adev->gfx.me_fw_version); 645 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_PFP, adev->gfx.pfp_fw_version); 646 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_CE, adev->gfx.ce_fw_version); 647 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC, adev->gfx.rlc_fw_version); 648 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLC, adev->gfx.rlc_srlc_fw_version); 649 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLG, adev->gfx.rlc_srlg_fw_version); 650 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version); 651 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); 652 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); 653 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); 654 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, 655 adev->psp.asd_context.bin_desc.fw_version); 656 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS, 657 adev->psp.ras_context.context.bin_desc.fw_version); 658 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI, 659 adev->psp.xgmi_context.context.bin_desc.fw_version); 660 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC, adev->pm.fw_version); 661 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA, adev->sdma.instance[0].fw_version); 662 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2, adev->sdma.instance[1].fw_version); 663 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCN, adev->vcn.fw_version); 664 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_DMCU, adev->dm.dmcu_fw_version); 665 } 666 667 static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) 668 { 669 struct amd_sriov_msg_vf2pf_info *vf2pf_info; 670 671 vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; 672 673 if (adev->virt.fw_reserve.p_vf2pf == NULL) 674 return -EINVAL; 675 676 memset(vf2pf_info, 0, sizeof(struct amd_sriov_msg_vf2pf_info)); 677 678 vf2pf_info->header.size = sizeof(struct amd_sriov_msg_vf2pf_info); 679 vf2pf_info->header.version = AMD_SRIOV_MSG_FW_VRAM_VF2PF_VER; 680 681 #ifdef MODULE 682 if (THIS_MODULE->version != NULL) 683 strscpy(vf2pf_info->driver_version, THIS_MODULE->version); 684 else 685 #endif 686 strscpy(vf2pf_info->driver_version, "N/A"); 687 688 vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all 689 vf2pf_info->driver_cert = 0; 690 vf2pf_info->os_info.all = 0; 691 692 vf2pf_info->fb_usage = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ? 693 ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) >> 20 : 0; 694 vf2pf_info->fb_vis_usage = 695 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr) >> 20; 696 vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20; 697 vf2pf_info->fb_vis_size = adev->gmc.visible_vram_size >> 20; 698 699 amdgpu_virt_populate_vf2pf_ucode_info(adev); 700 701 /* TODO: read dynamic info */ 702 vf2pf_info->gfx_usage = 0; 703 vf2pf_info->compute_usage = 0; 704 vf2pf_info->encode_usage = 0; 705 vf2pf_info->decode_usage = 0; 706 707 vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr; 708 if (amdgpu_sriov_is_mes_info_enable(adev)) { 709 vf2pf_info->mes_info_addr = 710 (uint64_t)(adev->mes.resource_1_gpu_addr[0] + AMDGPU_GPU_PAGE_SIZE); 711 vf2pf_info->mes_info_size = 712 adev->mes.resource_1[0]->tbo.base.size - AMDGPU_GPU_PAGE_SIZE; 713 } 714 vf2pf_info->checksum = 715 amd_sriov_msg_checksum( 716 vf2pf_info, sizeof(*vf2pf_info), 0, 0); 717 718 return 0; 719 } 720 721 static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) 722 { 723 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work); 724 int ret; 725 726 ret = amdgpu_virt_read_pf2vf_data(adev); 727 if (ret) { 728 adev->virt.vf2pf_update_retry_cnt++; 729 730 if ((amdgpu_virt_rcvd_ras_interrupt(adev) || 731 adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) && 732 amdgpu_sriov_runtime(adev)) { 733 734 amdgpu_ras_set_fed(adev, true); 735 if (amdgpu_reset_domain_schedule(adev->reset_domain, 736 &adev->kfd.reset_work)) 737 return; 738 else 739 dev_err(adev->dev, "Failed to queue work! at %s", __func__); 740 } 741 742 goto out; 743 } 744 745 adev->virt.vf2pf_update_retry_cnt = 0; 746 amdgpu_virt_write_vf2pf_data(adev); 747 748 out: 749 schedule_delayed_work(&(adev->virt.vf2pf_work), adev->virt.vf2pf_update_interval_ms); 750 } 751 752 static int amdgpu_virt_read_exchange_data_from_mem(struct amdgpu_device *adev, uint32_t *pfvf_data) 753 { 754 uint32_t dataexchange_offset = 755 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset; 756 uint32_t dataexchange_size = 757 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb << 10; 758 uint64_t pos = 0; 759 760 dev_info(adev->dev, 761 "Got data exchange info from dynamic crit_region_table at offset 0x%x with size of 0x%x bytes.\n", 762 dataexchange_offset, dataexchange_size); 763 764 if (!IS_ALIGNED(dataexchange_offset, 4) || !IS_ALIGNED(dataexchange_size, 4)) { 765 dev_err(adev->dev, "Data exchange data not aligned to 4 bytes\n"); 766 return -EINVAL; 767 } 768 769 pos = (uint64_t)dataexchange_offset; 770 amdgpu_device_vram_access(adev, pos, pfvf_data, 771 dataexchange_size, false); 772 773 return 0; 774 } 775 776 void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev) 777 { 778 if (adev->virt.vf2pf_update_interval_ms != 0) { 779 dev_info(adev->dev, "clean up the vf2pf work item\n"); 780 cancel_delayed_work_sync(&adev->virt.vf2pf_work); 781 adev->virt.vf2pf_update_interval_ms = 0; 782 } 783 } 784 785 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) 786 { 787 uint32_t *pfvf_data = NULL; 788 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 789 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 790 791 adev->virt.fw_reserve.p_pf2vf = NULL; 792 adev->virt.fw_reserve.p_vf2pf = NULL; 793 adev->virt.vf2pf_update_interval_ms = 0; 794 adev->virt.vf2pf_update_retry_cnt = 0; 795 796 if (fw_va && drv_va) { 797 dev_warn(adev->dev, "Currently fw_vram and drv_vram should not have values at the same time!"); 798 } else if (fw_va || drv_va) { 799 /* go through this logic in ip_init and reset to init workqueue*/ 800 amdgpu_virt_exchange_data(adev); 801 802 INIT_DELAYED_WORK(&adev->virt.vf2pf_work, amdgpu_virt_update_vf2pf_work_item); 803 schedule_delayed_work(&(adev->virt.vf2pf_work), msecs_to_jiffies(adev->virt.vf2pf_update_interval_ms)); 804 } else if (adev->bios != NULL) { 805 /* got through this logic in early init stage to get necessary flags, e.g. rlcg_acc related*/ 806 if (adev->virt.req_init_data_ver == GPU_CRIT_REGION_V2) { 807 pfvf_data = 808 kzalloc(adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb << 10, 809 GFP_KERNEL); 810 if (!pfvf_data) { 811 dev_err(adev->dev, "Failed to allocate memory for pfvf_data\n"); 812 return; 813 } 814 815 if (amdgpu_virt_read_exchange_data_from_mem(adev, pfvf_data)) 816 goto free_pfvf_data; 817 818 adev->virt.fw_reserve.p_pf2vf = 819 (struct amd_sriov_msg_pf2vf_info_header *)pfvf_data; 820 821 amdgpu_virt_read_pf2vf_data(adev); 822 823 free_pfvf_data: 824 kfree(pfvf_data); 825 pfvf_data = NULL; 826 adev->virt.fw_reserve.p_pf2vf = NULL; 827 } else { 828 adev->virt.fw_reserve.p_pf2vf = 829 (struct amd_sriov_msg_pf2vf_info_header *) 830 (adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 831 832 amdgpu_virt_read_pf2vf_data(adev); 833 } 834 } 835 } 836 837 838 void amdgpu_virt_exchange_data(struct amdgpu_device *adev) 839 { 840 uint64_t bp_block_offset = 0; 841 uint32_t bp_block_size = 0; 842 struct amd_sriov_msg_pf2vf_info *pf2vf_v2 = NULL; 843 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 844 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 845 846 if (fw_va || drv_va) { 847 if (fw_va) { 848 if (adev->virt.req_init_data_ver == GPU_CRIT_REGION_V2) { 849 adev->virt.fw_reserve.p_pf2vf = 850 (struct amd_sriov_msg_pf2vf_info_header *) 851 (fw_va + 852 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset); 853 adev->virt.fw_reserve.p_vf2pf = 854 (struct amd_sriov_msg_vf2pf_info_header *) 855 (fw_va + 856 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset + 857 (AMD_SRIOV_MSG_SIZE_KB << 10)); 858 adev->virt.fw_reserve.ras_telemetry = 859 (fw_va + 860 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].offset); 861 } else { 862 adev->virt.fw_reserve.p_pf2vf = 863 (struct amd_sriov_msg_pf2vf_info_header *) 864 (fw_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 865 adev->virt.fw_reserve.p_vf2pf = 866 (struct amd_sriov_msg_vf2pf_info_header *) 867 (fw_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB_V1 << 10)); 868 adev->virt.fw_reserve.ras_telemetry = 869 (fw_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB_V1 << 10)); 870 } 871 } else if (drv_va) { 872 adev->virt.fw_reserve.p_pf2vf = 873 (struct amd_sriov_msg_pf2vf_info_header *) 874 (drv_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 875 adev->virt.fw_reserve.p_vf2pf = 876 (struct amd_sriov_msg_vf2pf_info_header *) 877 (drv_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB_V1 << 10)); 878 adev->virt.fw_reserve.ras_telemetry = 879 (drv_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB_V1 << 10)); 880 } 881 882 amdgpu_virt_read_pf2vf_data(adev); 883 amdgpu_virt_write_vf2pf_data(adev); 884 885 /* bad page handling for version 2 */ 886 if (adev->virt.fw_reserve.p_pf2vf->version == 2) { 887 pf2vf_v2 = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf; 888 889 bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_low & 0xFFFFFFFF) | 890 ((((uint64_t)pf2vf_v2->bp_block_offset_high) << 32) & 0xFFFFFFFF00000000); 891 bp_block_size = pf2vf_v2->bp_block_size; 892 893 if (bp_block_size && !adev->virt.ras_init_done) 894 amdgpu_virt_init_ras_err_handler_data(adev); 895 896 if (adev->virt.ras_init_done) 897 amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size); 898 } 899 } 900 } 901 902 static u32 amdgpu_virt_init_detect_asic(struct amdgpu_device *adev) 903 { 904 uint32_t reg; 905 906 switch (adev->asic_type) { 907 case CHIP_TONGA: 908 case CHIP_FIJI: 909 reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER); 910 break; 911 case CHIP_VEGA10: 912 case CHIP_VEGA20: 913 case CHIP_NAVI10: 914 case CHIP_NAVI12: 915 case CHIP_SIENNA_CICHLID: 916 case CHIP_ARCTURUS: 917 case CHIP_ALDEBARAN: 918 case CHIP_IP_DISCOVERY: 919 reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER); 920 break; 921 default: /* other chip doesn't support SRIOV */ 922 reg = 0; 923 break; 924 } 925 926 if (reg & 1) 927 adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF; 928 929 if (reg & 0x80000000) 930 adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV; 931 932 if (!reg) { 933 /* passthrough mode exclus sriov mod */ 934 if (is_virtual_machine() && !xen_initial_domain()) 935 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE; 936 } 937 938 return reg; 939 } 940 941 static bool amdgpu_virt_init_req_data(struct amdgpu_device *adev, u32 reg) 942 { 943 bool is_sriov = false; 944 945 /* we have the ability to check now */ 946 if (amdgpu_sriov_vf(adev)) { 947 is_sriov = true; 948 949 switch (adev->asic_type) { 950 case CHIP_TONGA: 951 case CHIP_FIJI: 952 vi_set_virt_ops(adev); 953 break; 954 case CHIP_VEGA10: 955 soc15_set_virt_ops(adev); 956 #ifdef CONFIG_X86 957 /* not send GPU_INIT_DATA with MS_HYPERV*/ 958 if (!hypervisor_is_type(X86_HYPER_MS_HYPERV)) 959 #endif 960 /* send a dummy GPU_INIT_DATA request to host on vega10 */ 961 amdgpu_virt_request_init_data(adev); 962 break; 963 case CHIP_VEGA20: 964 case CHIP_ARCTURUS: 965 case CHIP_ALDEBARAN: 966 soc15_set_virt_ops(adev); 967 break; 968 case CHIP_NAVI10: 969 case CHIP_NAVI12: 970 case CHIP_SIENNA_CICHLID: 971 case CHIP_IP_DISCOVERY: 972 nv_set_virt_ops(adev); 973 /* try send GPU_INIT_DATA request to host */ 974 amdgpu_virt_request_init_data(adev); 975 break; 976 default: /* other chip doesn't support SRIOV */ 977 is_sriov = false; 978 dev_err(adev->dev, "Unknown asic type: %d!\n", adev->asic_type); 979 break; 980 } 981 } 982 983 return is_sriov; 984 } 985 986 static void amdgpu_virt_init_ras(struct amdgpu_device *adev) 987 { 988 ratelimit_state_init(&adev->virt.ras.ras_error_cnt_rs, 5 * HZ, 1); 989 ratelimit_state_init(&adev->virt.ras.ras_cper_dump_rs, 5 * HZ, 1); 990 ratelimit_state_init(&adev->virt.ras.ras_chk_criti_rs, 5 * HZ, 1); 991 992 ratelimit_set_flags(&adev->virt.ras.ras_error_cnt_rs, 993 RATELIMIT_MSG_ON_RELEASE); 994 ratelimit_set_flags(&adev->virt.ras.ras_cper_dump_rs, 995 RATELIMIT_MSG_ON_RELEASE); 996 ratelimit_set_flags(&adev->virt.ras.ras_chk_criti_rs, 997 RATELIMIT_MSG_ON_RELEASE); 998 999 mutex_init(&adev->virt.ras.ras_telemetry_mutex); 1000 mutex_init(&adev->virt.access_req_mutex); 1001 1002 adev->virt.ras.cper_rptr = 0; 1003 } 1004 1005 static uint8_t amdgpu_virt_crit_region_calc_checksum(uint8_t *buf_start, uint8_t *buf_end) 1006 { 1007 uint32_t sum = 0; 1008 1009 if (buf_start >= buf_end) 1010 return 0; 1011 1012 for (; buf_start < buf_end; buf_start++) 1013 sum += buf_start[0]; 1014 1015 return 0xffffffff - sum; 1016 } 1017 1018 int amdgpu_virt_init_critical_region(struct amdgpu_device *adev) 1019 { 1020 struct amd_sriov_msg_init_data_header *init_data_hdr = NULL; 1021 u64 init_hdr_offset = adev->virt.init_data_header.offset; 1022 u64 init_hdr_size = (u64)adev->virt.init_data_header.size_kb << 10; /* KB → bytes */ 1023 u64 vram_size; 1024 u64 end; 1025 int r = 0; 1026 uint8_t checksum = 0; 1027 1028 /* Skip below init if critical region version != v2 */ 1029 if (adev->virt.req_init_data_ver != GPU_CRIT_REGION_V2) 1030 return 0; 1031 1032 vram_size = RREG32(mmRCC_CONFIG_MEMSIZE); 1033 if (!vram_size || vram_size == U32_MAX) 1034 return -EINVAL; 1035 vram_size <<= 20; 1036 1037 if (check_add_overflow(init_hdr_offset, init_hdr_size, &end) || end > vram_size) { 1038 dev_err(adev->dev, "init_data_header exceeds VRAM size, exiting\n"); 1039 return -EINVAL; 1040 } 1041 1042 /* Allocate for init_data_hdr */ 1043 init_data_hdr = kzalloc_obj(struct amd_sriov_msg_init_data_header); 1044 if (!init_data_hdr) 1045 return -ENOMEM; 1046 1047 amdgpu_device_vram_access(adev, (uint64_t)init_hdr_offset, (uint32_t *)init_data_hdr, 1048 sizeof(struct amd_sriov_msg_init_data_header), false); 1049 1050 /* Table validation */ 1051 if (strncmp(init_data_hdr->signature, 1052 AMDGPU_SRIOV_CRIT_DATA_SIGNATURE, 1053 AMDGPU_SRIOV_CRIT_DATA_SIG_LEN) != 0) { 1054 dev_err(adev->dev, "Invalid init data signature: %.4s\n", 1055 init_data_hdr->signature); 1056 r = -EINVAL; 1057 goto out; 1058 } 1059 1060 checksum = amdgpu_virt_crit_region_calc_checksum( 1061 (uint8_t *)&init_data_hdr->initdata_offset, 1062 (uint8_t *)init_data_hdr + 1063 sizeof(struct amd_sriov_msg_init_data_header)); 1064 if (checksum != init_data_hdr->checksum) { 1065 dev_err(adev->dev, "Found unmatching checksum from calculation 0x%x and init_data 0x%x\n", 1066 checksum, init_data_hdr->checksum); 1067 r = -EINVAL; 1068 goto out; 1069 } 1070 1071 memset(&adev->virt.crit_regn, 0, sizeof(adev->virt.crit_regn)); 1072 memset(adev->virt.crit_regn_tbl, 0, sizeof(adev->virt.crit_regn_tbl)); 1073 1074 adev->virt.crit_regn.offset = init_data_hdr->initdata_offset; 1075 adev->virt.crit_regn.size_kb = init_data_hdr->initdata_size_in_kb; 1076 1077 /* Validation and initialization for each table entry */ 1078 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_IPD_TABLE_ID)) { 1079 if (!init_data_hdr->ip_discovery_size_in_kb || 1080 init_data_hdr->ip_discovery_size_in_kb > DISCOVERY_TMR_SIZE) { 1081 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1082 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_IPD_TABLE_ID], 1083 init_data_hdr->ip_discovery_size_in_kb); 1084 r = -EINVAL; 1085 goto out; 1086 } 1087 1088 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset = 1089 init_data_hdr->ip_discovery_offset; 1090 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb = 1091 init_data_hdr->ip_discovery_size_in_kb; 1092 } 1093 1094 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID)) { 1095 if (!init_data_hdr->vbios_img_size_in_kb) { 1096 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1097 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID], 1098 init_data_hdr->vbios_img_size_in_kb); 1099 r = -EINVAL; 1100 goto out; 1101 } 1102 1103 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID].offset = 1104 init_data_hdr->vbios_img_offset; 1105 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID].size_kb = 1106 init_data_hdr->vbios_img_size_in_kb; 1107 } 1108 1109 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID)) { 1110 if (!init_data_hdr->ras_tele_info_size_in_kb) { 1111 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1112 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID], 1113 init_data_hdr->ras_tele_info_size_in_kb); 1114 r = -EINVAL; 1115 goto out; 1116 } 1117 1118 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].offset = 1119 init_data_hdr->ras_tele_info_offset; 1120 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].size_kb = 1121 init_data_hdr->ras_tele_info_size_in_kb; 1122 } 1123 1124 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID)) { 1125 if (!init_data_hdr->dataexchange_size_in_kb) { 1126 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1127 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID], 1128 init_data_hdr->dataexchange_size_in_kb); 1129 r = -EINVAL; 1130 goto out; 1131 } 1132 1133 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset = 1134 init_data_hdr->dataexchange_offset; 1135 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb = 1136 init_data_hdr->dataexchange_size_in_kb; 1137 } 1138 1139 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID)) { 1140 if (!init_data_hdr->bad_page_size_in_kb) { 1141 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1142 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID], 1143 init_data_hdr->bad_page_size_in_kb); 1144 r = -EINVAL; 1145 goto out; 1146 } 1147 1148 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID].offset = 1149 init_data_hdr->bad_page_info_offset; 1150 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID].size_kb = 1151 init_data_hdr->bad_page_size_in_kb; 1152 } 1153 1154 /* Validation for critical region info */ 1155 if (adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb > DISCOVERY_TMR_SIZE) { 1156 dev_err(adev->dev, "Invalid IP discovery size: 0x%x\n", 1157 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb); 1158 r = -EINVAL; 1159 goto out; 1160 } 1161 1162 /* reserved memory starts from crit region base offset with the size of 5MB */ 1163 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_FW_VRAM_USAGE, 1164 adev->virt.crit_regn.offset, 1165 adev->virt.crit_regn.size_kb << 10, true); 1166 dev_info(adev->dev, 1167 "critical region v%d requested to reserve memory start at %08llx with %llu KB.\n", 1168 init_data_hdr->version, 1169 adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].offset, 1170 adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size >> 10); 1171 1172 adev->virt.is_dynamic_crit_regn_enabled = true; 1173 1174 out: 1175 kfree(init_data_hdr); 1176 init_data_hdr = NULL; 1177 1178 return r; 1179 } 1180 1181 int amdgpu_virt_get_dynamic_data_info(struct amdgpu_device *adev, 1182 int data_id, uint8_t *binary, u32 *size) 1183 { 1184 uint32_t data_offset = 0; 1185 uint32_t data_size = 0; 1186 enum amd_sriov_msg_table_id_enum data_table_id = data_id; 1187 1188 if (data_table_id >= AMD_SRIOV_MSG_MAX_TABLE_ID) 1189 return -EINVAL; 1190 1191 data_offset = adev->virt.crit_regn_tbl[data_table_id].offset; 1192 data_size = adev->virt.crit_regn_tbl[data_table_id].size_kb << 10; 1193 1194 /* Validate on input params */ 1195 if (!binary || !size || *size < (uint64_t)data_size) 1196 return -EINVAL; 1197 1198 /* Proceed to copy the dynamic content */ 1199 amdgpu_device_vram_access(adev, 1200 (uint64_t)data_offset, (uint32_t *)binary, data_size, false); 1201 *size = (uint64_t)data_size; 1202 1203 dev_dbg(adev->dev, 1204 "Got %s info from dynamic crit_region_table at offset 0x%x with size of 0x%x bytes.\n", 1205 amdgpu_virt_dynamic_crit_table_name[data_id], data_offset, data_size); 1206 1207 return 0; 1208 } 1209 1210 void amdgpu_virt_init(struct amdgpu_device *adev) 1211 { 1212 bool is_sriov = false; 1213 uint32_t reg = amdgpu_virt_init_detect_asic(adev); 1214 1215 is_sriov = amdgpu_virt_init_req_data(adev, reg); 1216 1217 if (is_sriov) 1218 amdgpu_virt_init_ras(adev); 1219 } 1220 1221 static bool amdgpu_virt_access_debugfs_is_mmio(struct amdgpu_device *adev) 1222 { 1223 return amdgpu_sriov_is_debug(adev) ? true : false; 1224 } 1225 1226 static bool amdgpu_virt_access_debugfs_is_kiq(struct amdgpu_device *adev) 1227 { 1228 return amdgpu_sriov_is_normal(adev) ? true : false; 1229 } 1230 1231 int amdgpu_virt_enable_access_debugfs(struct amdgpu_device *adev) 1232 { 1233 if (!amdgpu_sriov_vf(adev) || 1234 amdgpu_virt_access_debugfs_is_kiq(adev)) 1235 return 0; 1236 1237 if (amdgpu_virt_access_debugfs_is_mmio(adev)) 1238 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 1239 else 1240 return -EPERM; 1241 1242 return 0; 1243 } 1244 1245 void amdgpu_virt_disable_access_debugfs(struct amdgpu_device *adev) 1246 { 1247 if (amdgpu_sriov_vf(adev)) 1248 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 1249 } 1250 1251 enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *adev) 1252 { 1253 enum amdgpu_sriov_vf_mode mode; 1254 1255 if (amdgpu_sriov_vf(adev)) { 1256 if (amdgpu_sriov_is_pp_one_vf(adev)) 1257 mode = SRIOV_VF_MODE_ONE_VF; 1258 else 1259 mode = SRIOV_VF_MODE_MULTI_VF; 1260 } else { 1261 mode = SRIOV_VF_MODE_BARE_METAL; 1262 } 1263 1264 return mode; 1265 } 1266 1267 void amdgpu_virt_pre_reset(struct amdgpu_device *adev) 1268 { 1269 /* stop the data exchange thread */ 1270 amdgpu_virt_fini_data_exchange(adev); 1271 amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_FLR); 1272 } 1273 1274 void amdgpu_virt_post_reset(struct amdgpu_device *adev) 1275 { 1276 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) { 1277 /* force set to GFXOFF state after reset, 1278 * to avoid some invalid operation before GC enable 1279 */ 1280 adev->gfx.is_poweron = false; 1281 } 1282 1283 adev->mes.ring[0].sched.ready = false; 1284 } 1285 1286 bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id) 1287 { 1288 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 1289 case IP_VERSION(13, 0, 0): 1290 /* no vf autoload, white list */ 1291 if (ucode_id == AMDGPU_UCODE_ID_VCN1 || 1292 ucode_id == AMDGPU_UCODE_ID_VCN) 1293 return false; 1294 else 1295 return true; 1296 case IP_VERSION(11, 0, 9): 1297 case IP_VERSION(11, 0, 7): 1298 /* black list for CHIP_NAVI12 and CHIP_SIENNA_CICHLID */ 1299 if (ucode_id == AMDGPU_UCODE_ID_RLC_G 1300 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL 1301 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM 1302 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM 1303 || ucode_id == AMDGPU_UCODE_ID_SMC) 1304 return true; 1305 else 1306 return false; 1307 case IP_VERSION(13, 0, 10): 1308 /* white list */ 1309 if (ucode_id == AMDGPU_UCODE_ID_CAP 1310 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP 1311 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME 1312 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC 1313 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK 1314 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK 1315 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK 1316 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK 1317 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK 1318 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK 1319 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK 1320 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK 1321 || ucode_id == AMDGPU_UCODE_ID_CP_MES 1322 || ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA 1323 || ucode_id == AMDGPU_UCODE_ID_CP_MES1 1324 || ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA 1325 || ucode_id == AMDGPU_UCODE_ID_VCN1 1326 || ucode_id == AMDGPU_UCODE_ID_VCN) 1327 return false; 1328 else 1329 return true; 1330 default: 1331 /* lagacy black list */ 1332 if (ucode_id == AMDGPU_UCODE_ID_SDMA0 1333 || ucode_id == AMDGPU_UCODE_ID_SDMA1 1334 || ucode_id == AMDGPU_UCODE_ID_SDMA2 1335 || ucode_id == AMDGPU_UCODE_ID_SDMA3 1336 || ucode_id == AMDGPU_UCODE_ID_SDMA4 1337 || ucode_id == AMDGPU_UCODE_ID_SDMA5 1338 || ucode_id == AMDGPU_UCODE_ID_SDMA6 1339 || ucode_id == AMDGPU_UCODE_ID_SDMA7 1340 || ucode_id == AMDGPU_UCODE_ID_SDMA_RS64 1341 || ucode_id == AMDGPU_UCODE_ID_RLC_G 1342 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL 1343 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM 1344 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM 1345 || ucode_id == AMDGPU_UCODE_ID_SMC) 1346 return true; 1347 else 1348 return false; 1349 } 1350 } 1351 1352 void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev, 1353 struct amdgpu_video_codec_info *encode, uint32_t encode_array_size, 1354 struct amdgpu_video_codec_info *decode, uint32_t decode_array_size) 1355 { 1356 uint32_t i; 1357 1358 if (!adev->virt.is_mm_bw_enabled) 1359 return; 1360 1361 if (encode) { 1362 for (i = 0; i < encode_array_size; i++) { 1363 encode[i].max_width = adev->virt.encode_max_dimension_pixels; 1364 encode[i].max_pixels_per_frame = adev->virt.encode_max_frame_pixels; 1365 if (encode[i].max_width > 0) 1366 encode[i].max_height = encode[i].max_pixels_per_frame / encode[i].max_width; 1367 else 1368 encode[i].max_height = 0; 1369 } 1370 } 1371 1372 if (decode) { 1373 for (i = 0; i < decode_array_size; i++) { 1374 decode[i].max_width = adev->virt.decode_max_dimension_pixels; 1375 decode[i].max_pixels_per_frame = adev->virt.decode_max_frame_pixels; 1376 if (decode[i].max_width > 0) 1377 decode[i].max_height = decode[i].max_pixels_per_frame / decode[i].max_width; 1378 else 1379 decode[i].max_height = 0; 1380 } 1381 } 1382 } 1383 1384 bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev, 1385 u32 acc_flags, u32 hwip, 1386 bool write, u32 *rlcg_flag) 1387 { 1388 bool ret = false; 1389 1390 switch (hwip) { 1391 case GC_HWIP: 1392 if (amdgpu_sriov_reg_indirect_gc(adev)) { 1393 *rlcg_flag = 1394 write ? AMDGPU_RLCG_GC_WRITE : AMDGPU_RLCG_GC_READ; 1395 ret = true; 1396 /* only in new version, AMDGPU_REGS_NO_KIQ and 1397 * AMDGPU_REGS_RLC are enabled simultaneously */ 1398 } else if ((acc_flags & AMDGPU_REGS_RLC) && 1399 !(acc_flags & AMDGPU_REGS_NO_KIQ) && write) { 1400 *rlcg_flag = AMDGPU_RLCG_GC_WRITE_LEGACY; 1401 ret = true; 1402 } 1403 break; 1404 case MMHUB_HWIP: 1405 if (amdgpu_sriov_reg_indirect_mmhub(adev) && 1406 (acc_flags & AMDGPU_REGS_RLC) && write) { 1407 *rlcg_flag = AMDGPU_RLCG_MMHUB_WRITE; 1408 ret = true; 1409 } 1410 break; 1411 default: 1412 break; 1413 } 1414 return ret; 1415 } 1416 1417 static u32 amdgpu_virt_rlcg_vfi_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id) 1418 { 1419 uint32_t timeout = 100; 1420 uint32_t i; 1421 1422 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1423 void *vfi_cmd; 1424 void *vfi_stat; 1425 void *vfi_addr; 1426 void *vfi_data; 1427 void *vfi_grbm_cntl; 1428 void *vfi_grbm_idx; 1429 uint32_t cmd; 1430 uint32_t stat; 1431 uint32_t addr = offset; 1432 uint32_t data; 1433 uint32_t grbm_cntl_data; 1434 uint32_t grbm_idx_data; 1435 1436 unsigned long flags; 1437 bool is_err = true; 1438 1439 if (!adev->gfx.rlc.rlcg_reg_access_supported) { 1440 dev_err(adev->dev, "VFi interface is not available\n"); 1441 return 0; 1442 } 1443 1444 if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) { 1445 dev_err(adev->dev, "VFi invalid XCC, xcc_id=0x%x\n", xcc_id); 1446 return 0; 1447 } 1448 1449 if (amdgpu_device_skip_hw_access(adev)) 1450 return 0; 1451 1452 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id]; 1453 vfi_cmd = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_cmd; 1454 vfi_stat = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_stat; 1455 vfi_addr = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_addr; 1456 vfi_data = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_data; 1457 vfi_grbm_cntl = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_grbm_cntl; 1458 vfi_grbm_idx = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_grbm_idx; 1459 grbm_cntl_data = reg_access_ctrl->vfi_grbm_cntl_data; 1460 grbm_idx_data = reg_access_ctrl->vfi_grbm_idx_data; 1461 1462 if (flag == AMDGPU_RLCG_GC_WRITE) { 1463 data = v; 1464 cmd = AMDGPU_RLCG_VFI_CMD__WR; 1465 1466 // the GRBM_GFX_CNTL and GRBM_GFX_INDEX are protected by mutex outside this call 1467 if (addr == reg_access_ctrl->grbm_cntl) { 1468 reg_access_ctrl->vfi_grbm_cntl_data = data; 1469 return 0; 1470 } else if (addr == reg_access_ctrl->grbm_idx) { 1471 reg_access_ctrl->vfi_grbm_idx_data = data; 1472 return 0; 1473 } 1474 1475 } else if (flag == AMDGPU_RLCG_GC_READ) { 1476 data = 0; 1477 cmd = AMDGPU_RLCG_VFI_CMD__RD; 1478 1479 // the GRBM_GFX_CNTL and GRBM_GFX_INDEX are protected by mutex outside this call 1480 if (addr == reg_access_ctrl->grbm_cntl) 1481 return grbm_cntl_data; 1482 else if (addr == reg_access_ctrl->grbm_idx) 1483 return grbm_idx_data; 1484 1485 } else { 1486 dev_err(adev->dev, "VFi invalid access, flag=0x%x\n", flag); 1487 return 0; 1488 } 1489 1490 spin_lock_irqsave(&adev->virt.rlcg_reg_lock, flags); 1491 1492 writel(addr, vfi_addr); 1493 writel(data, vfi_data); 1494 writel(grbm_cntl_data, vfi_grbm_cntl); 1495 writel(grbm_idx_data, vfi_grbm_idx); 1496 1497 writel(AMDGPU_RLCG_VFI_STAT__BUSY, vfi_stat); 1498 writel(cmd, vfi_cmd); 1499 1500 for (i = 0; i < timeout; i++) { 1501 stat = readl(vfi_stat); 1502 if (stat != AMDGPU_RLCG_VFI_STAT__BUSY) 1503 break; 1504 udelay(10); 1505 } 1506 1507 switch (stat) { 1508 case AMDGPU_RLCG_VFI_STAT__DONE: 1509 is_err = false; 1510 if (cmd == AMDGPU_RLCG_VFI_CMD__RD) 1511 data = readl(vfi_data); 1512 break; 1513 case AMDGPU_RLCG_VFI_STAT__BUSY: 1514 dev_err(adev->dev, "VFi access timeout\n"); 1515 break; 1516 case AMDGPU_RLCG_VFI_STAT__INV_CMD: 1517 dev_err(adev->dev, "VFi invalid command\n"); 1518 break; 1519 case AMDGPU_RLCG_VFI_STAT__INV_ADDR: 1520 dev_err(adev->dev, "VFi invalid address\n"); 1521 break; 1522 case AMDGPU_RLCG_VFI_STAT__ERR: 1523 dev_err(adev->dev, "VFi unknown error\n"); 1524 break; 1525 default: 1526 dev_err(adev->dev, "VFi unknown status code\n"); 1527 break; 1528 } 1529 1530 spin_unlock_irqrestore(&adev->virt.rlcg_reg_lock, flags); 1531 1532 if (is_err) 1533 dev_err(adev->dev, "VFi: [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n", 1534 grbm_cntl_data, grbm_idx_data, 1535 addr, addr * 4, data, cmd); 1536 else 1537 dev_dbg(adev->dev, "VFi: [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n", 1538 grbm_cntl_data, grbm_idx_data, 1539 addr, addr * 4, data, cmd); 1540 1541 return data; 1542 } 1543 1544 u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id) 1545 { 1546 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1547 uint32_t timeout = 50000; 1548 uint32_t i, tmp; 1549 uint32_t ret = 0; 1550 void *scratch_reg0; 1551 void *scratch_reg1; 1552 void *scratch_reg2; 1553 void *scratch_reg3; 1554 void *spare_int; 1555 unsigned long flags; 1556 1557 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) 1558 return amdgpu_virt_rlcg_vfi_reg_rw(adev, offset, v, flag, xcc_id); 1559 1560 if (!adev->gfx.rlc.rlcg_reg_access_supported) { 1561 dev_err(adev->dev, 1562 "indirect registers access through rlcg is not available\n"); 1563 return 0; 1564 } 1565 1566 if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) { 1567 dev_err(adev->dev, "invalid xcc\n"); 1568 return 0; 1569 } 1570 1571 if (amdgpu_device_skip_hw_access(adev)) 1572 return 0; 1573 1574 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id]; 1575 scratch_reg0 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg0; 1576 scratch_reg1 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg1; 1577 scratch_reg2 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg2; 1578 scratch_reg3 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg3; 1579 1580 spin_lock_irqsave(&adev->virt.rlcg_reg_lock, flags); 1581 1582 if (reg_access_ctrl->spare_int) 1583 spare_int = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->spare_int; 1584 1585 if (offset == reg_access_ctrl->grbm_cntl) { 1586 /* if the target reg offset is grbm_cntl, write to scratch_reg2 */ 1587 writel(v, scratch_reg2); 1588 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) 1589 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 1590 } else if (offset == reg_access_ctrl->grbm_idx) { 1591 /* if the target reg offset is grbm_idx, write to scratch_reg3 */ 1592 writel(v, scratch_reg3); 1593 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) 1594 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 1595 } else { 1596 /* 1597 * SCRATCH_REG0 = read/write value 1598 * SCRATCH_REG1[30:28] = command 1599 * SCRATCH_REG1[19:0] = address in dword 1600 * SCRATCH_REG1[27:24] = Error reporting 1601 */ 1602 writel(v, scratch_reg0); 1603 writel((offset | flag), scratch_reg1); 1604 if (reg_access_ctrl->spare_int) 1605 writel(1, spare_int); 1606 1607 for (i = 0; i < timeout; i++) { 1608 tmp = readl(scratch_reg1); 1609 if (!(tmp & AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK)) 1610 break; 1611 udelay(10); 1612 } 1613 1614 tmp = readl(scratch_reg1); 1615 if (i >= timeout || (tmp & AMDGPU_RLCG_SCRATCH1_ERROR_MASK) != 0) { 1616 if (amdgpu_sriov_rlcg_error_report_enabled(adev)) { 1617 if (tmp & AMDGPU_RLCG_VFGATE_DISABLED) { 1618 dev_err(adev->dev, 1619 "vfgate is disabled, rlcg failed to program reg: 0x%05x\n", offset); 1620 } else if (tmp & AMDGPU_RLCG_WRONG_OPERATION_TYPE) { 1621 dev_err(adev->dev, 1622 "wrong operation type, rlcg failed to program reg: 0x%05x\n", offset); 1623 } else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) { 1624 dev_err(adev->dev, 1625 "register is not in range, rlcg failed to program reg: 0x%05x\n", offset); 1626 } else { 1627 dev_err(adev->dev, 1628 "unknown error type, rlcg failed to program reg: 0x%05x\n", offset); 1629 } 1630 } else { 1631 dev_err(adev->dev, 1632 "timeout: rlcg faled to program reg: 0x%05x\n", offset); 1633 } 1634 } 1635 } 1636 1637 ret = readl(scratch_reg0); 1638 1639 spin_unlock_irqrestore(&adev->virt.rlcg_reg_lock, flags); 1640 1641 return ret; 1642 } 1643 1644 void amdgpu_sriov_wreg(struct amdgpu_device *adev, 1645 u32 offset, u32 value, 1646 u32 acc_flags, u32 hwip, u32 xcc_id) 1647 { 1648 u32 rlcg_flag; 1649 1650 if (amdgpu_device_skip_hw_access(adev)) 1651 return; 1652 1653 if (!amdgpu_sriov_runtime(adev) && 1654 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, true, &rlcg_flag)) { 1655 amdgpu_virt_rlcg_reg_rw(adev, offset, value, rlcg_flag, xcc_id); 1656 return; 1657 } 1658 1659 if (acc_flags & AMDGPU_REGS_NO_KIQ) 1660 WREG32_NO_KIQ(offset, value); 1661 else 1662 WREG32(offset, value); 1663 } 1664 1665 u32 amdgpu_sriov_rreg(struct amdgpu_device *adev, 1666 u32 offset, u32 acc_flags, u32 hwip, u32 xcc_id) 1667 { 1668 u32 rlcg_flag; 1669 1670 if (amdgpu_device_skip_hw_access(adev)) 1671 return 0; 1672 1673 if (!amdgpu_sriov_runtime(adev) && 1674 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, false, &rlcg_flag)) 1675 return amdgpu_virt_rlcg_reg_rw(adev, offset, 0, rlcg_flag, xcc_id); 1676 1677 if (acc_flags & AMDGPU_REGS_NO_KIQ) 1678 return RREG32_NO_KIQ(offset); 1679 else 1680 return RREG32(offset); 1681 } 1682 1683 bool amdgpu_sriov_xnack_support(struct amdgpu_device *adev) 1684 { 1685 bool xnack_mode = true; 1686 1687 if (amdgpu_sriov_vf(adev) && 1688 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 1689 xnack_mode = false; 1690 1691 return xnack_mode; 1692 } 1693 1694 bool amdgpu_virt_get_ras_capability(struct amdgpu_device *adev) 1695 { 1696 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1697 1698 if (!amdgpu_sriov_ras_caps_en(adev)) 1699 return false; 1700 1701 if (adev->virt.ras_en_caps.bits.block_umc) 1702 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__UMC); 1703 if (adev->virt.ras_en_caps.bits.block_sdma) 1704 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SDMA); 1705 if (adev->virt.ras_en_caps.bits.block_gfx) 1706 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__GFX); 1707 if (adev->virt.ras_en_caps.bits.block_mmhub) 1708 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MMHUB); 1709 if (adev->virt.ras_en_caps.bits.block_athub) 1710 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__ATHUB); 1711 if (adev->virt.ras_en_caps.bits.block_pcie_bif) 1712 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__PCIE_BIF); 1713 if (adev->virt.ras_en_caps.bits.block_hdp) 1714 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__HDP); 1715 if (adev->virt.ras_en_caps.bits.block_xgmi_wafl) 1716 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__XGMI_WAFL); 1717 if (adev->virt.ras_en_caps.bits.block_df) 1718 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__DF); 1719 if (adev->virt.ras_en_caps.bits.block_smn) 1720 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SMN); 1721 if (adev->virt.ras_en_caps.bits.block_sem) 1722 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SEM); 1723 if (adev->virt.ras_en_caps.bits.block_mp0) 1724 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP0); 1725 if (adev->virt.ras_en_caps.bits.block_mp1) 1726 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP1); 1727 if (adev->virt.ras_en_caps.bits.block_fuse) 1728 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__FUSE); 1729 if (adev->virt.ras_en_caps.bits.block_mca) 1730 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MCA); 1731 if (adev->virt.ras_en_caps.bits.block_vcn) 1732 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__VCN); 1733 if (adev->virt.ras_en_caps.bits.block_jpeg) 1734 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__JPEG); 1735 if (adev->virt.ras_en_caps.bits.block_ih) 1736 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__IH); 1737 if (adev->virt.ras_en_caps.bits.block_mpio) 1738 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MPIO); 1739 1740 if (adev->virt.ras_en_caps.bits.poison_propogation_mode) 1741 con->poison_supported = true; /* Poison is handled by host */ 1742 1743 if (adev->virt.ras_en_caps.bits.uniras_supported) 1744 amdgpu_virt_ras_set_remote_uniras(adev, true); 1745 1746 return true; 1747 } 1748 1749 static inline enum amd_sriov_ras_telemetry_gpu_block 1750 amdgpu_ras_block_to_sriov(struct amdgpu_device *adev, enum amdgpu_ras_block block) { 1751 switch (block) { 1752 case AMDGPU_RAS_BLOCK__UMC: 1753 return RAS_TELEMETRY_GPU_BLOCK_UMC; 1754 case AMDGPU_RAS_BLOCK__SDMA: 1755 return RAS_TELEMETRY_GPU_BLOCK_SDMA; 1756 case AMDGPU_RAS_BLOCK__GFX: 1757 return RAS_TELEMETRY_GPU_BLOCK_GFX; 1758 case AMDGPU_RAS_BLOCK__MMHUB: 1759 return RAS_TELEMETRY_GPU_BLOCK_MMHUB; 1760 case AMDGPU_RAS_BLOCK__ATHUB: 1761 return RAS_TELEMETRY_GPU_BLOCK_ATHUB; 1762 case AMDGPU_RAS_BLOCK__PCIE_BIF: 1763 return RAS_TELEMETRY_GPU_BLOCK_PCIE_BIF; 1764 case AMDGPU_RAS_BLOCK__HDP: 1765 return RAS_TELEMETRY_GPU_BLOCK_HDP; 1766 case AMDGPU_RAS_BLOCK__XGMI_WAFL: 1767 return RAS_TELEMETRY_GPU_BLOCK_XGMI_WAFL; 1768 case AMDGPU_RAS_BLOCK__DF: 1769 return RAS_TELEMETRY_GPU_BLOCK_DF; 1770 case AMDGPU_RAS_BLOCK__SMN: 1771 return RAS_TELEMETRY_GPU_BLOCK_SMN; 1772 case AMDGPU_RAS_BLOCK__SEM: 1773 return RAS_TELEMETRY_GPU_BLOCK_SEM; 1774 case AMDGPU_RAS_BLOCK__MP0: 1775 return RAS_TELEMETRY_GPU_BLOCK_MP0; 1776 case AMDGPU_RAS_BLOCK__MP1: 1777 return RAS_TELEMETRY_GPU_BLOCK_MP1; 1778 case AMDGPU_RAS_BLOCK__FUSE: 1779 return RAS_TELEMETRY_GPU_BLOCK_FUSE; 1780 case AMDGPU_RAS_BLOCK__MCA: 1781 return RAS_TELEMETRY_GPU_BLOCK_MCA; 1782 case AMDGPU_RAS_BLOCK__VCN: 1783 return RAS_TELEMETRY_GPU_BLOCK_VCN; 1784 case AMDGPU_RAS_BLOCK__JPEG: 1785 return RAS_TELEMETRY_GPU_BLOCK_JPEG; 1786 case AMDGPU_RAS_BLOCK__IH: 1787 return RAS_TELEMETRY_GPU_BLOCK_IH; 1788 case AMDGPU_RAS_BLOCK__MPIO: 1789 return RAS_TELEMETRY_GPU_BLOCK_MPIO; 1790 default: 1791 dev_warn(adev->dev, "Unsupported SRIOV RAS telemetry block 0x%x\n", 1792 block); 1793 return RAS_TELEMETRY_GPU_BLOCK_COUNT; 1794 } 1795 } 1796 1797 static int amdgpu_virt_cache_host_error_counts(struct amdgpu_device *adev, 1798 struct amdsriov_ras_telemetry *host_telemetry) 1799 { 1800 struct amd_sriov_ras_telemetry_error_count *tmp = NULL; 1801 uint32_t checksum, used_size; 1802 1803 checksum = host_telemetry->header.checksum; 1804 used_size = host_telemetry->header.used_size; 1805 1806 if (used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 1807 return 0; 1808 1809 tmp = kmemdup(&host_telemetry->body.error_count, used_size, GFP_KERNEL); 1810 if (!tmp) 1811 return -ENOMEM; 1812 1813 if (checksum != amd_sriov_msg_checksum(tmp, used_size, 0, 0)) 1814 goto out; 1815 1816 memcpy(&adev->virt.count_cache, tmp, 1817 min(used_size, sizeof(adev->virt.count_cache))); 1818 out: 1819 kfree(tmp); 1820 1821 return 0; 1822 } 1823 1824 static int amdgpu_virt_req_ras_err_count_internal(struct amdgpu_device *adev, bool force_update) 1825 { 1826 struct amdgpu_virt *virt = &adev->virt; 1827 1828 if (!virt->ops || !virt->ops->req_ras_err_count) 1829 return -EOPNOTSUPP; 1830 1831 /* Host allows 15 ras telemetry requests per 60 seconds. Afterwhich, the Host 1832 * will ignore incoming guest messages. Ratelimit the guest messages to 1833 * prevent guest self DOS. 1834 */ 1835 if (__ratelimit(&virt->ras.ras_error_cnt_rs) || force_update) { 1836 mutex_lock(&virt->ras.ras_telemetry_mutex); 1837 if (!virt->ops->req_ras_err_count(adev)) 1838 amdgpu_virt_cache_host_error_counts(adev, 1839 virt->fw_reserve.ras_telemetry); 1840 mutex_unlock(&virt->ras.ras_telemetry_mutex); 1841 } 1842 1843 return 0; 1844 } 1845 1846 /* Bypass ACA interface and query ECC counts directly from host */ 1847 int amdgpu_virt_req_ras_err_count(struct amdgpu_device *adev, enum amdgpu_ras_block block, 1848 struct ras_err_data *err_data) 1849 { 1850 enum amd_sriov_ras_telemetry_gpu_block sriov_block; 1851 1852 sriov_block = amdgpu_ras_block_to_sriov(adev, block); 1853 1854 if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT || 1855 !amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block)) 1856 return -EOPNOTSUPP; 1857 1858 /* Host Access may be lost during reset, just return last cached data. */ 1859 if (down_read_trylock(&adev->reset_domain->sem)) { 1860 amdgpu_virt_req_ras_err_count_internal(adev, false); 1861 up_read(&adev->reset_domain->sem); 1862 } 1863 1864 err_data->ue_count = adev->virt.count_cache.block[sriov_block].ue_count; 1865 err_data->ce_count = adev->virt.count_cache.block[sriov_block].ce_count; 1866 err_data->de_count = adev->virt.count_cache.block[sriov_block].de_count; 1867 1868 return 0; 1869 } 1870 1871 static int 1872 amdgpu_virt_write_cpers_to_ring(struct amdgpu_device *adev, 1873 struct amdsriov_ras_telemetry *host_telemetry, 1874 u32 *more) 1875 { 1876 struct amd_sriov_ras_cper_dump *cper_dump = NULL; 1877 struct cper_hdr *entry = NULL; 1878 struct amdgpu_ring *ring = &adev->cper.ring_buf; 1879 uint32_t checksum, used_size; 1880 u64 remaining, cnt, i; 1881 int ret = 0; 1882 1883 checksum = host_telemetry->header.checksum; 1884 used_size = host_telemetry->header.used_size; 1885 1886 if (used_size < offsetof(struct amd_sriov_ras_cper_dump, buf) || 1887 used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 1888 return -EINVAL; 1889 1890 cper_dump = kmemdup(&host_telemetry->body.cper_dump, used_size, GFP_KERNEL); 1891 if (!cper_dump) 1892 return -ENOMEM; 1893 1894 if (checksum != amd_sriov_msg_checksum(cper_dump, used_size, 0, 0)) { 1895 ret = -EINVAL; 1896 goto out; 1897 } 1898 1899 *more = cper_dump->more; 1900 1901 if (cper_dump->wptr < adev->virt.ras.cper_rptr) { 1902 dev_warn( 1903 adev->dev, 1904 "guest specified rptr that was too high! guest rptr: 0x%llx, host rptr: 0x%llx\n", 1905 adev->virt.ras.cper_rptr, cper_dump->wptr); 1906 1907 adev->virt.ras.cper_rptr = cper_dump->wptr; 1908 goto out; 1909 } 1910 1911 entry = (struct cper_hdr *)&cper_dump->buf[0]; 1912 remaining = (u64)used_size - offsetof(struct amd_sriov_ras_cper_dump, buf); 1913 cnt = min_t(u64, cper_dump->count, CPER_MAX_ALLOWED_COUNT); 1914 1915 for (i = 0; i < cnt; i++) { 1916 if (entry->record_length < sizeof(struct cper_hdr) || 1917 entry->record_length > remaining) { 1918 ret = -EINVAL; 1919 goto out; 1920 } 1921 1922 amdgpu_cper_ring_write(ring, entry, entry->record_length); 1923 remaining -= entry->record_length; 1924 entry = (struct cper_hdr *)((char *)entry + entry->record_length); 1925 } 1926 1927 if (cper_dump->overflow_count) 1928 dev_warn(adev->dev, 1929 "host reported CPER overflow of 0x%llx entries!\n", 1930 cper_dump->overflow_count); 1931 1932 adev->virt.ras.cper_rptr = cper_dump->wptr; 1933 out: 1934 kfree(cper_dump); 1935 1936 return ret; 1937 } 1938 1939 static int amdgpu_virt_req_ras_cper_dump_internal(struct amdgpu_device *adev) 1940 { 1941 struct amdgpu_virt *virt = &adev->virt; 1942 int ret = 0; 1943 uint32_t more = 0; 1944 1945 if (!virt->ops || !virt->ops->req_ras_cper_dump) 1946 return -EOPNOTSUPP; 1947 1948 do { 1949 if (!virt->ops->req_ras_cper_dump(adev, virt->ras.cper_rptr)) 1950 ret = amdgpu_virt_write_cpers_to_ring( 1951 adev, virt->fw_reserve.ras_telemetry, &more); 1952 else 1953 ret = 0; 1954 } while (more && !ret); 1955 1956 return ret; 1957 } 1958 1959 int amdgpu_virt_req_ras_cper_dump(struct amdgpu_device *adev, bool force_update) 1960 { 1961 struct amdgpu_virt *virt = &adev->virt; 1962 int ret = 0; 1963 1964 if (!amdgpu_sriov_ras_cper_en(adev)) 1965 return -EOPNOTSUPP; 1966 1967 if ((__ratelimit(&virt->ras.ras_cper_dump_rs) || force_update) && 1968 down_read_trylock(&adev->reset_domain->sem)) { 1969 mutex_lock(&virt->ras.ras_telemetry_mutex); 1970 ret = amdgpu_virt_req_ras_cper_dump_internal(adev); 1971 mutex_unlock(&virt->ras.ras_telemetry_mutex); 1972 up_read(&adev->reset_domain->sem); 1973 } 1974 1975 return ret; 1976 } 1977 1978 int amdgpu_virt_ras_telemetry_post_reset(struct amdgpu_device *adev) 1979 { 1980 unsigned long ue_count, ce_count; 1981 1982 if (amdgpu_sriov_ras_telemetry_en(adev)) { 1983 amdgpu_virt_req_ras_err_count_internal(adev, true); 1984 amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL); 1985 } 1986 1987 return 0; 1988 } 1989 1990 bool amdgpu_virt_ras_telemetry_block_en(struct amdgpu_device *adev, 1991 enum amdgpu_ras_block block) 1992 { 1993 enum amd_sriov_ras_telemetry_gpu_block sriov_block; 1994 1995 sriov_block = amdgpu_ras_block_to_sriov(adev, block); 1996 1997 if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT || 1998 !amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block)) 1999 return false; 2000 2001 return true; 2002 } 2003 2004 /* 2005 * amdgpu_virt_request_bad_pages() - request bad pages 2006 * @adev: amdgpu device. 2007 * Send command to GPU hypervisor to write new bad pages into the shared PF2VF region 2008 */ 2009 void amdgpu_virt_request_bad_pages(struct amdgpu_device *adev) 2010 { 2011 struct amdgpu_virt *virt = &adev->virt; 2012 2013 if (virt->ops && virt->ops->req_bad_pages) 2014 virt->ops->req_bad_pages(adev); 2015 } 2016 2017 static int amdgpu_virt_cache_chk_criti_hit(struct amdgpu_device *adev, 2018 struct amdsriov_ras_telemetry *host_telemetry, 2019 bool *hit) 2020 { 2021 struct amd_sriov_ras_chk_criti *tmp = NULL; 2022 uint32_t checksum, used_size; 2023 2024 checksum = host_telemetry->header.checksum; 2025 used_size = host_telemetry->header.used_size; 2026 2027 if (used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 2028 return 0; 2029 2030 tmp = kmemdup(&host_telemetry->body.chk_criti, used_size, GFP_KERNEL); 2031 if (!tmp) 2032 return -ENOMEM; 2033 2034 if (checksum != amd_sriov_msg_checksum(tmp, used_size, 0, 0)) 2035 goto out; 2036 2037 if (hit) 2038 *hit = tmp->hit ? true : false; 2039 2040 out: 2041 kfree(tmp); 2042 2043 return 0; 2044 } 2045 2046 int amdgpu_virt_check_vf_critical_region(struct amdgpu_device *adev, u64 addr, bool *hit) 2047 { 2048 struct amdgpu_virt *virt = &adev->virt; 2049 int r = -EPERM; 2050 2051 if (!virt->ops || !virt->ops->req_ras_chk_criti) 2052 return -EOPNOTSUPP; 2053 2054 /* Host allows 15 ras telemetry requests per 60 seconds. Afterwhich, the Host 2055 * will ignore incoming guest messages. Ratelimit the guest messages to 2056 * prevent guest self DOS. 2057 */ 2058 if (__ratelimit(&virt->ras.ras_chk_criti_rs)) { 2059 mutex_lock(&virt->ras.ras_telemetry_mutex); 2060 if (!virt->ops->req_ras_chk_criti(adev, addr)) 2061 r = amdgpu_virt_cache_chk_criti_hit( 2062 adev, virt->fw_reserve.ras_telemetry, hit); 2063 mutex_unlock(&virt->ras.ras_telemetry_mutex); 2064 } 2065 2066 return r; 2067 } 2068 2069 static int req_remote_ras_cmd(struct amdgpu_device *adev, 2070 u32 param1, u32 param2, u32 param3) 2071 { 2072 struct amdgpu_virt *virt = &adev->virt; 2073 2074 if (virt->ops && virt->ops->req_remote_ras_cmd) 2075 return virt->ops->req_remote_ras_cmd(adev, param1, param2, param3); 2076 return -ENOENT; 2077 } 2078 2079 int amdgpu_virt_send_remote_ras_cmd(struct amdgpu_device *adev, 2080 uint64_t buf, uint32_t buf_len) 2081 { 2082 uint64_t gpa = buf; 2083 int ret = -EIO; 2084 2085 if (down_read_trylock(&adev->reset_domain->sem)) { 2086 ret = req_remote_ras_cmd(adev, 2087 lower_32_bits(gpa), upper_32_bits(gpa), buf_len); 2088 up_read(&adev->reset_domain->sem); 2089 } 2090 2091 return ret; 2092 } 2093 2094 int amdgpu_virt_ptl_request(struct amdgpu_device *adev, u32 req_code, 2095 uint32_t *ptl_state, uint32_t *fmt1, uint32_t *fmt2) 2096 { 2097 int ret; 2098 2099 if (!ptl_state || !fmt1 || !fmt2) 2100 return -EINVAL; 2101 2102 if (!amdgpu_sriov_ptl_support(adev) || 2103 !adev->virt.ops || !adev->virt.ops->req_ptl_update) 2104 return -EOPNOTSUPP; 2105 2106 if (req_code == PSP_PTL_PERF_MON_SET && *ptl_state) { 2107 if (*fmt1 == *fmt2) { 2108 dev_warn(adev->dev, 2109 "PTL formats must be different (fmt1=%u, fmt2=%u)\n", 2110 *fmt1, *fmt2); 2111 return -EINVAL; 2112 } 2113 } 2114 2115 ret = adev->virt.ops->req_ptl_update(adev, req_code, *ptl_state, *fmt1, *fmt2); 2116 if (ret) { 2117 dev_warn(adev->dev, "VF PTL update request failed: %d\n", ret); 2118 return ret; 2119 } 2120 2121 if (req_code == PSP_PTL_PERF_MON_QUERY) { 2122 *ptl_state = adev->virt.ptl_state; 2123 *fmt1 = adev->virt.ptl_pref_format1; 2124 *fmt2 = adev->virt.ptl_pref_format2; 2125 } 2126 2127 return 0; 2128 } 2129