1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/module.h> 25 26 #ifdef CONFIG_X86 27 #include <asm/hypervisor.h> 28 #endif 29 30 #include <drm/drm_drv.h> 31 #include <xen/xen.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_dpm.h" 37 #include "amdgpu_video_codecs.h" 38 #include "vi.h" 39 #include "soc15.h" 40 #include "nv.h" 41 #include "amdgpu_virt_ras_cmd.h" 42 43 #define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \ 44 do { \ 45 vf2pf_info->ucode_info[ucode].id = ucode; \ 46 vf2pf_info->ucode_info[ucode].version = ver; \ 47 } while (0) 48 49 #define mmRCC_CONFIG_MEMSIZE 0xde3 50 #define mmRCC_HW_DEBUG 0x10c01 51 52 const char *amdgpu_virt_dynamic_crit_table_name[] = { 53 "IP DISCOVERY", 54 "VBIOS IMG", 55 "RAS TELEMETRY", 56 "DATA EXCHANGE", 57 "BAD PAGE INFO", 58 "INIT HEADER", 59 "LAST", 60 }; 61 62 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) 63 { 64 /* By now all MMIO pages except mailbox are blocked */ 65 /* if blocking is enabled in hypervisor. Choose the */ 66 /* SCRATCH_REG0 to test. */ 67 return RREG32_NO_KIQ(0xc040) == 0xffffffff; 68 } 69 70 void amdgpu_virt_init_setting(struct amdgpu_device *adev) 71 { 72 struct drm_device *ddev = adev_to_drm(adev); 73 74 /* enable virtual display */ 75 if (adev->asic_type != CHIP_ALDEBARAN && 76 adev->asic_type != CHIP_ARCTURUS && 77 ((adev->pdev->class >> 8) != PCI_CLASS_ACCELERATOR_PROCESSING)) { 78 if (adev->mode_info.num_crtc == 0) 79 adev->mode_info.num_crtc = 1; 80 adev->enable_virtual_display = true; 81 } 82 ddev->driver_features &= ~DRIVER_ATOMIC; 83 adev->cg_flags = 0; 84 adev->pg_flags = 0; 85 86 /* Reduce kcq number to 2 to reduce latency */ 87 if (amdgpu_num_kcq == -1) 88 amdgpu_num_kcq = 2; 89 } 90 91 /** 92 * amdgpu_virt_request_full_gpu() - request full gpu access 93 * @adev: amdgpu device. 94 * @init: is driver init time. 95 * When start to init/fini driver, first need to request full gpu access. 96 * Return: Zero if request success, otherwise will return error. 97 */ 98 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init) 99 { 100 struct amdgpu_virt *virt = &adev->virt; 101 int r; 102 103 if (virt->ops && virt->ops->req_full_gpu) { 104 r = virt->ops->req_full_gpu(adev, init); 105 if (r) { 106 adev->no_hw_access = true; 107 return r; 108 } 109 110 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 111 } 112 113 return 0; 114 } 115 116 /** 117 * amdgpu_virt_release_full_gpu() - release full gpu access 118 * @adev: amdgpu device. 119 * @init: is driver init time. 120 * When finishing driver init/fini, need to release full gpu access. 121 * Return: Zero if release success, otherwise will returen error. 122 */ 123 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init) 124 { 125 struct amdgpu_virt *virt = &adev->virt; 126 int r; 127 128 if (virt->ops && virt->ops->rel_full_gpu) { 129 r = virt->ops->rel_full_gpu(adev, init); 130 if (r) 131 return r; 132 133 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 134 } 135 return 0; 136 } 137 138 /** 139 * amdgpu_virt_reset_gpu() - reset gpu 140 * @adev: amdgpu device. 141 * Send reset command to GPU hypervisor to reset GPU that VM is using 142 * Return: Zero if reset success, otherwise will return error. 143 */ 144 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev) 145 { 146 struct amdgpu_virt *virt = &adev->virt; 147 int r; 148 149 if (virt->ops && virt->ops->reset_gpu) { 150 r = virt->ops->reset_gpu(adev); 151 if (r) 152 return r; 153 154 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 155 } 156 157 return 0; 158 } 159 160 void amdgpu_virt_request_init_data(struct amdgpu_device *adev) 161 { 162 struct amdgpu_virt *virt = &adev->virt; 163 164 if (virt->ops && virt->ops->req_init_data) 165 virt->ops->req_init_data(adev); 166 167 if (adev->virt.req_init_data_ver > 0) 168 dev_info(adev->dev, "host supports REQ_INIT_DATA handshake of critical_region_version %d\n", 169 adev->virt.req_init_data_ver); 170 else 171 dev_warn(adev->dev, "host doesn't support REQ_INIT_DATA handshake\n"); 172 } 173 174 /** 175 * amdgpu_virt_ready_to_reset() - send ready to reset to host 176 * @adev: amdgpu device. 177 * Send ready to reset message to GPU hypervisor to signal we have stopped GPU 178 * activity and is ready for host FLR 179 */ 180 void amdgpu_virt_ready_to_reset(struct amdgpu_device *adev) 181 { 182 struct amdgpu_virt *virt = &adev->virt; 183 184 if (virt->ops && virt->ops->reset_gpu) 185 virt->ops->ready_to_reset(adev); 186 } 187 188 /** 189 * amdgpu_virt_wait_reset() - wait for reset gpu completed 190 * @adev: amdgpu device. 191 * Wait for GPU reset completed. 192 * Return: Zero if reset success, otherwise will return error. 193 */ 194 int amdgpu_virt_wait_reset(struct amdgpu_device *adev) 195 { 196 struct amdgpu_virt *virt = &adev->virt; 197 198 if (!virt->ops || !virt->ops->wait_reset) 199 return -EINVAL; 200 201 return virt->ops->wait_reset(adev); 202 } 203 204 /** 205 * amdgpu_virt_alloc_mm_table() - alloc memory for mm table 206 * @adev: amdgpu device. 207 * MM table is used by UVD and VCE for its initialization 208 * Return: Zero if allocate success. 209 */ 210 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev) 211 { 212 int r; 213 214 if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr) 215 return 0; 216 217 r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 218 AMDGPU_GEM_DOMAIN_VRAM | 219 AMDGPU_GEM_DOMAIN_GTT, 220 &adev->virt.mm_table.bo, 221 &adev->virt.mm_table.gpu_addr, 222 (void *)&adev->virt.mm_table.cpu_addr); 223 if (r) { 224 dev_err(adev->dev, "failed to alloc mm table and error = %d.\n", r); 225 return r; 226 } 227 228 memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE); 229 dev_info(adev->dev, "MM table gpu addr = 0x%llx, cpu addr = %p.\n", 230 adev->virt.mm_table.gpu_addr, 231 adev->virt.mm_table.cpu_addr); 232 return 0; 233 } 234 235 /** 236 * amdgpu_virt_free_mm_table() - free mm table memory 237 * @adev: amdgpu device. 238 * Free MM table memory 239 */ 240 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev) 241 { 242 if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr) 243 return; 244 245 amdgpu_bo_free_kernel(&adev->virt.mm_table.bo, 246 &adev->virt.mm_table.gpu_addr, 247 (void *)&adev->virt.mm_table.cpu_addr); 248 adev->virt.mm_table.gpu_addr = 0; 249 } 250 251 /** 252 * amdgpu_virt_rcvd_ras_interrupt() - receive ras interrupt 253 * @adev: amdgpu device. 254 * Check whether host sent RAS error message 255 * Return: true if found, otherwise false 256 */ 257 bool amdgpu_virt_rcvd_ras_interrupt(struct amdgpu_device *adev) 258 { 259 struct amdgpu_virt *virt = &adev->virt; 260 261 if (!virt->ops || !virt->ops->rcvd_ras_intr) 262 return false; 263 264 return virt->ops->rcvd_ras_intr(adev); 265 } 266 267 268 unsigned int amd_sriov_msg_checksum(void *obj, 269 unsigned long obj_size, 270 unsigned int key, 271 unsigned int checksum) 272 { 273 unsigned int ret = key; 274 unsigned long i = 0; 275 unsigned char *pos; 276 277 pos = (char *)obj; 278 /* calculate checksum */ 279 for (i = 0; i < obj_size; ++i) 280 ret += *(pos + i); 281 /* minus the checksum itself */ 282 pos = (char *)&checksum; 283 for (i = 0; i < sizeof(checksum); ++i) 284 ret -= *(pos + i); 285 return ret; 286 } 287 288 #define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY 512 289 /* Max bad page slots allowed for SRIOV*/ 290 #define AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY 10665U 291 292 /** 293 * amdgpu_virt_ras_realloc_eh_data_space - alloc/realloc VF bad-page @data->bps and @data->bps_bo 294 * @adev: amdgpu device 295 * @data: VF RAS error-handler data 296 * @pages: minimum number of new slots to add beyond @data->capacity 297 * 298 * Return: 0 on success, %-ENOMEM on failure. 299 */ 300 static int amdgpu_virt_ras_realloc_eh_data_space(struct amdgpu_device *adev, 301 struct amdgpu_virt_ras_err_handler_data *data, 302 int pages) 303 { 304 struct eeprom_table_record *new_bps; 305 struct amdgpu_bo **new_bo; 306 unsigned int old_space; 307 unsigned int new_space; 308 unsigned int align_space; 309 310 old_space = (unsigned int)data->capacity; 311 new_space = old_space + max_t(unsigned int, (unsigned int)pages, 312 (unsigned int)AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); 313 if (new_space < old_space || new_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) 314 return -ENOMEM; 315 316 align_space = ALIGN(new_space, AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY); 317 if (align_space > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY) 318 return -ENOMEM; 319 320 new_bps = kmalloc_objs(*data->bps, align_space); 321 new_bo = kzalloc_objs(*data->bps_bo, align_space); 322 if (!new_bps || !new_bo) { 323 kfree(new_bps); 324 kfree(new_bo); 325 dev_warn_ratelimited(adev->dev, 326 "RAS WARN: failed to grow bad page table to %u slots\n", 327 align_space); 328 return -ENOMEM; 329 } 330 331 memcpy(new_bps, data->bps, data->count * sizeof(*data->bps)); 332 memcpy(new_bo, data->bps_bo, data->count * sizeof(*data->bps_bo)); 333 334 kfree(data->bps); 335 kfree(data->bps_bo); 336 data->bps = new_bps; 337 data->bps_bo = new_bo; 338 data->capacity = (int)align_space; 339 340 return 0; 341 } 342 343 int amdgpu_virt_get_uniras_ras_caps(struct amdgpu_device *adev, 344 struct amd_sriov_uniras_caps *caps) 345 { 346 struct amd_sriov_msg_pf2vf_info *pf2vf_msg; 347 348 if (!adev || !caps) 349 return -EINVAL; 350 351 if (!adev->virt.fw_reserve.p_pf2vf) 352 return -EINVAL; 353 354 if (adev->virt.fw_reserve.p_pf2vf->version != AMD_SRIOV_MSG_FW_VRAM_PF2VF_VER) 355 return -EINVAL; 356 357 pf2vf_msg = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf; 358 359 if (pf2vf_msg->header.size > (AMD_SRIOV_MSG_SIZE_KB << 10) || 360 pf2vf_msg->header.size < 361 offsetof(struct amd_sriov_msg_pf2vf_info, pf2vf_ras_caps) + 362 sizeof(pf2vf_msg->pf2vf_ras_caps)) 363 return -EINVAL; 364 365 if (!pf2vf_msg->feature_flags.flags.uniras_support) 366 return -EOPNOTSUPP; 367 368 *caps = pf2vf_msg->pf2vf_ras_caps.uniras_caps; 369 370 dev_dbg(adev->dev, 371 "uniras caps: ras_ext_ecc_type=0x%x ras_int_ecc_attributes=0x%x ras_en_block_mask=0x%llx\n", 372 caps->ras_ext_ecc_type, caps->ras_int_ecc_attributes, 373 caps->ras_en_block_mask); 374 375 return 0; 376 } 377 378 static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev) 379 { 380 struct amdgpu_virt *virt = &adev->virt; 381 struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data; 382 unsigned int align_space = AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_INIT_CAPACITY; 383 void *bps = NULL; 384 struct amdgpu_bo **bps_bo = NULL; 385 386 *data = kmalloc_obj(struct amdgpu_virt_ras_err_handler_data); 387 if (!*data) 388 goto data_failure; 389 390 bps = kmalloc_objs(*(*data)->bps, align_space); 391 if (!bps) 392 goto bps_failure; 393 394 bps_bo = kzalloc_objs(*(*data)->bps_bo, align_space); 395 if (!bps_bo) 396 goto bps_bo_failure; 397 398 (*data)->bps = bps; 399 (*data)->bps_bo = bps_bo; 400 (*data)->capacity = align_space; 401 (*data)->count = 0; 402 (*data)->last_reserved = 0; 403 404 virt->ras_init_done = true; 405 406 return 0; 407 408 bps_bo_failure: 409 kfree(bps); 410 bps_failure: 411 kfree(*data); 412 data_failure: 413 return -ENOMEM; 414 } 415 416 static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev) 417 { 418 struct amdgpu_virt *virt = &adev->virt; 419 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 420 struct amdgpu_bo *bo; 421 int i; 422 423 if (!data) 424 return; 425 426 for (i = data->last_reserved - 1; i >= 0; i--) { 427 bo = data->bps_bo[i]; 428 if (bo) { 429 amdgpu_bo_free_kernel(&bo, NULL, NULL); 430 data->bps_bo[i] = bo; 431 } 432 data->last_reserved = i; 433 } 434 } 435 436 void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev) 437 { 438 struct amdgpu_virt *virt = &adev->virt; 439 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 440 441 virt->ras_init_done = false; 442 443 if (!data) 444 return; 445 446 amdgpu_virt_ras_release_bp(adev); 447 448 kfree(data->bps); 449 kfree(data->bps_bo); 450 kfree(data); 451 virt->virt_eh_data = NULL; 452 } 453 454 static bool amdgpu_virt_ras_add_bps(struct amdgpu_device *adev, 455 const struct eeprom_table_record *bps, int pages) 456 { 457 struct amdgpu_virt *virt = &adev->virt; 458 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 459 int need; 460 461 if (!data || pages <= 0) 462 return false; 463 464 if (pages > AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY - data->count) { 465 dev_warn_ratelimited(adev->dev, 466 "RAS WARN: bad page table at capacity (count=%d pages=%d max=%u)\n", 467 data->count, pages, 468 AMDGPU_VIRT_RAS_BAD_PAGE_TABLE_MAX_CAPACITY); 469 return false; 470 } 471 472 need = data->count + pages; 473 if (need > data->capacity && 474 amdgpu_virt_ras_realloc_eh_data_space(adev, data, need - data->capacity)) 475 return false; 476 477 memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps)); 478 data->count += pages; 479 480 return true; 481 } 482 483 static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) 484 { 485 struct amdgpu_virt *virt = &adev->virt; 486 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 487 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 488 struct ttm_resource_manager *man = &mgr->manager; 489 struct amdgpu_bo *bo = NULL; 490 uint64_t bp; 491 int i; 492 493 if (!data) 494 return; 495 496 for (i = data->last_reserved; i < data->count; i++) { 497 bp = data->bps[i].retired_page; 498 499 /* There are two cases of reserve error should be ignored: 500 * 1) a ras bad page has been allocated (used by someone); 501 * 2) a ras bad page has been reserved (duplicate error injection 502 * for one page); 503 */ 504 if (ttm_resource_manager_used(man)) { 505 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr, 506 bp << AMDGPU_GPU_PAGE_SHIFT, 507 AMDGPU_GPU_PAGE_SIZE); 508 data->bps_bo[i] = NULL; 509 } else { 510 if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT, 511 AMDGPU_GPU_PAGE_SIZE, 512 &bo, NULL)) 513 dev_dbg(adev->dev, 514 "RAS WARN: reserve vram for retired page %llx fail\n", 515 bp); 516 data->bps_bo[i] = bo; 517 } 518 data->last_reserved = i + 1; 519 bo = NULL; 520 } 521 } 522 523 static bool amdgpu_virt_ras_check_bad_page(struct amdgpu_device *adev, 524 uint64_t retired_page) 525 { 526 struct amdgpu_virt *virt = &adev->virt; 527 struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data; 528 int i; 529 530 if (!data) 531 return true; 532 533 for (i = 0; i < data->count; i++) 534 if (retired_page == data->bps[i].retired_page) 535 return true; 536 537 return false; 538 } 539 540 static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, 541 uint64_t bp_block_offset, uint32_t bp_block_size) 542 { 543 struct eeprom_table_record bp; 544 uint64_t retired_page; 545 uint32_t bp_idx, bp_cnt; 546 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 547 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 548 void *vram_usage_va = fw_va ? fw_va : drv_va; 549 550 memset(&bp, 0, sizeof(bp)); 551 552 if (!bp_block_size) 553 return; 554 555 bp_cnt = bp_block_size / sizeof(uint64_t); 556 for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) { 557 retired_page = *(uint64_t *)(vram_usage_va + 558 bp_block_offset + bp_idx * sizeof(uint64_t)); 559 bp.retired_page = retired_page; 560 561 if (amdgpu_virt_ras_check_bad_page(adev, retired_page)) 562 continue; 563 564 if (!amdgpu_virt_ras_add_bps(adev, &bp, 1)) 565 break; 566 567 amdgpu_virt_ras_reserve_bps(adev); 568 } 569 } 570 571 static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) 572 { 573 struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf; 574 struct amdgim_pf2vf_info_v1 *pf2vf_v1; 575 struct amd_sriov_msg_pf2vf_info *pf2vf; 576 577 uint32_t checksum; 578 uint32_t checkval; 579 580 uint32_t i; 581 uint32_t tmp; 582 583 if (adev->virt.fw_reserve.p_pf2vf == NULL) 584 return -EINVAL; 585 586 if (pf2vf_info->size > 1024) { 587 dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size); 588 return -EINVAL; 589 } 590 591 switch (pf2vf_info->version) { 592 case 1: 593 pf2vf_v1 = (struct amdgim_pf2vf_info_v1 *)pf2vf_info; 594 checksum = pf2vf_v1->checksum; 595 checkval = amd_sriov_msg_checksum( 596 adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 597 adev->virt.fw_reserve.checksum_key, checksum); 598 if (checksum != checkval) { 599 dev_err(adev->dev, 600 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", 601 checksum, checkval); 602 return -EINVAL; 603 } 604 605 adev->virt.gim_feature = pf2vf_v1->feature_flags; 606 break; 607 case 2: 608 /* TODO: missing key, need to add it later */ 609 pf2vf = (struct amd_sriov_msg_pf2vf_info *)pf2vf_info; 610 checksum = pf2vf->checksum; 611 checkval = amd_sriov_msg_checksum( 612 adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 613 0, checksum); 614 if (checksum != checkval) { 615 dev_err(adev->dev, 616 "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n", 617 checksum, checkval); 618 return -EINVAL; 619 } 620 621 adev->virt.vf2pf_update_interval_ms = 622 pf2vf->vf2pf_update_interval_ms; 623 adev->virt.gim_feature = pf2vf->feature_flags.all; 624 adev->virt.reg_access = pf2vf->reg_access_flags.all; 625 626 adev->virt.decode_max_dimension_pixels = 0; 627 adev->virt.decode_max_frame_pixels = 0; 628 adev->virt.encode_max_dimension_pixels = 0; 629 adev->virt.encode_max_frame_pixels = 0; 630 adev->virt.is_mm_bw_enabled = false; 631 for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) { 632 tmp = pf2vf->mm_bw_management[i].decode_max_dimension_pixels; 633 adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels); 634 635 tmp = pf2vf->mm_bw_management[i].decode_max_frame_pixels; 636 adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels); 637 638 tmp = pf2vf->mm_bw_management[i].encode_max_dimension_pixels; 639 adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels); 640 641 tmp = pf2vf->mm_bw_management[i].encode_max_frame_pixels; 642 adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels); 643 } 644 if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0)) 645 adev->virt.is_mm_bw_enabled = true; 646 647 adev->unique_id = pf2vf->uuid; 648 649 adev->unitid = 0; 650 if (amdgpu_sriov_is_unitid_support(adev)) 651 adev->unitid = pf2vf->unitid; 652 653 adev->virt.ras_en_caps.all = pf2vf->pf2vf_ras_caps.ras_en_caps.all; 654 adev->virt.ras_telemetry_en_caps.all = 655 pf2vf->pf2vf_ras_caps.ras_telemetry_en_caps.all; 656 657 adev->have_atomics_support = pf2vf->pcie_atomic_ops_support_flags == 658 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 659 break; 660 default: 661 dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version); 662 return -EINVAL; 663 } 664 665 /* correct too large or too little interval value */ 666 if (adev->virt.vf2pf_update_interval_ms < 200 || adev->virt.vf2pf_update_interval_ms > 10000) 667 adev->virt.vf2pf_update_interval_ms = 2000; 668 669 return 0; 670 } 671 672 static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) 673 { 674 struct amd_sriov_msg_vf2pf_info *vf2pf_info; 675 vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; 676 677 if (adev->virt.fw_reserve.p_vf2pf == NULL) 678 return; 679 680 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCE, adev->vce.fw_version); 681 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_UVD, adev->uvd.fw_version); 682 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MC, adev->gmc.fw_version); 683 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ME, adev->gfx.me_fw_version); 684 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_PFP, adev->gfx.pfp_fw_version); 685 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_CE, adev->gfx.ce_fw_version); 686 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC, adev->gfx.rlc_fw_version); 687 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLC, adev->gfx.rlc_srlc_fw_version); 688 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLG, adev->gfx.rlc_srlg_fw_version); 689 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version); 690 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); 691 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); 692 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); 693 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, 694 adev->psp.asd_context.bin_desc.fw_version); 695 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS, 696 adev->psp.ras_context.context.bin_desc.fw_version); 697 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI, 698 adev->psp.xgmi_context.context.bin_desc.fw_version); 699 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC, adev->pm.fw_version); 700 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA, adev->sdma.instance[0].fw_version); 701 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2, adev->sdma.instance[1].fw_version); 702 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCN, adev->vcn.fw_version); 703 POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_DMCU, adev->dm.dmcu_fw_version); 704 } 705 706 static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) 707 { 708 struct amd_sriov_msg_vf2pf_info *vf2pf_info; 709 710 vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf; 711 712 if (adev->virt.fw_reserve.p_vf2pf == NULL) 713 return -EINVAL; 714 715 memset(vf2pf_info, 0, sizeof(struct amd_sriov_msg_vf2pf_info)); 716 717 vf2pf_info->header.size = sizeof(struct amd_sriov_msg_vf2pf_info); 718 vf2pf_info->header.version = AMD_SRIOV_MSG_FW_VRAM_VF2PF_VER; 719 720 #ifdef MODULE 721 if (THIS_MODULE->version != NULL) 722 strscpy(vf2pf_info->driver_version, THIS_MODULE->version); 723 else 724 #endif 725 strscpy(vf2pf_info->driver_version, "N/A"); 726 727 vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all 728 vf2pf_info->driver_cert = 0; 729 vf2pf_info->os_info.all = 0; 730 731 vf2pf_info->fb_usage = ttm_resource_manager_used(&adev->mman.vram_mgr.manager) ? 732 ttm_resource_manager_usage(&adev->mman.vram_mgr.manager) >> 20 : 0; 733 vf2pf_info->fb_vis_usage = 734 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr) >> 20; 735 vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20; 736 vf2pf_info->fb_vis_size = adev->gmc.visible_vram_size >> 20; 737 738 amdgpu_virt_populate_vf2pf_ucode_info(adev); 739 740 /* TODO: read dynamic info */ 741 vf2pf_info->gfx_usage = 0; 742 vf2pf_info->compute_usage = 0; 743 vf2pf_info->encode_usage = 0; 744 vf2pf_info->decode_usage = 0; 745 746 vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr; 747 if (amdgpu_sriov_is_mes_info_enable(adev)) { 748 vf2pf_info->mes_info_addr = 749 (uint64_t)(adev->mes.resource_1_gpu_addr[0] + AMDGPU_GPU_PAGE_SIZE); 750 vf2pf_info->mes_info_size = 751 adev->mes.resource_1[0]->tbo.base.size - AMDGPU_GPU_PAGE_SIZE; 752 } 753 vf2pf_info->checksum = 754 amd_sriov_msg_checksum( 755 vf2pf_info, sizeof(*vf2pf_info), 0, 0); 756 757 return 0; 758 } 759 760 static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) 761 { 762 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work); 763 int ret; 764 765 ret = amdgpu_virt_read_pf2vf_data(adev); 766 if (ret) { 767 adev->virt.vf2pf_update_retry_cnt++; 768 769 if ((amdgpu_virt_rcvd_ras_interrupt(adev) || 770 adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) && 771 amdgpu_sriov_runtime(adev)) { 772 773 amdgpu_ras_set_fed(adev, true); 774 if (amdgpu_reset_domain_schedule(adev->reset_domain, 775 &adev->kfd.reset_work)) 776 return; 777 else 778 dev_err(adev->dev, "Failed to queue work! at %s", __func__); 779 } 780 781 goto out; 782 } 783 784 adev->virt.vf2pf_update_retry_cnt = 0; 785 amdgpu_virt_write_vf2pf_data(adev); 786 787 out: 788 schedule_delayed_work(&(adev->virt.vf2pf_work), adev->virt.vf2pf_update_interval_ms); 789 } 790 791 static int amdgpu_virt_read_exchange_data_from_mem(struct amdgpu_device *adev, uint32_t *pfvf_data) 792 { 793 uint32_t dataexchange_offset = 794 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset; 795 uint32_t dataexchange_size = 796 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb << 10; 797 uint64_t pos = 0; 798 799 dev_info(adev->dev, 800 "Got data exchange info from dynamic crit_region_table at offset 0x%x with size of 0x%x bytes.\n", 801 dataexchange_offset, dataexchange_size); 802 803 if (!IS_ALIGNED(dataexchange_offset, 4) || !IS_ALIGNED(dataexchange_size, 4)) { 804 dev_err(adev->dev, "Data exchange data not aligned to 4 bytes\n"); 805 return -EINVAL; 806 } 807 808 pos = (uint64_t)dataexchange_offset; 809 amdgpu_device_vram_access(adev, pos, pfvf_data, 810 dataexchange_size, false); 811 812 return 0; 813 } 814 815 void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev) 816 { 817 if (adev->virt.vf2pf_update_interval_ms != 0) { 818 dev_info(adev->dev, "clean up the vf2pf work item\n"); 819 cancel_delayed_work_sync(&adev->virt.vf2pf_work); 820 adev->virt.vf2pf_update_interval_ms = 0; 821 } 822 } 823 824 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) 825 { 826 uint32_t *pfvf_data = NULL; 827 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 828 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 829 830 adev->virt.fw_reserve.p_pf2vf = NULL; 831 adev->virt.fw_reserve.p_vf2pf = NULL; 832 adev->virt.vf2pf_update_interval_ms = 0; 833 adev->virt.vf2pf_update_retry_cnt = 0; 834 835 if (fw_va && drv_va) { 836 dev_warn(adev->dev, "Currently fw_vram and drv_vram should not have values at the same time!"); 837 } else if (fw_va || drv_va) { 838 /* go through this logic in ip_init and reset to init workqueue*/ 839 amdgpu_virt_exchange_data(adev); 840 841 INIT_DELAYED_WORK(&adev->virt.vf2pf_work, amdgpu_virt_update_vf2pf_work_item); 842 schedule_delayed_work(&(adev->virt.vf2pf_work), msecs_to_jiffies(adev->virt.vf2pf_update_interval_ms)); 843 } else if (adev->bios != NULL) { 844 /* got through this logic in early init stage to get necessary flags, e.g. rlcg_acc related*/ 845 if (adev->virt.req_init_data_ver == GPU_CRIT_REGION_V2) { 846 pfvf_data = 847 kzalloc(adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb << 10, 848 GFP_KERNEL); 849 if (!pfvf_data) { 850 dev_err(adev->dev, "Failed to allocate memory for pfvf_data\n"); 851 return; 852 } 853 854 if (amdgpu_virt_read_exchange_data_from_mem(adev, pfvf_data)) 855 goto free_pfvf_data; 856 857 adev->virt.fw_reserve.p_pf2vf = 858 (struct amd_sriov_msg_pf2vf_info_header *)pfvf_data; 859 860 amdgpu_virt_read_pf2vf_data(adev); 861 862 free_pfvf_data: 863 kfree(pfvf_data); 864 pfvf_data = NULL; 865 adev->virt.fw_reserve.p_pf2vf = NULL; 866 } else { 867 adev->virt.fw_reserve.p_pf2vf = 868 (struct amd_sriov_msg_pf2vf_info_header *) 869 (adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 870 871 amdgpu_virt_read_pf2vf_data(adev); 872 } 873 } 874 } 875 876 877 void amdgpu_virt_exchange_data(struct amdgpu_device *adev) 878 { 879 uint64_t bp_block_offset = 0; 880 uint32_t bp_block_size = 0; 881 struct amd_sriov_msg_pf2vf_info *pf2vf_v2 = NULL; 882 void *fw_va = adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].cpu_ptr; 883 void *drv_va = adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].cpu_ptr; 884 885 if (fw_va || drv_va) { 886 if (fw_va) { 887 if (adev->virt.req_init_data_ver == GPU_CRIT_REGION_V2) { 888 adev->virt.fw_reserve.p_pf2vf = 889 (struct amd_sriov_msg_pf2vf_info_header *) 890 (fw_va + 891 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset); 892 adev->virt.fw_reserve.p_vf2pf = 893 (struct amd_sriov_msg_vf2pf_info_header *) 894 (fw_va + 895 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset + 896 (AMD_SRIOV_MSG_SIZE_KB << 10)); 897 adev->virt.fw_reserve.ras_telemetry = 898 (fw_va + 899 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].offset); 900 } else { 901 adev->virt.fw_reserve.p_pf2vf = 902 (struct amd_sriov_msg_pf2vf_info_header *) 903 (fw_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 904 adev->virt.fw_reserve.p_vf2pf = 905 (struct amd_sriov_msg_vf2pf_info_header *) 906 (fw_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB_V1 << 10)); 907 adev->virt.fw_reserve.ras_telemetry = 908 (fw_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB_V1 << 10)); 909 } 910 } else if (drv_va) { 911 adev->virt.fw_reserve.p_pf2vf = 912 (struct amd_sriov_msg_pf2vf_info_header *) 913 (drv_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB_V1 << 10)); 914 adev->virt.fw_reserve.p_vf2pf = 915 (struct amd_sriov_msg_vf2pf_info_header *) 916 (drv_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB_V1 << 10)); 917 adev->virt.fw_reserve.ras_telemetry = 918 (drv_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB_V1 << 10)); 919 } 920 921 amdgpu_virt_read_pf2vf_data(adev); 922 amdgpu_virt_write_vf2pf_data(adev); 923 924 /* bad page handling for version 2 */ 925 if (adev->virt.fw_reserve.p_pf2vf->version == 2) { 926 pf2vf_v2 = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf; 927 928 bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_low & 0xFFFFFFFF) | 929 ((((uint64_t)pf2vf_v2->bp_block_offset_high) << 32) & 0xFFFFFFFF00000000); 930 bp_block_size = pf2vf_v2->bp_block_size; 931 932 if (bp_block_size && !adev->virt.ras_init_done) 933 amdgpu_virt_init_ras_err_handler_data(adev); 934 935 if (adev->virt.ras_init_done) 936 amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size); 937 } 938 } 939 } 940 941 static u32 amdgpu_virt_init_detect_asic(struct amdgpu_device *adev) 942 { 943 uint32_t reg; 944 945 switch (adev->asic_type) { 946 case CHIP_TONGA: 947 case CHIP_FIJI: 948 reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER); 949 break; 950 case CHIP_VEGA10: 951 case CHIP_VEGA20: 952 case CHIP_NAVI10: 953 case CHIP_NAVI12: 954 case CHIP_SIENNA_CICHLID: 955 case CHIP_ARCTURUS: 956 case CHIP_ALDEBARAN: 957 case CHIP_IP_DISCOVERY: 958 reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER); 959 break; 960 default: /* other chip doesn't support SRIOV */ 961 reg = 0; 962 break; 963 } 964 965 if (reg & 1) 966 adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF; 967 968 if (reg & 0x80000000) 969 adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV; 970 971 if (!reg) { 972 /* passthrough mode exclus sriov mod */ 973 if (is_virtual_machine() && !xen_initial_domain()) 974 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE; 975 } 976 977 return reg; 978 } 979 980 static bool amdgpu_virt_init_req_data(struct amdgpu_device *adev, u32 reg) 981 { 982 bool is_sriov = false; 983 984 /* we have the ability to check now */ 985 if (amdgpu_sriov_vf(adev)) { 986 is_sriov = true; 987 988 switch (adev->asic_type) { 989 case CHIP_TONGA: 990 case CHIP_FIJI: 991 vi_set_virt_ops(adev); 992 break; 993 case CHIP_VEGA10: 994 soc15_set_virt_ops(adev); 995 #ifdef CONFIG_X86 996 /* not send GPU_INIT_DATA with MS_HYPERV*/ 997 if (!hypervisor_is_type(X86_HYPER_MS_HYPERV)) 998 #endif 999 /* send a dummy GPU_INIT_DATA request to host on vega10 */ 1000 amdgpu_virt_request_init_data(adev); 1001 break; 1002 case CHIP_VEGA20: 1003 case CHIP_ARCTURUS: 1004 case CHIP_ALDEBARAN: 1005 soc15_set_virt_ops(adev); 1006 break; 1007 case CHIP_NAVI10: 1008 case CHIP_NAVI12: 1009 case CHIP_SIENNA_CICHLID: 1010 case CHIP_IP_DISCOVERY: 1011 nv_set_virt_ops(adev); 1012 /* try send GPU_INIT_DATA request to host */ 1013 amdgpu_virt_request_init_data(adev); 1014 break; 1015 default: /* other chip doesn't support SRIOV */ 1016 is_sriov = false; 1017 dev_err(adev->dev, "Unknown asic type: %d!\n", adev->asic_type); 1018 break; 1019 } 1020 } 1021 1022 return is_sriov; 1023 } 1024 1025 static void amdgpu_virt_init_ras(struct amdgpu_device *adev) 1026 { 1027 ratelimit_state_init(&adev->virt.ras.ras_error_cnt_rs, 5 * HZ, 1); 1028 ratelimit_state_init(&adev->virt.ras.ras_cper_dump_rs, 5 * HZ, 1); 1029 ratelimit_state_init(&adev->virt.ras.ras_chk_criti_rs, 5 * HZ, 1); 1030 1031 ratelimit_set_flags(&adev->virt.ras.ras_error_cnt_rs, 1032 RATELIMIT_MSG_ON_RELEASE); 1033 ratelimit_set_flags(&adev->virt.ras.ras_cper_dump_rs, 1034 RATELIMIT_MSG_ON_RELEASE); 1035 ratelimit_set_flags(&adev->virt.ras.ras_chk_criti_rs, 1036 RATELIMIT_MSG_ON_RELEASE); 1037 1038 mutex_init(&adev->virt.ras.ras_telemetry_mutex); 1039 mutex_init(&adev->virt.access_req_mutex); 1040 1041 adev->virt.ras.cper_rptr = 0; 1042 } 1043 1044 static uint8_t amdgpu_virt_crit_region_calc_checksum(uint8_t *buf_start, uint8_t *buf_end) 1045 { 1046 uint32_t sum = 0; 1047 1048 if (buf_start >= buf_end) 1049 return 0; 1050 1051 for (; buf_start < buf_end; buf_start++) 1052 sum += buf_start[0]; 1053 1054 return 0xffffffff - sum; 1055 } 1056 1057 int amdgpu_virt_init_critical_region(struct amdgpu_device *adev) 1058 { 1059 struct amd_sriov_msg_init_data_header *init_data_hdr = NULL; 1060 u64 init_hdr_offset = adev->virt.init_data_header.offset; 1061 u64 init_hdr_size = (u64)adev->virt.init_data_header.size_kb << 10; /* KB → bytes */ 1062 u64 vram_size; 1063 u64 end; 1064 int r = 0; 1065 uint8_t checksum = 0; 1066 1067 /* Skip below init if critical region version != v2 */ 1068 if (adev->virt.req_init_data_ver != GPU_CRIT_REGION_V2) 1069 return 0; 1070 1071 vram_size = RREG32(mmRCC_CONFIG_MEMSIZE); 1072 /* Fall back to RCC_HW_DEBUG when RCC_CONFIG_MEMSIZE reads 0 on a VF. */ 1073 if (!vram_size) 1074 vram_size = RREG32(mmRCC_HW_DEBUG); 1075 if (!vram_size || vram_size == U32_MAX) 1076 return -EINVAL; 1077 vram_size <<= 20; 1078 1079 if (check_add_overflow(init_hdr_offset, init_hdr_size, &end) || end > vram_size) { 1080 dev_err(adev->dev, "init_data_header exceeds VRAM size, exiting\n"); 1081 return -EINVAL; 1082 } 1083 1084 /* Allocate for init_data_hdr */ 1085 init_data_hdr = kzalloc_obj(struct amd_sriov_msg_init_data_header); 1086 if (!init_data_hdr) 1087 return -ENOMEM; 1088 1089 amdgpu_device_vram_access(adev, (uint64_t)init_hdr_offset, (uint32_t *)init_data_hdr, 1090 sizeof(struct amd_sriov_msg_init_data_header), false); 1091 1092 /* Table validation */ 1093 if (strncmp(init_data_hdr->signature, 1094 AMDGPU_SRIOV_CRIT_DATA_SIGNATURE, 1095 AMDGPU_SRIOV_CRIT_DATA_SIG_LEN) != 0) { 1096 dev_err(adev->dev, "Invalid init data signature: %.4s\n", 1097 init_data_hdr->signature); 1098 r = -EINVAL; 1099 goto out; 1100 } 1101 1102 checksum = amdgpu_virt_crit_region_calc_checksum( 1103 (uint8_t *)&init_data_hdr->initdata_offset, 1104 (uint8_t *)init_data_hdr + 1105 sizeof(struct amd_sriov_msg_init_data_header)); 1106 if (checksum != init_data_hdr->checksum) { 1107 dev_err(adev->dev, "Found unmatching checksum from calculation 0x%x and init_data 0x%x\n", 1108 checksum, init_data_hdr->checksum); 1109 r = -EINVAL; 1110 goto out; 1111 } 1112 1113 memset(&adev->virt.crit_regn, 0, sizeof(adev->virt.crit_regn)); 1114 memset(adev->virt.crit_regn_tbl, 0, sizeof(adev->virt.crit_regn_tbl)); 1115 1116 adev->virt.crit_regn.offset = init_data_hdr->initdata_offset; 1117 adev->virt.crit_regn.size_kb = init_data_hdr->initdata_size_in_kb; 1118 1119 /* Validation and initialization for each table entry */ 1120 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_IPD_TABLE_ID)) { 1121 if (!init_data_hdr->ip_discovery_size_in_kb || 1122 init_data_hdr->ip_discovery_size_in_kb > 1123 (DISCOVERY_TMR_SIZE_SRIOV >> 10)) { 1124 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1125 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_IPD_TABLE_ID], 1126 init_data_hdr->ip_discovery_size_in_kb); 1127 r = -EINVAL; 1128 goto out; 1129 } 1130 1131 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset = 1132 init_data_hdr->ip_discovery_offset; 1133 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb = 1134 init_data_hdr->ip_discovery_size_in_kb; 1135 } 1136 1137 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID)) { 1138 if (!init_data_hdr->vbios_img_size_in_kb) { 1139 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1140 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID], 1141 init_data_hdr->vbios_img_size_in_kb); 1142 r = -EINVAL; 1143 goto out; 1144 } 1145 1146 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID].offset = 1147 init_data_hdr->vbios_img_offset; 1148 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_VBIOS_IMG_TABLE_ID].size_kb = 1149 init_data_hdr->vbios_img_size_in_kb; 1150 } 1151 1152 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID)) { 1153 if (!init_data_hdr->ras_tele_info_size_in_kb) { 1154 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1155 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID], 1156 init_data_hdr->ras_tele_info_size_in_kb); 1157 r = -EINVAL; 1158 goto out; 1159 } 1160 1161 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].offset = 1162 init_data_hdr->ras_tele_info_offset; 1163 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_RAS_TELEMETRY_TABLE_ID].size_kb = 1164 init_data_hdr->ras_tele_info_size_in_kb; 1165 } 1166 1167 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID)) { 1168 if (!init_data_hdr->dataexchange_size_in_kb) { 1169 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1170 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID], 1171 init_data_hdr->dataexchange_size_in_kb); 1172 r = -EINVAL; 1173 goto out; 1174 } 1175 1176 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].offset = 1177 init_data_hdr->dataexchange_offset; 1178 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_DATAEXCHANGE_TABLE_ID].size_kb = 1179 init_data_hdr->dataexchange_size_in_kb; 1180 } 1181 1182 if (IS_SRIOV_CRIT_REGN_ENTRY_VALID(init_data_hdr, AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID)) { 1183 if (!init_data_hdr->bad_page_size_in_kb) { 1184 dev_err(adev->dev, "Invalid %s size: 0x%x\n", 1185 amdgpu_virt_dynamic_crit_table_name[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID], 1186 init_data_hdr->bad_page_size_in_kb); 1187 r = -EINVAL; 1188 goto out; 1189 } 1190 1191 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID].offset = 1192 init_data_hdr->bad_page_info_offset; 1193 adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_BAD_PAGE_INFO_TABLE_ID].size_kb = 1194 init_data_hdr->bad_page_size_in_kb; 1195 } 1196 1197 /* reserved memory starts from crit region base offset with the size of 5MB */ 1198 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_FW_VRAM_USAGE, 1199 adev->virt.crit_regn.offset, 1200 adev->virt.crit_regn.size_kb << 10, true); 1201 dev_info(adev->dev, 1202 "critical region v%d requested to reserve memory start at %08llx with %llu KB.\n", 1203 init_data_hdr->version, 1204 adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].offset, 1205 adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size >> 10); 1206 1207 adev->virt.is_dynamic_crit_regn_enabled = true; 1208 1209 out: 1210 kfree(init_data_hdr); 1211 init_data_hdr = NULL; 1212 1213 return r; 1214 } 1215 1216 int amdgpu_virt_get_dynamic_data_info(struct amdgpu_device *adev, 1217 int data_id, uint8_t *binary, u32 *size) 1218 { 1219 uint32_t data_offset = 0; 1220 uint32_t data_size = 0; 1221 enum amd_sriov_msg_table_id_enum data_table_id = data_id; 1222 1223 if (data_table_id >= AMD_SRIOV_MSG_MAX_TABLE_ID) 1224 return -EINVAL; 1225 1226 data_offset = adev->virt.crit_regn_tbl[data_table_id].offset; 1227 data_size = adev->virt.crit_regn_tbl[data_table_id].size_kb << 10; 1228 1229 /* Validate on input params */ 1230 if (!binary || !size || *size < (uint64_t)data_size) 1231 return -EINVAL; 1232 1233 /* Proceed to copy the dynamic content */ 1234 amdgpu_device_vram_access(adev, 1235 (uint64_t)data_offset, (uint32_t *)binary, data_size, false); 1236 *size = (uint64_t)data_size; 1237 1238 dev_dbg(adev->dev, 1239 "Got %s info from dynamic crit_region_table at offset 0x%x with size of 0x%x bytes.\n", 1240 amdgpu_virt_dynamic_crit_table_name[data_id], data_offset, data_size); 1241 1242 return 0; 1243 } 1244 1245 void amdgpu_virt_init(struct amdgpu_device *adev) 1246 { 1247 bool is_sriov = false; 1248 uint32_t reg = amdgpu_virt_init_detect_asic(adev); 1249 1250 is_sriov = amdgpu_virt_init_req_data(adev, reg); 1251 1252 if (is_sriov) 1253 amdgpu_virt_init_ras(adev); 1254 } 1255 1256 static bool amdgpu_virt_access_debugfs_is_mmio(struct amdgpu_device *adev) 1257 { 1258 return amdgpu_sriov_is_debug(adev) ? true : false; 1259 } 1260 1261 static bool amdgpu_virt_access_debugfs_is_kiq(struct amdgpu_device *adev) 1262 { 1263 return amdgpu_sriov_is_normal(adev) ? true : false; 1264 } 1265 1266 int amdgpu_virt_enable_access_debugfs(struct amdgpu_device *adev) 1267 { 1268 if (!amdgpu_sriov_vf(adev) || 1269 amdgpu_virt_access_debugfs_is_kiq(adev)) 1270 return 0; 1271 1272 if (amdgpu_virt_access_debugfs_is_mmio(adev)) 1273 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 1274 else 1275 return -EPERM; 1276 1277 return 0; 1278 } 1279 1280 void amdgpu_virt_disable_access_debugfs(struct amdgpu_device *adev) 1281 { 1282 if (amdgpu_sriov_vf(adev)) 1283 adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME; 1284 } 1285 1286 enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *adev) 1287 { 1288 enum amdgpu_sriov_vf_mode mode; 1289 1290 if (amdgpu_sriov_vf(adev)) { 1291 if (amdgpu_sriov_is_pp_one_vf(adev)) 1292 mode = SRIOV_VF_MODE_ONE_VF; 1293 else 1294 mode = SRIOV_VF_MODE_MULTI_VF; 1295 } else { 1296 mode = SRIOV_VF_MODE_BARE_METAL; 1297 } 1298 1299 return mode; 1300 } 1301 1302 void amdgpu_virt_pre_reset(struct amdgpu_device *adev) 1303 { 1304 /* stop the data exchange thread */ 1305 amdgpu_virt_fini_data_exchange(adev); 1306 amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_FLR); 1307 } 1308 1309 void amdgpu_virt_post_reset(struct amdgpu_device *adev) 1310 { 1311 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) { 1312 /* force set to GFXOFF state after reset, 1313 * to avoid some invalid operation before GC enable 1314 */ 1315 adev->gfx.is_poweron = false; 1316 } 1317 1318 adev->mes.ring[0].sched.ready = false; 1319 } 1320 1321 bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id) 1322 { 1323 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 1324 case IP_VERSION(13, 0, 0): 1325 /* no vf autoload, white list */ 1326 if (ucode_id == AMDGPU_UCODE_ID_VCN1 || 1327 ucode_id == AMDGPU_UCODE_ID_VCN) 1328 return false; 1329 else 1330 return true; 1331 case IP_VERSION(11, 0, 9): 1332 case IP_VERSION(11, 0, 7): 1333 /* black list for CHIP_NAVI12 and CHIP_SIENNA_CICHLID */ 1334 if (ucode_id == AMDGPU_UCODE_ID_RLC_G 1335 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL 1336 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM 1337 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM 1338 || ucode_id == AMDGPU_UCODE_ID_SMC) 1339 return true; 1340 else 1341 return false; 1342 case IP_VERSION(13, 0, 10): 1343 /* white list */ 1344 if (ucode_id == AMDGPU_UCODE_ID_CAP 1345 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP 1346 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME 1347 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC 1348 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK 1349 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK 1350 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK 1351 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK 1352 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK 1353 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK 1354 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK 1355 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK 1356 || ucode_id == AMDGPU_UCODE_ID_CP_MES 1357 || ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA 1358 || ucode_id == AMDGPU_UCODE_ID_CP_MES1 1359 || ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA 1360 || ucode_id == AMDGPU_UCODE_ID_VCN1 1361 || ucode_id == AMDGPU_UCODE_ID_VCN) 1362 return false; 1363 else 1364 return true; 1365 case IP_VERSION(15, 0, 3): 1366 if (ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP 1367 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME 1368 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC 1369 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK 1370 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK 1371 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK 1372 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK 1373 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK 1374 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK 1375 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK 1376 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK 1377 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P4_STACK 1378 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P5_STACK 1379 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P6_STACK 1380 || ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P7_STACK 1381 || ucode_id == AMDGPU_UCODE_ID_CP_MES 1382 || ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA 1383 || ucode_id == AMDGPU_UCODE_ID_CP_MES1 1384 || ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA 1385 || ucode_id == AMDGPU_UCODE_ID_VPE_CTX 1386 || ucode_id == AMDGPU_UCODE_ID_VPE_CTL) 1387 return false; 1388 else 1389 return true; 1390 default: 1391 /* lagacy black list */ 1392 if (ucode_id == AMDGPU_UCODE_ID_SDMA0 1393 || ucode_id == AMDGPU_UCODE_ID_SDMA1 1394 || ucode_id == AMDGPU_UCODE_ID_SDMA2 1395 || ucode_id == AMDGPU_UCODE_ID_SDMA3 1396 || ucode_id == AMDGPU_UCODE_ID_SDMA4 1397 || ucode_id == AMDGPU_UCODE_ID_SDMA5 1398 || ucode_id == AMDGPU_UCODE_ID_SDMA6 1399 || ucode_id == AMDGPU_UCODE_ID_SDMA7 1400 || ucode_id == AMDGPU_UCODE_ID_SDMA_RS64 1401 || ucode_id == AMDGPU_UCODE_ID_RLC_G 1402 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL 1403 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM 1404 || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM 1405 || ucode_id == AMDGPU_UCODE_ID_SMC) 1406 return true; 1407 else 1408 return false; 1409 } 1410 } 1411 1412 void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev, 1413 struct amdgpu_video_codec_info *encode, uint32_t encode_array_size, 1414 struct amdgpu_video_codec_info *decode, uint32_t decode_array_size) 1415 { 1416 uint32_t i; 1417 1418 if (!adev->virt.is_mm_bw_enabled) 1419 return; 1420 1421 if (encode) { 1422 for (i = 0; i < encode_array_size; i++) { 1423 encode[i].max_width = adev->virt.encode_max_dimension_pixels; 1424 encode[i].max_pixels_per_frame = adev->virt.encode_max_frame_pixels; 1425 if (encode[i].max_width > 0) 1426 encode[i].max_height = encode[i].max_pixels_per_frame / encode[i].max_width; 1427 else 1428 encode[i].max_height = 0; 1429 } 1430 } 1431 1432 if (decode) { 1433 for (i = 0; i < decode_array_size; i++) { 1434 decode[i].max_width = adev->virt.decode_max_dimension_pixels; 1435 decode[i].max_pixels_per_frame = adev->virt.decode_max_frame_pixels; 1436 if (decode[i].max_width > 0) 1437 decode[i].max_height = decode[i].max_pixels_per_frame / decode[i].max_width; 1438 else 1439 decode[i].max_height = 0; 1440 } 1441 } 1442 } 1443 1444 bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev, 1445 u32 acc_flags, u32 hwip, 1446 bool write, u32 *rlcg_flag) 1447 { 1448 bool ret = false; 1449 1450 switch (hwip) { 1451 case GC_HWIP: 1452 if (amdgpu_sriov_reg_indirect_gc(adev)) { 1453 *rlcg_flag = 1454 write ? AMDGPU_RLCG_GC_WRITE : AMDGPU_RLCG_GC_READ; 1455 ret = true; 1456 /* only in new version, AMDGPU_REGS_NO_KIQ and 1457 * AMDGPU_REGS_RLC are enabled simultaneously */ 1458 } else if ((acc_flags & AMDGPU_REGS_RLC) && 1459 !(acc_flags & AMDGPU_REGS_NO_KIQ) && write) { 1460 *rlcg_flag = AMDGPU_RLCG_GC_WRITE_LEGACY; 1461 ret = true; 1462 } 1463 break; 1464 case MMHUB_HWIP: 1465 if (amdgpu_sriov_reg_indirect_mmhub(adev) && 1466 (acc_flags & AMDGPU_REGS_RLC) && write) { 1467 *rlcg_flag = AMDGPU_RLCG_MMHUB_WRITE; 1468 ret = true; 1469 } 1470 break; 1471 default: 1472 break; 1473 } 1474 return ret; 1475 } 1476 1477 static u32 amdgpu_virt_rlcg_vfi_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id) 1478 { 1479 uint32_t timeout = 100; 1480 uint32_t i; 1481 1482 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1483 void *vfi_cmd; 1484 void *vfi_stat; 1485 void *vfi_addr; 1486 void *vfi_data; 1487 void *vfi_grbm_cntl; 1488 void *vfi_grbm_idx; 1489 uint32_t cmd; 1490 uint32_t stat; 1491 uint32_t addr = offset; 1492 uint32_t data; 1493 uint32_t grbm_cntl_data; 1494 uint32_t grbm_idx_data; 1495 1496 unsigned long flags; 1497 bool is_err = true; 1498 1499 if (!adev->gfx.rlc.rlcg_reg_access_supported) { 1500 dev_err(adev->dev, "VFi interface is not available\n"); 1501 return 0; 1502 } 1503 1504 if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) { 1505 dev_err(adev->dev, "VFi invalid XCC, xcc_id=0x%x\n", xcc_id); 1506 return 0; 1507 } 1508 1509 if (amdgpu_device_skip_hw_access(adev)) 1510 return 0; 1511 1512 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id]; 1513 vfi_cmd = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_cmd; 1514 vfi_stat = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_stat; 1515 vfi_addr = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_addr; 1516 vfi_data = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_data; 1517 vfi_grbm_cntl = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_grbm_cntl; 1518 vfi_grbm_idx = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->vfi_grbm_idx; 1519 grbm_cntl_data = reg_access_ctrl->vfi_grbm_cntl_data; 1520 grbm_idx_data = reg_access_ctrl->vfi_grbm_idx_data; 1521 1522 if (flag == AMDGPU_RLCG_GC_WRITE) { 1523 data = v; 1524 cmd = AMDGPU_RLCG_VFI_CMD__WR; 1525 1526 // the GRBM_GFX_CNTL and GRBM_GFX_INDEX are protected by mutex outside this call 1527 if (addr == reg_access_ctrl->grbm_cntl) { 1528 reg_access_ctrl->vfi_grbm_cntl_data = data; 1529 return 0; 1530 } else if (addr == reg_access_ctrl->grbm_idx) { 1531 reg_access_ctrl->vfi_grbm_idx_data = data; 1532 return 0; 1533 } 1534 1535 } else if (flag == AMDGPU_RLCG_GC_READ) { 1536 data = 0; 1537 cmd = AMDGPU_RLCG_VFI_CMD__RD; 1538 1539 // the GRBM_GFX_CNTL and GRBM_GFX_INDEX are protected by mutex outside this call 1540 if (addr == reg_access_ctrl->grbm_cntl) 1541 return grbm_cntl_data; 1542 else if (addr == reg_access_ctrl->grbm_idx) 1543 return grbm_idx_data; 1544 1545 } else { 1546 dev_err(adev->dev, "VFi invalid access, flag=0x%x\n", flag); 1547 return 0; 1548 } 1549 1550 spin_lock_irqsave(&adev->virt.rlcg_reg_lock, flags); 1551 1552 writel(addr, vfi_addr); 1553 writel(data, vfi_data); 1554 writel(grbm_cntl_data, vfi_grbm_cntl); 1555 writel(grbm_idx_data, vfi_grbm_idx); 1556 1557 writel(AMDGPU_RLCG_VFI_STAT__BUSY, vfi_stat); 1558 writel(cmd, vfi_cmd); 1559 1560 for (i = 0; i < timeout; i++) { 1561 stat = readl(vfi_stat); 1562 if (stat != AMDGPU_RLCG_VFI_STAT__BUSY) 1563 break; 1564 udelay(10); 1565 } 1566 1567 switch (stat) { 1568 case AMDGPU_RLCG_VFI_STAT__DONE: 1569 is_err = false; 1570 if (cmd == AMDGPU_RLCG_VFI_CMD__RD) 1571 data = readl(vfi_data); 1572 break; 1573 case AMDGPU_RLCG_VFI_STAT__BUSY: 1574 dev_err(adev->dev, "VFi access timeout\n"); 1575 break; 1576 case AMDGPU_RLCG_VFI_STAT__INV_CMD: 1577 dev_err(adev->dev, "VFi invalid command\n"); 1578 break; 1579 case AMDGPU_RLCG_VFI_STAT__INV_ADDR: 1580 dev_err(adev->dev, "VFi invalid address\n"); 1581 break; 1582 case AMDGPU_RLCG_VFI_STAT__ERR: 1583 dev_err(adev->dev, "VFi unknown error\n"); 1584 break; 1585 default: 1586 dev_err(adev->dev, "VFi unknown status code\n"); 1587 break; 1588 } 1589 1590 spin_unlock_irqrestore(&adev->virt.rlcg_reg_lock, flags); 1591 1592 if (is_err) 1593 dev_err(adev->dev, "VFi: xcc%u [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n", 1594 xcc_id, grbm_cntl_data, grbm_idx_data, 1595 addr, addr * 4, data, cmd); 1596 else 1597 dev_dbg(adev->dev, "VFi: xcc%u [grbm_cntl=0x%x grbm_idx=0x%x] addr=0x%x (byte addr 0x%x), data=0x%x, cmd=0x%x\n", 1598 xcc_id, grbm_cntl_data, grbm_idx_data, 1599 addr, addr * 4, data, cmd); 1600 1601 return data; 1602 } 1603 1604 u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id) 1605 { 1606 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1607 uint32_t timeout = 50000; 1608 uint32_t i, tmp; 1609 uint32_t ret = 0; 1610 void *scratch_reg0; 1611 void *scratch_reg1; 1612 void *scratch_reg2; 1613 void *scratch_reg3; 1614 void *spare_int; 1615 unsigned long flags; 1616 1617 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) 1618 return amdgpu_virt_rlcg_vfi_reg_rw(adev, offset, v, flag, xcc_id); 1619 1620 if (!adev->gfx.rlc.rlcg_reg_access_supported) { 1621 dev_err(adev->dev, 1622 "indirect registers access through rlcg is not available\n"); 1623 return 0; 1624 } 1625 1626 if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) { 1627 dev_err(adev->dev, "invalid xcc\n"); 1628 return 0; 1629 } 1630 1631 if (amdgpu_device_skip_hw_access(adev)) 1632 return 0; 1633 1634 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id]; 1635 scratch_reg0 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg0; 1636 scratch_reg1 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg1; 1637 scratch_reg2 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg2; 1638 scratch_reg3 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg3; 1639 1640 spin_lock_irqsave(&adev->virt.rlcg_reg_lock, flags); 1641 1642 if (reg_access_ctrl->spare_int) 1643 spare_int = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->spare_int; 1644 1645 if (offset == reg_access_ctrl->grbm_cntl) { 1646 /* if the target reg offset is grbm_cntl, write to scratch_reg2 */ 1647 writel(v, scratch_reg2); 1648 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) 1649 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 1650 } else if (offset == reg_access_ctrl->grbm_idx) { 1651 /* if the target reg offset is grbm_idx, write to scratch_reg3 */ 1652 writel(v, scratch_reg3); 1653 if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY) 1654 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 1655 } else { 1656 /* 1657 * SCRATCH_REG0 = read/write value 1658 * SCRATCH_REG1[30:28] = command 1659 * SCRATCH_REG1[19:0] = address in dword 1660 * SCRATCH_REG1[27:24] = Error reporting 1661 */ 1662 writel(v, scratch_reg0); 1663 writel((offset | flag), scratch_reg1); 1664 if (reg_access_ctrl->spare_int) 1665 writel(1, spare_int); 1666 1667 for (i = 0; i < timeout; i++) { 1668 tmp = readl(scratch_reg1); 1669 if (!(tmp & AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK)) 1670 break; 1671 udelay(10); 1672 } 1673 1674 tmp = readl(scratch_reg1); 1675 if (i >= timeout || (tmp & AMDGPU_RLCG_SCRATCH1_ERROR_MASK) != 0) { 1676 if (amdgpu_sriov_rlcg_error_report_enabled(adev)) { 1677 if (tmp & AMDGPU_RLCG_VFGATE_DISABLED) { 1678 dev_err(adev->dev, 1679 "vfgate is disabled, rlcg failed to program reg: 0x%05x\n", offset); 1680 } else if (tmp & AMDGPU_RLCG_WRONG_OPERATION_TYPE) { 1681 dev_err(adev->dev, 1682 "wrong operation type, rlcg failed to program reg: 0x%05x\n", offset); 1683 } else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) { 1684 dev_err(adev->dev, 1685 "register is not in range, rlcg failed to program reg: 0x%05x\n", offset); 1686 } else { 1687 dev_err(adev->dev, 1688 "unknown error type, rlcg failed to program reg: 0x%05x\n", offset); 1689 } 1690 } else { 1691 dev_err(adev->dev, 1692 "timeout: rlcg faled to program reg: 0x%05x\n", offset); 1693 } 1694 } 1695 } 1696 1697 ret = readl(scratch_reg0); 1698 1699 spin_unlock_irqrestore(&adev->virt.rlcg_reg_lock, flags); 1700 1701 return ret; 1702 } 1703 1704 void amdgpu_sriov_wreg(struct amdgpu_device *adev, 1705 u32 offset, u32 value, 1706 u32 acc_flags, u32 hwip, u32 xcc_id) 1707 { 1708 u32 rlcg_flag; 1709 1710 if (amdgpu_device_skip_hw_access(adev)) 1711 return; 1712 1713 if (!amdgpu_sriov_runtime(adev) && 1714 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, true, &rlcg_flag)) { 1715 amdgpu_virt_rlcg_reg_rw(adev, offset, value, rlcg_flag, xcc_id); 1716 return; 1717 } 1718 1719 if (acc_flags & AMDGPU_REGS_NO_KIQ) 1720 WREG32_NO_KIQ(offset, value); 1721 else 1722 WREG32(offset, value); 1723 } 1724 1725 u32 amdgpu_sriov_rreg(struct amdgpu_device *adev, 1726 u32 offset, u32 acc_flags, u32 hwip, u32 xcc_id) 1727 { 1728 u32 rlcg_flag; 1729 1730 if (amdgpu_device_skip_hw_access(adev)) 1731 return 0; 1732 1733 if (!amdgpu_sriov_runtime(adev) && 1734 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, false, &rlcg_flag)) 1735 return amdgpu_virt_rlcg_reg_rw(adev, offset, 0, rlcg_flag, xcc_id); 1736 1737 if (acc_flags & AMDGPU_REGS_NO_KIQ) 1738 return RREG32_NO_KIQ(offset); 1739 else 1740 return RREG32(offset); 1741 } 1742 1743 bool amdgpu_sriov_xnack_support(struct amdgpu_device *adev) 1744 { 1745 bool xnack_mode = true; 1746 1747 if (amdgpu_sriov_vf(adev) && 1748 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 1749 xnack_mode = false; 1750 1751 return xnack_mode; 1752 } 1753 1754 bool amdgpu_virt_get_ras_capability(struct amdgpu_device *adev) 1755 { 1756 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1757 1758 if (!amdgpu_sriov_ras_caps_en(adev)) 1759 return false; 1760 1761 if (adev->virt.ras_en_caps.bits.block_umc) 1762 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__UMC); 1763 if (adev->virt.ras_en_caps.bits.block_sdma) 1764 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SDMA); 1765 if (adev->virt.ras_en_caps.bits.block_gfx) 1766 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__GFX); 1767 if (adev->virt.ras_en_caps.bits.block_mmhub) 1768 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MMHUB); 1769 if (adev->virt.ras_en_caps.bits.block_athub) 1770 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__ATHUB); 1771 if (adev->virt.ras_en_caps.bits.block_pcie_bif) 1772 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__PCIE_BIF); 1773 if (adev->virt.ras_en_caps.bits.block_hdp) 1774 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__HDP); 1775 if (adev->virt.ras_en_caps.bits.block_xgmi_wafl) 1776 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__XGMI_WAFL); 1777 if (adev->virt.ras_en_caps.bits.block_df) 1778 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__DF); 1779 if (adev->virt.ras_en_caps.bits.block_smn) 1780 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SMN); 1781 if (adev->virt.ras_en_caps.bits.block_sem) 1782 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SEM); 1783 if (adev->virt.ras_en_caps.bits.block_mp0) 1784 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP0); 1785 if (adev->virt.ras_en_caps.bits.block_mp1) 1786 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP1); 1787 if (adev->virt.ras_en_caps.bits.block_fuse) 1788 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__FUSE); 1789 if (adev->virt.ras_en_caps.bits.block_mca) 1790 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MCA); 1791 if (adev->virt.ras_en_caps.bits.block_vcn) 1792 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__VCN); 1793 if (adev->virt.ras_en_caps.bits.block_jpeg) 1794 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__JPEG); 1795 if (adev->virt.ras_en_caps.bits.block_ih) 1796 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__IH); 1797 if (adev->virt.ras_en_caps.bits.block_mpio) 1798 adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MPIO); 1799 1800 if (adev->virt.ras_en_caps.bits.poison_propogation_mode) 1801 con->poison_supported = true; /* Poison is handled by host */ 1802 1803 if (adev->virt.ras_en_caps.bits.uniras_supported) 1804 amdgpu_virt_ras_set_remote_uniras(adev, true); 1805 1806 return true; 1807 } 1808 1809 static inline enum amd_sriov_ras_telemetry_gpu_block 1810 amdgpu_ras_block_to_sriov(struct amdgpu_device *adev, enum amdgpu_ras_block block) { 1811 switch (block) { 1812 case AMDGPU_RAS_BLOCK__UMC: 1813 return RAS_TELEMETRY_GPU_BLOCK_UMC; 1814 case AMDGPU_RAS_BLOCK__SDMA: 1815 return RAS_TELEMETRY_GPU_BLOCK_SDMA; 1816 case AMDGPU_RAS_BLOCK__GFX: 1817 return RAS_TELEMETRY_GPU_BLOCK_GFX; 1818 case AMDGPU_RAS_BLOCK__MMHUB: 1819 return RAS_TELEMETRY_GPU_BLOCK_MMHUB; 1820 case AMDGPU_RAS_BLOCK__ATHUB: 1821 return RAS_TELEMETRY_GPU_BLOCK_ATHUB; 1822 case AMDGPU_RAS_BLOCK__PCIE_BIF: 1823 return RAS_TELEMETRY_GPU_BLOCK_PCIE_BIF; 1824 case AMDGPU_RAS_BLOCK__HDP: 1825 return RAS_TELEMETRY_GPU_BLOCK_HDP; 1826 case AMDGPU_RAS_BLOCK__XGMI_WAFL: 1827 return RAS_TELEMETRY_GPU_BLOCK_XGMI_WAFL; 1828 case AMDGPU_RAS_BLOCK__DF: 1829 return RAS_TELEMETRY_GPU_BLOCK_DF; 1830 case AMDGPU_RAS_BLOCK__SMN: 1831 return RAS_TELEMETRY_GPU_BLOCK_SMN; 1832 case AMDGPU_RAS_BLOCK__SEM: 1833 return RAS_TELEMETRY_GPU_BLOCK_SEM; 1834 case AMDGPU_RAS_BLOCK__MP0: 1835 return RAS_TELEMETRY_GPU_BLOCK_MP0; 1836 case AMDGPU_RAS_BLOCK__MP1: 1837 return RAS_TELEMETRY_GPU_BLOCK_MP1; 1838 case AMDGPU_RAS_BLOCK__FUSE: 1839 return RAS_TELEMETRY_GPU_BLOCK_FUSE; 1840 case AMDGPU_RAS_BLOCK__MCA: 1841 return RAS_TELEMETRY_GPU_BLOCK_MCA; 1842 case AMDGPU_RAS_BLOCK__VCN: 1843 return RAS_TELEMETRY_GPU_BLOCK_VCN; 1844 case AMDGPU_RAS_BLOCK__JPEG: 1845 return RAS_TELEMETRY_GPU_BLOCK_JPEG; 1846 case AMDGPU_RAS_BLOCK__IH: 1847 return RAS_TELEMETRY_GPU_BLOCK_IH; 1848 case AMDGPU_RAS_BLOCK__MPIO: 1849 return RAS_TELEMETRY_GPU_BLOCK_MPIO; 1850 default: 1851 dev_warn(adev->dev, "Unsupported SRIOV RAS telemetry block 0x%x\n", 1852 block); 1853 return RAS_TELEMETRY_GPU_BLOCK_COUNT; 1854 } 1855 } 1856 1857 static int amdgpu_virt_cache_host_error_counts(struct amdgpu_device *adev, 1858 struct amdsriov_ras_telemetry *host_telemetry) 1859 { 1860 struct amd_sriov_ras_telemetry_error_count *tmp = NULL; 1861 uint32_t checksum, used_size; 1862 1863 checksum = host_telemetry->header.checksum; 1864 used_size = host_telemetry->header.used_size; 1865 1866 if (used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 1867 return 0; 1868 1869 tmp = kmemdup(&host_telemetry->body.error_count, used_size, GFP_KERNEL); 1870 if (!tmp) 1871 return -ENOMEM; 1872 1873 if (checksum != amd_sriov_msg_checksum(tmp, used_size, 0, 0)) 1874 goto out; 1875 1876 memcpy(&adev->virt.count_cache, tmp, 1877 min(used_size, sizeof(adev->virt.count_cache))); 1878 out: 1879 kfree(tmp); 1880 1881 return 0; 1882 } 1883 1884 static int amdgpu_virt_req_ras_err_count_internal(struct amdgpu_device *adev, bool force_update) 1885 { 1886 struct amdgpu_virt *virt = &adev->virt; 1887 1888 if (!virt->ops || !virt->ops->req_ras_err_count) 1889 return -EOPNOTSUPP; 1890 1891 /* Host allows 15 ras telemetry requests per 60 seconds. Afterwhich, the Host 1892 * will ignore incoming guest messages. Ratelimit the guest messages to 1893 * prevent guest self DOS. 1894 */ 1895 if (__ratelimit(&virt->ras.ras_error_cnt_rs) || force_update) { 1896 mutex_lock(&virt->ras.ras_telemetry_mutex); 1897 if (!virt->ops->req_ras_err_count(adev)) 1898 amdgpu_virt_cache_host_error_counts(adev, 1899 virt->fw_reserve.ras_telemetry); 1900 mutex_unlock(&virt->ras.ras_telemetry_mutex); 1901 } 1902 1903 return 0; 1904 } 1905 1906 /* Bypass ACA interface and query ECC counts directly from host */ 1907 int amdgpu_virt_req_ras_err_count(struct amdgpu_device *adev, enum amdgpu_ras_block block, 1908 struct ras_err_data *err_data) 1909 { 1910 enum amd_sriov_ras_telemetry_gpu_block sriov_block; 1911 1912 sriov_block = amdgpu_ras_block_to_sriov(adev, block); 1913 1914 if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT || 1915 !amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block)) 1916 return -EOPNOTSUPP; 1917 1918 /* Host Access may be lost during reset, just return last cached data. */ 1919 if (down_read_trylock(&adev->reset_domain->sem)) { 1920 amdgpu_virt_req_ras_err_count_internal(adev, false); 1921 up_read(&adev->reset_domain->sem); 1922 } 1923 1924 err_data->ue_count = adev->virt.count_cache.block[sriov_block].ue_count; 1925 err_data->ce_count = adev->virt.count_cache.block[sriov_block].ce_count; 1926 err_data->de_count = adev->virt.count_cache.block[sriov_block].de_count; 1927 1928 return 0; 1929 } 1930 1931 static int 1932 amdgpu_virt_write_cpers_to_ring(struct amdgpu_device *adev, 1933 struct amdsriov_ras_telemetry *host_telemetry, 1934 u32 *more) 1935 { 1936 struct amd_sriov_ras_cper_dump *cper_dump = NULL; 1937 struct cper_hdr *entry = NULL; 1938 struct amdgpu_ring *ring = &adev->cper.ring_buf; 1939 uint32_t checksum, used_size; 1940 u64 remaining, cnt, i; 1941 int ret = 0; 1942 1943 checksum = host_telemetry->header.checksum; 1944 used_size = host_telemetry->header.used_size; 1945 1946 if (used_size < offsetof(struct amd_sriov_ras_cper_dump, buf) || 1947 used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 1948 return -EINVAL; 1949 1950 cper_dump = kmemdup(&host_telemetry->body.cper_dump, used_size, GFP_KERNEL); 1951 if (!cper_dump) 1952 return -ENOMEM; 1953 1954 if (checksum != amd_sriov_msg_checksum(cper_dump, used_size, 0, 0)) { 1955 ret = -EINVAL; 1956 goto out; 1957 } 1958 1959 *more = cper_dump->more; 1960 1961 if (cper_dump->wptr < adev->virt.ras.cper_rptr) { 1962 dev_warn( 1963 adev->dev, 1964 "guest specified rptr that was too high! guest rptr: 0x%llx, host rptr: 0x%llx\n", 1965 adev->virt.ras.cper_rptr, cper_dump->wptr); 1966 1967 adev->virt.ras.cper_rptr = cper_dump->wptr; 1968 goto out; 1969 } 1970 1971 entry = (struct cper_hdr *)&cper_dump->buf[0]; 1972 remaining = (u64)used_size - offsetof(struct amd_sriov_ras_cper_dump, buf); 1973 cnt = min_t(u64, cper_dump->count, CPER_MAX_ALLOWED_COUNT); 1974 1975 for (i = 0; i < cnt; i++) { 1976 if (entry->record_length < sizeof(struct cper_hdr) || 1977 entry->record_length > remaining) { 1978 ret = -EINVAL; 1979 goto out; 1980 } 1981 1982 amdgpu_cper_ring_write(ring, entry, entry->record_length); 1983 remaining -= entry->record_length; 1984 entry = (struct cper_hdr *)((char *)entry + entry->record_length); 1985 } 1986 1987 if (cper_dump->overflow_count) 1988 dev_warn(adev->dev, 1989 "host reported CPER overflow of 0x%llx entries!\n", 1990 cper_dump->overflow_count); 1991 1992 adev->virt.ras.cper_rptr = cper_dump->wptr; 1993 out: 1994 kfree(cper_dump); 1995 1996 return ret; 1997 } 1998 1999 static int amdgpu_virt_req_ras_cper_dump_internal(struct amdgpu_device *adev) 2000 { 2001 struct amdgpu_virt *virt = &adev->virt; 2002 int ret = 0; 2003 uint32_t more = 0; 2004 2005 if (!virt->ops || !virt->ops->req_ras_cper_dump) 2006 return -EOPNOTSUPP; 2007 2008 do { 2009 if (!virt->ops->req_ras_cper_dump(adev, virt->ras.cper_rptr)) 2010 ret = amdgpu_virt_write_cpers_to_ring( 2011 adev, virt->fw_reserve.ras_telemetry, &more); 2012 else 2013 ret = 0; 2014 } while (more && !ret); 2015 2016 return ret; 2017 } 2018 2019 int amdgpu_virt_req_ras_cper_dump(struct amdgpu_device *adev, bool force_update) 2020 { 2021 struct amdgpu_virt *virt = &adev->virt; 2022 int ret = 0; 2023 2024 if (!amdgpu_sriov_ras_cper_en(adev)) 2025 return -EOPNOTSUPP; 2026 2027 if ((__ratelimit(&virt->ras.ras_cper_dump_rs) || force_update) && 2028 down_read_trylock(&adev->reset_domain->sem)) { 2029 mutex_lock(&virt->ras.ras_telemetry_mutex); 2030 ret = amdgpu_virt_req_ras_cper_dump_internal(adev); 2031 mutex_unlock(&virt->ras.ras_telemetry_mutex); 2032 up_read(&adev->reset_domain->sem); 2033 } 2034 2035 return ret; 2036 } 2037 2038 int amdgpu_virt_ras_telemetry_post_reset(struct amdgpu_device *adev) 2039 { 2040 unsigned long ue_count, ce_count; 2041 2042 if (amdgpu_sriov_ras_telemetry_en(adev)) { 2043 amdgpu_virt_req_ras_err_count_internal(adev, true); 2044 amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL); 2045 } 2046 2047 return 0; 2048 } 2049 2050 bool amdgpu_virt_ras_telemetry_block_en(struct amdgpu_device *adev, 2051 enum amdgpu_ras_block block) 2052 { 2053 enum amd_sriov_ras_telemetry_gpu_block sriov_block; 2054 2055 sriov_block = amdgpu_ras_block_to_sriov(adev, block); 2056 2057 if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT || 2058 !amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block)) 2059 return false; 2060 2061 return true; 2062 } 2063 2064 /* 2065 * amdgpu_virt_request_bad_pages() - request bad pages 2066 * @adev: amdgpu device. 2067 * Send command to GPU hypervisor to write new bad pages into the shared PF2VF region 2068 */ 2069 void amdgpu_virt_request_bad_pages(struct amdgpu_device *adev) 2070 { 2071 struct amdgpu_virt *virt = &adev->virt; 2072 2073 if (virt->ops && virt->ops->req_bad_pages) 2074 virt->ops->req_bad_pages(adev); 2075 } 2076 2077 static int amdgpu_virt_cache_chk_criti_hit(struct amdgpu_device *adev, 2078 struct amdsriov_ras_telemetry *host_telemetry, 2079 bool *hit) 2080 { 2081 struct amd_sriov_ras_chk_criti *tmp = NULL; 2082 uint32_t checksum, used_size; 2083 2084 checksum = host_telemetry->header.checksum; 2085 used_size = host_telemetry->header.used_size; 2086 2087 if (used_size > (AMD_SRIOV_MSG_RAS_TELEMETRY_SIZE_KB_V1 << 10)) 2088 return 0; 2089 2090 tmp = kmemdup(&host_telemetry->body.chk_criti, used_size, GFP_KERNEL); 2091 if (!tmp) 2092 return -ENOMEM; 2093 2094 if (checksum != amd_sriov_msg_checksum(tmp, used_size, 0, 0)) 2095 goto out; 2096 2097 if (hit) 2098 *hit = tmp->hit ? true : false; 2099 2100 out: 2101 kfree(tmp); 2102 2103 return 0; 2104 } 2105 2106 int amdgpu_virt_check_vf_critical_region(struct amdgpu_device *adev, u64 addr, bool *hit) 2107 { 2108 struct amdgpu_virt *virt = &adev->virt; 2109 int r = -EPERM; 2110 2111 if (!virt->ops || !virt->ops->req_ras_chk_criti) 2112 return -EOPNOTSUPP; 2113 2114 /* Host allows 15 ras telemetry requests per 60 seconds. Afterwhich, the Host 2115 * will ignore incoming guest messages. Ratelimit the guest messages to 2116 * prevent guest self DOS. 2117 */ 2118 if (__ratelimit(&virt->ras.ras_chk_criti_rs)) { 2119 mutex_lock(&virt->ras.ras_telemetry_mutex); 2120 if (!virt->ops->req_ras_chk_criti(adev, addr)) 2121 r = amdgpu_virt_cache_chk_criti_hit( 2122 adev, virt->fw_reserve.ras_telemetry, hit); 2123 mutex_unlock(&virt->ras.ras_telemetry_mutex); 2124 } 2125 2126 return r; 2127 } 2128 2129 static int req_remote_ras_cmd(struct amdgpu_device *adev, 2130 u32 param1, u32 param2, u32 param3) 2131 { 2132 struct amdgpu_virt *virt = &adev->virt; 2133 2134 if (virt->ops && virt->ops->req_remote_ras_cmd) 2135 return virt->ops->req_remote_ras_cmd(adev, param1, param2, param3); 2136 return -ENOENT; 2137 } 2138 2139 int amdgpu_virt_send_remote_ras_cmd(struct amdgpu_device *adev, 2140 uint64_t buf, uint32_t buf_len) 2141 { 2142 uint64_t gpa = buf; 2143 int ret = -EIO; 2144 2145 if (down_read_trylock(&adev->reset_domain->sem)) { 2146 ret = req_remote_ras_cmd(adev, 2147 lower_32_bits(gpa), upper_32_bits(gpa), buf_len); 2148 up_read(&adev->reset_domain->sem); 2149 } 2150 2151 return ret; 2152 } 2153 2154 int amdgpu_virt_ptl_request(struct amdgpu_device *adev, u32 req_code, 2155 uint32_t *ptl_state, uint32_t *fmt1, uint32_t *fmt2) 2156 { 2157 int ret; 2158 2159 if (!ptl_state || !fmt1 || !fmt2) 2160 return -EINVAL; 2161 2162 if (!amdgpu_sriov_ptl_support(adev) || 2163 !adev->virt.ops || !adev->virt.ops->req_ptl_update) 2164 return -EOPNOTSUPP; 2165 2166 if (req_code == PSP_PTL_PERF_MON_SET && *ptl_state) { 2167 if (*fmt1 == *fmt2) { 2168 dev_warn(adev->dev, 2169 "PTL formats must be different (fmt1=%u, fmt2=%u)\n", 2170 *fmt1, *fmt2); 2171 return -EINVAL; 2172 } 2173 } 2174 2175 ret = adev->virt.ops->req_ptl_update(adev, req_code, *ptl_state, *fmt1, *fmt2); 2176 if (ret) { 2177 dev_warn(adev->dev, "VF PTL update request failed: %d\n", ret); 2178 return ret; 2179 } 2180 2181 if (req_code == PSP_PTL_PERF_MON_QUERY) { 2182 *ptl_state = adev->virt.ptl_state; 2183 *fmt1 = adev->virt.ptl_pref_format1; 2184 *fmt2 = adev->virt.ptl_pref_format2; 2185 } 2186 2187 return 0; 2188 } 2189