1 /* 2 * Copyright 2011 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Christian König <deathsimple@vodafone.de> 29 */ 30 31 #include <linux/firmware.h> 32 #include <linux/module.h> 33 #include <drm/drmP.h> 34 #include <drm/drm.h> 35 36 #include "amdgpu.h" 37 #include "amdgpu_pm.h" 38 #include "amdgpu_uvd.h" 39 #include "cikd.h" 40 #include "uvd/uvd_4_2_d.h" 41 42 /* 1 second timeout */ 43 #define UVD_IDLE_TIMEOUT_MS 1000 44 45 /* Firmware Names */ 46 #ifdef CONFIG_DRM_AMDGPU_CIK 47 #define FIRMWARE_BONAIRE "radeon/bonaire_uvd.bin" 48 #define FIRMWARE_KABINI "radeon/kabini_uvd.bin" 49 #define FIRMWARE_KAVERI "radeon/kaveri_uvd.bin" 50 #define FIRMWARE_HAWAII "radeon/hawaii_uvd.bin" 51 #define FIRMWARE_MULLINS "radeon/mullins_uvd.bin" 52 #endif 53 #define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin" 54 #define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin" 55 #define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin" 56 #define FIRMWARE_STONEY "amdgpu/stoney_uvd.bin" 57 58 /** 59 * amdgpu_uvd_cs_ctx - Command submission parser context 60 * 61 * Used for emulating virtual memory support on UVD 4.2. 62 */ 63 struct amdgpu_uvd_cs_ctx { 64 struct amdgpu_cs_parser *parser; 65 unsigned reg, count; 66 unsigned data0, data1; 67 unsigned idx; 68 unsigned ib_idx; 69 70 /* does the IB has a msg command */ 71 bool has_msg_cmd; 72 73 /* minimum buffer sizes */ 74 unsigned *buf_sizes; 75 }; 76 77 #ifdef CONFIG_DRM_AMDGPU_CIK 78 MODULE_FIRMWARE(FIRMWARE_BONAIRE); 79 MODULE_FIRMWARE(FIRMWARE_KABINI); 80 MODULE_FIRMWARE(FIRMWARE_KAVERI); 81 MODULE_FIRMWARE(FIRMWARE_HAWAII); 82 MODULE_FIRMWARE(FIRMWARE_MULLINS); 83 #endif 84 MODULE_FIRMWARE(FIRMWARE_TONGA); 85 MODULE_FIRMWARE(FIRMWARE_CARRIZO); 86 MODULE_FIRMWARE(FIRMWARE_FIJI); 87 MODULE_FIRMWARE(FIRMWARE_STONEY); 88 89 static void amdgpu_uvd_note_usage(struct amdgpu_device *adev); 90 static void amdgpu_uvd_idle_work_handler(struct work_struct *work); 91 92 int amdgpu_uvd_sw_init(struct amdgpu_device *adev) 93 { 94 struct amdgpu_ring *ring; 95 struct amd_sched_rq *rq; 96 unsigned long bo_size; 97 const char *fw_name; 98 const struct common_firmware_header *hdr; 99 unsigned version_major, version_minor, family_id; 100 int i, r; 101 102 INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler); 103 104 switch (adev->asic_type) { 105 #ifdef CONFIG_DRM_AMDGPU_CIK 106 case CHIP_BONAIRE: 107 fw_name = FIRMWARE_BONAIRE; 108 break; 109 case CHIP_KABINI: 110 fw_name = FIRMWARE_KABINI; 111 break; 112 case CHIP_KAVERI: 113 fw_name = FIRMWARE_KAVERI; 114 break; 115 case CHIP_HAWAII: 116 fw_name = FIRMWARE_HAWAII; 117 break; 118 case CHIP_MULLINS: 119 fw_name = FIRMWARE_MULLINS; 120 break; 121 #endif 122 case CHIP_TONGA: 123 fw_name = FIRMWARE_TONGA; 124 break; 125 case CHIP_FIJI: 126 fw_name = FIRMWARE_FIJI; 127 break; 128 case CHIP_CARRIZO: 129 fw_name = FIRMWARE_CARRIZO; 130 break; 131 case CHIP_STONEY: 132 fw_name = FIRMWARE_STONEY; 133 break; 134 default: 135 return -EINVAL; 136 } 137 138 r = request_firmware(&adev->uvd.fw, fw_name, adev->dev); 139 if (r) { 140 dev_err(adev->dev, "amdgpu_uvd: Can't load firmware \"%s\"\n", 141 fw_name); 142 return r; 143 } 144 145 r = amdgpu_ucode_validate(adev->uvd.fw); 146 if (r) { 147 dev_err(adev->dev, "amdgpu_uvd: Can't validate firmware \"%s\"\n", 148 fw_name); 149 release_firmware(adev->uvd.fw); 150 adev->uvd.fw = NULL; 151 return r; 152 } 153 154 /* Set the default UVD handles that the firmware can handle */ 155 adev->uvd.max_handles = AMDGPU_DEFAULT_UVD_HANDLES; 156 157 hdr = (const struct common_firmware_header *)adev->uvd.fw->data; 158 family_id = le32_to_cpu(hdr->ucode_version) & 0xff; 159 version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff; 160 version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff; 161 DRM_INFO("Found UVD firmware Version: %hu.%hu Family ID: %hu\n", 162 version_major, version_minor, family_id); 163 164 /* 165 * Limit the number of UVD handles depending on microcode major 166 * and minor versions. The firmware version which has 40 UVD 167 * instances support is 1.80. So all subsequent versions should 168 * also have the same support. 169 */ 170 if ((version_major > 0x01) || 171 ((version_major == 0x01) && (version_minor >= 0x50))) 172 adev->uvd.max_handles = AMDGPU_MAX_UVD_HANDLES; 173 174 bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8) 175 + AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE 176 + AMDGPU_UVD_SESSION_SIZE * adev->uvd.max_handles; 177 r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true, 178 AMDGPU_GEM_DOMAIN_VRAM, 179 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, 180 NULL, NULL, &adev->uvd.vcpu_bo); 181 if (r) { 182 dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r); 183 return r; 184 } 185 186 r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false); 187 if (r) { 188 amdgpu_bo_unref(&adev->uvd.vcpu_bo); 189 dev_err(adev->dev, "(%d) failed to reserve UVD bo\n", r); 190 return r; 191 } 192 193 r = amdgpu_bo_pin(adev->uvd.vcpu_bo, AMDGPU_GEM_DOMAIN_VRAM, 194 &adev->uvd.gpu_addr); 195 if (r) { 196 amdgpu_bo_unreserve(adev->uvd.vcpu_bo); 197 amdgpu_bo_unref(&adev->uvd.vcpu_bo); 198 dev_err(adev->dev, "(%d) UVD bo pin failed\n", r); 199 return r; 200 } 201 202 r = amdgpu_bo_kmap(adev->uvd.vcpu_bo, &adev->uvd.cpu_addr); 203 if (r) { 204 dev_err(adev->dev, "(%d) UVD map failed\n", r); 205 return r; 206 } 207 208 amdgpu_bo_unreserve(adev->uvd.vcpu_bo); 209 210 ring = &adev->uvd.ring; 211 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL]; 212 r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity, 213 rq, amdgpu_sched_jobs); 214 if (r != 0) { 215 DRM_ERROR("Failed setting up UVD run queue.\n"); 216 return r; 217 } 218 219 for (i = 0; i < adev->uvd.max_handles; ++i) { 220 atomic_set(&adev->uvd.handles[i], 0); 221 adev->uvd.filp[i] = NULL; 222 } 223 224 /* from uvd v5.0 HW addressing capacity increased to 64 bits */ 225 if (!amdgpu_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) 226 adev->uvd.address_64_bit = true; 227 228 return 0; 229 } 230 231 int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) 232 { 233 int r; 234 235 if (adev->uvd.vcpu_bo == NULL) 236 return 0; 237 238 amd_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity); 239 240 r = amdgpu_bo_reserve(adev->uvd.vcpu_bo, false); 241 if (!r) { 242 amdgpu_bo_kunmap(adev->uvd.vcpu_bo); 243 amdgpu_bo_unpin(adev->uvd.vcpu_bo); 244 amdgpu_bo_unreserve(adev->uvd.vcpu_bo); 245 } 246 247 amdgpu_bo_unref(&adev->uvd.vcpu_bo); 248 249 amdgpu_ring_fini(&adev->uvd.ring); 250 251 release_firmware(adev->uvd.fw); 252 253 return 0; 254 } 255 256 int amdgpu_uvd_suspend(struct amdgpu_device *adev) 257 { 258 unsigned size; 259 void *ptr; 260 int i; 261 262 if (adev->uvd.vcpu_bo == NULL) 263 return 0; 264 265 for (i = 0; i < adev->uvd.max_handles; ++i) 266 if (atomic_read(&adev->uvd.handles[i])) 267 break; 268 269 if (i == AMDGPU_MAX_UVD_HANDLES) 270 return 0; 271 272 size = amdgpu_bo_size(adev->uvd.vcpu_bo); 273 ptr = adev->uvd.cpu_addr; 274 275 adev->uvd.saved_bo = kmalloc(size, GFP_KERNEL); 276 if (!adev->uvd.saved_bo) 277 return -ENOMEM; 278 279 memcpy(adev->uvd.saved_bo, ptr, size); 280 281 return 0; 282 } 283 284 int amdgpu_uvd_resume(struct amdgpu_device *adev) 285 { 286 unsigned size; 287 void *ptr; 288 289 if (adev->uvd.vcpu_bo == NULL) 290 return -EINVAL; 291 292 size = amdgpu_bo_size(adev->uvd.vcpu_bo); 293 ptr = adev->uvd.cpu_addr; 294 295 if (adev->uvd.saved_bo != NULL) { 296 memcpy(ptr, adev->uvd.saved_bo, size); 297 kfree(adev->uvd.saved_bo); 298 adev->uvd.saved_bo = NULL; 299 } else { 300 const struct common_firmware_header *hdr; 301 unsigned offset; 302 303 hdr = (const struct common_firmware_header *)adev->uvd.fw->data; 304 offset = le32_to_cpu(hdr->ucode_array_offset_bytes); 305 memcpy(adev->uvd.cpu_addr, (adev->uvd.fw->data) + offset, 306 (adev->uvd.fw->size) - offset); 307 size -= le32_to_cpu(hdr->ucode_size_bytes); 308 ptr += le32_to_cpu(hdr->ucode_size_bytes); 309 memset(ptr, 0, size); 310 } 311 312 return 0; 313 } 314 315 void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) 316 { 317 struct amdgpu_ring *ring = &adev->uvd.ring; 318 int i, r; 319 320 for (i = 0; i < adev->uvd.max_handles; ++i) { 321 uint32_t handle = atomic_read(&adev->uvd.handles[i]); 322 if (handle != 0 && adev->uvd.filp[i] == filp) { 323 struct fence *fence; 324 325 amdgpu_uvd_note_usage(adev); 326 327 r = amdgpu_uvd_get_destroy_msg(ring, handle, 328 false, &fence); 329 if (r) { 330 DRM_ERROR("Error destroying UVD (%d)!\n", r); 331 continue; 332 } 333 334 fence_wait(fence, false); 335 fence_put(fence); 336 337 adev->uvd.filp[i] = NULL; 338 atomic_set(&adev->uvd.handles[i], 0); 339 } 340 } 341 } 342 343 static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *rbo) 344 { 345 int i; 346 for (i = 0; i < rbo->placement.num_placement; ++i) { 347 rbo->placements[i].fpfn = 0 >> PAGE_SHIFT; 348 rbo->placements[i].lpfn = (256 * 1024 * 1024) >> PAGE_SHIFT; 349 } 350 } 351 352 /** 353 * amdgpu_uvd_cs_pass1 - first parsing round 354 * 355 * @ctx: UVD parser context 356 * 357 * Make sure UVD message and feedback buffers are in VRAM and 358 * nobody is violating an 256MB boundary. 359 */ 360 static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) 361 { 362 struct amdgpu_bo_va_mapping *mapping; 363 struct amdgpu_bo *bo; 364 uint32_t cmd, lo, hi; 365 uint64_t addr; 366 int r = 0; 367 368 lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0); 369 hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1); 370 addr = ((uint64_t)lo) | (((uint64_t)hi) << 32); 371 372 mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo); 373 if (mapping == NULL) { 374 DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr); 375 return -EINVAL; 376 } 377 378 if (!ctx->parser->adev->uvd.address_64_bit) { 379 /* check if it's a message or feedback command */ 380 cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1; 381 if (cmd == 0x0 || cmd == 0x3) { 382 /* yes, force it into VRAM */ 383 uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; 384 amdgpu_ttm_placement_from_domain(bo, domain); 385 } 386 amdgpu_uvd_force_into_uvd_segment(bo); 387 388 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 389 } 390 391 return r; 392 } 393 394 /** 395 * amdgpu_uvd_cs_msg_decode - handle UVD decode message 396 * 397 * @msg: pointer to message structure 398 * @buf_sizes: returned buffer sizes 399 * 400 * Peek into the decode message and calculate the necessary buffer sizes. 401 */ 402 static int amdgpu_uvd_cs_msg_decode(uint32_t *msg, unsigned buf_sizes[]) 403 { 404 unsigned stream_type = msg[4]; 405 unsigned width = msg[6]; 406 unsigned height = msg[7]; 407 unsigned dpb_size = msg[9]; 408 unsigned pitch = msg[28]; 409 unsigned level = msg[57]; 410 411 unsigned width_in_mb = width / 16; 412 unsigned height_in_mb = ALIGN(height / 16, 2); 413 unsigned fs_in_mb = width_in_mb * height_in_mb; 414 415 unsigned image_size, tmp, min_dpb_size, num_dpb_buffer; 416 unsigned min_ctx_size = 0; 417 418 image_size = width * height; 419 image_size += image_size / 2; 420 image_size = ALIGN(image_size, 1024); 421 422 switch (stream_type) { 423 case 0: /* H264 */ 424 case 7: /* H264 Perf */ 425 switch(level) { 426 case 30: 427 num_dpb_buffer = 8100 / fs_in_mb; 428 break; 429 case 31: 430 num_dpb_buffer = 18000 / fs_in_mb; 431 break; 432 case 32: 433 num_dpb_buffer = 20480 / fs_in_mb; 434 break; 435 case 41: 436 num_dpb_buffer = 32768 / fs_in_mb; 437 break; 438 case 42: 439 num_dpb_buffer = 34816 / fs_in_mb; 440 break; 441 case 50: 442 num_dpb_buffer = 110400 / fs_in_mb; 443 break; 444 case 51: 445 num_dpb_buffer = 184320 / fs_in_mb; 446 break; 447 default: 448 num_dpb_buffer = 184320 / fs_in_mb; 449 break; 450 } 451 num_dpb_buffer++; 452 if (num_dpb_buffer > 17) 453 num_dpb_buffer = 17; 454 455 /* reference picture buffer */ 456 min_dpb_size = image_size * num_dpb_buffer; 457 458 /* macroblock context buffer */ 459 min_dpb_size += width_in_mb * height_in_mb * num_dpb_buffer * 192; 460 461 /* IT surface buffer */ 462 min_dpb_size += width_in_mb * height_in_mb * 32; 463 break; 464 465 case 1: /* VC1 */ 466 467 /* reference picture buffer */ 468 min_dpb_size = image_size * 3; 469 470 /* CONTEXT_BUFFER */ 471 min_dpb_size += width_in_mb * height_in_mb * 128; 472 473 /* IT surface buffer */ 474 min_dpb_size += width_in_mb * 64; 475 476 /* DB surface buffer */ 477 min_dpb_size += width_in_mb * 128; 478 479 /* BP */ 480 tmp = max(width_in_mb, height_in_mb); 481 min_dpb_size += ALIGN(tmp * 7 * 16, 64); 482 break; 483 484 case 3: /* MPEG2 */ 485 486 /* reference picture buffer */ 487 min_dpb_size = image_size * 3; 488 break; 489 490 case 4: /* MPEG4 */ 491 492 /* reference picture buffer */ 493 min_dpb_size = image_size * 3; 494 495 /* CM */ 496 min_dpb_size += width_in_mb * height_in_mb * 64; 497 498 /* IT surface buffer */ 499 min_dpb_size += ALIGN(width_in_mb * height_in_mb * 32, 64); 500 break; 501 502 case 16: /* H265 */ 503 image_size = (ALIGN(width, 16) * ALIGN(height, 16) * 3) / 2; 504 image_size = ALIGN(image_size, 256); 505 506 num_dpb_buffer = (le32_to_cpu(msg[59]) & 0xff) + 2; 507 min_dpb_size = image_size * num_dpb_buffer; 508 min_ctx_size = ((width + 255) / 16) * ((height + 255) / 16) 509 * 16 * num_dpb_buffer + 52 * 1024; 510 break; 511 512 default: 513 DRM_ERROR("UVD codec not handled %d!\n", stream_type); 514 return -EINVAL; 515 } 516 517 if (width > pitch) { 518 DRM_ERROR("Invalid UVD decoding target pitch!\n"); 519 return -EINVAL; 520 } 521 522 if (dpb_size < min_dpb_size) { 523 DRM_ERROR("Invalid dpb_size in UVD message (%d / %d)!\n", 524 dpb_size, min_dpb_size); 525 return -EINVAL; 526 } 527 528 buf_sizes[0x1] = dpb_size; 529 buf_sizes[0x2] = image_size; 530 buf_sizes[0x4] = min_ctx_size; 531 return 0; 532 } 533 534 /** 535 * amdgpu_uvd_cs_msg - handle UVD message 536 * 537 * @ctx: UVD parser context 538 * @bo: buffer object containing the message 539 * @offset: offset into the buffer object 540 * 541 * Peek into the UVD message and extract the session id. 542 * Make sure that we don't open up to many sessions. 543 */ 544 static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx, 545 struct amdgpu_bo *bo, unsigned offset) 546 { 547 struct amdgpu_device *adev = ctx->parser->adev; 548 int32_t *msg, msg_type, handle; 549 void *ptr; 550 long r; 551 int i; 552 553 if (offset & 0x3F) { 554 DRM_ERROR("UVD messages must be 64 byte aligned!\n"); 555 return -EINVAL; 556 } 557 558 r = amdgpu_bo_kmap(bo, &ptr); 559 if (r) { 560 DRM_ERROR("Failed mapping the UVD message (%ld)!\n", r); 561 return r; 562 } 563 564 msg = ptr + offset; 565 566 msg_type = msg[1]; 567 handle = msg[2]; 568 569 if (handle == 0) { 570 DRM_ERROR("Invalid UVD handle!\n"); 571 return -EINVAL; 572 } 573 574 switch (msg_type) { 575 case 0: 576 /* it's a create msg, calc image size (width * height) */ 577 amdgpu_bo_kunmap(bo); 578 579 /* try to alloc a new handle */ 580 for (i = 0; i < adev->uvd.max_handles; ++i) { 581 if (atomic_read(&adev->uvd.handles[i]) == handle) { 582 DRM_ERROR("Handle 0x%x already in use!\n", handle); 583 return -EINVAL; 584 } 585 586 if (!atomic_cmpxchg(&adev->uvd.handles[i], 0, handle)) { 587 adev->uvd.filp[i] = ctx->parser->filp; 588 return 0; 589 } 590 } 591 592 DRM_ERROR("No more free UVD handles!\n"); 593 return -EINVAL; 594 595 case 1: 596 /* it's a decode msg, calc buffer sizes */ 597 r = amdgpu_uvd_cs_msg_decode(msg, ctx->buf_sizes); 598 amdgpu_bo_kunmap(bo); 599 if (r) 600 return r; 601 602 /* validate the handle */ 603 for (i = 0; i < adev->uvd.max_handles; ++i) { 604 if (atomic_read(&adev->uvd.handles[i]) == handle) { 605 if (adev->uvd.filp[i] != ctx->parser->filp) { 606 DRM_ERROR("UVD handle collision detected!\n"); 607 return -EINVAL; 608 } 609 return 0; 610 } 611 } 612 613 DRM_ERROR("Invalid UVD handle 0x%x!\n", handle); 614 return -ENOENT; 615 616 case 2: 617 /* it's a destroy msg, free the handle */ 618 for (i = 0; i < adev->uvd.max_handles; ++i) 619 atomic_cmpxchg(&adev->uvd.handles[i], handle, 0); 620 amdgpu_bo_kunmap(bo); 621 return 0; 622 623 default: 624 DRM_ERROR("Illegal UVD message type (%d)!\n", msg_type); 625 return -EINVAL; 626 } 627 BUG(); 628 return -EINVAL; 629 } 630 631 /** 632 * amdgpu_uvd_cs_pass2 - second parsing round 633 * 634 * @ctx: UVD parser context 635 * 636 * Patch buffer addresses, make sure buffer sizes are correct. 637 */ 638 static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx) 639 { 640 struct amdgpu_bo_va_mapping *mapping; 641 struct amdgpu_bo *bo; 642 uint32_t cmd, lo, hi; 643 uint64_t start, end; 644 uint64_t addr; 645 int r; 646 647 lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0); 648 hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1); 649 addr = ((uint64_t)lo) | (((uint64_t)hi) << 32); 650 651 mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo); 652 if (mapping == NULL) 653 return -EINVAL; 654 655 start = amdgpu_bo_gpu_offset(bo); 656 657 end = (mapping->it.last + 1 - mapping->it.start); 658 end = end * AMDGPU_GPU_PAGE_SIZE + start; 659 660 addr -= ((uint64_t)mapping->it.start) * AMDGPU_GPU_PAGE_SIZE; 661 start += addr; 662 663 amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data0, 664 lower_32_bits(start)); 665 amdgpu_set_ib_value(ctx->parser, ctx->ib_idx, ctx->data1, 666 upper_32_bits(start)); 667 668 cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx) >> 1; 669 if (cmd < 0x4) { 670 if ((end - start) < ctx->buf_sizes[cmd]) { 671 DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, 672 (unsigned)(end - start), 673 ctx->buf_sizes[cmd]); 674 return -EINVAL; 675 } 676 677 } else if (cmd == 0x206) { 678 if ((end - start) < ctx->buf_sizes[4]) { 679 DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, 680 (unsigned)(end - start), 681 ctx->buf_sizes[4]); 682 return -EINVAL; 683 } 684 } else if ((cmd != 0x100) && (cmd != 0x204)) { 685 DRM_ERROR("invalid UVD command %X!\n", cmd); 686 return -EINVAL; 687 } 688 689 if (!ctx->parser->adev->uvd.address_64_bit) { 690 if ((start >> 28) != ((end - 1) >> 28)) { 691 DRM_ERROR("reloc %LX-%LX crossing 256MB boundary!\n", 692 start, end); 693 return -EINVAL; 694 } 695 696 if ((cmd == 0 || cmd == 0x3) && 697 (start >> 28) != (ctx->parser->adev->uvd.gpu_addr >> 28)) { 698 DRM_ERROR("msg/fb buffer %LX-%LX out of 256MB segment!\n", 699 start, end); 700 return -EINVAL; 701 } 702 } 703 704 if (cmd == 0) { 705 ctx->has_msg_cmd = true; 706 r = amdgpu_uvd_cs_msg(ctx, bo, addr); 707 if (r) 708 return r; 709 } else if (!ctx->has_msg_cmd) { 710 DRM_ERROR("Message needed before other commands are send!\n"); 711 return -EINVAL; 712 } 713 714 return 0; 715 } 716 717 /** 718 * amdgpu_uvd_cs_reg - parse register writes 719 * 720 * @ctx: UVD parser context 721 * @cb: callback function 722 * 723 * Parse the register writes, call cb on each complete command. 724 */ 725 static int amdgpu_uvd_cs_reg(struct amdgpu_uvd_cs_ctx *ctx, 726 int (*cb)(struct amdgpu_uvd_cs_ctx *ctx)) 727 { 728 struct amdgpu_ib *ib = &ctx->parser->job->ibs[ctx->ib_idx]; 729 int i, r; 730 731 ctx->idx++; 732 for (i = 0; i <= ctx->count; ++i) { 733 unsigned reg = ctx->reg + i; 734 735 if (ctx->idx >= ib->length_dw) { 736 DRM_ERROR("Register command after end of CS!\n"); 737 return -EINVAL; 738 } 739 740 switch (reg) { 741 case mmUVD_GPCOM_VCPU_DATA0: 742 ctx->data0 = ctx->idx; 743 break; 744 case mmUVD_GPCOM_VCPU_DATA1: 745 ctx->data1 = ctx->idx; 746 break; 747 case mmUVD_GPCOM_VCPU_CMD: 748 r = cb(ctx); 749 if (r) 750 return r; 751 break; 752 case mmUVD_ENGINE_CNTL: 753 break; 754 default: 755 DRM_ERROR("Invalid reg 0x%X!\n", reg); 756 return -EINVAL; 757 } 758 ctx->idx++; 759 } 760 return 0; 761 } 762 763 /** 764 * amdgpu_uvd_cs_packets - parse UVD packets 765 * 766 * @ctx: UVD parser context 767 * @cb: callback function 768 * 769 * Parse the command stream packets. 770 */ 771 static int amdgpu_uvd_cs_packets(struct amdgpu_uvd_cs_ctx *ctx, 772 int (*cb)(struct amdgpu_uvd_cs_ctx *ctx)) 773 { 774 struct amdgpu_ib *ib = &ctx->parser->job->ibs[ctx->ib_idx]; 775 int r; 776 777 for (ctx->idx = 0 ; ctx->idx < ib->length_dw; ) { 778 uint32_t cmd = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->idx); 779 unsigned type = CP_PACKET_GET_TYPE(cmd); 780 switch (type) { 781 case PACKET_TYPE0: 782 ctx->reg = CP_PACKET0_GET_REG(cmd); 783 ctx->count = CP_PACKET_GET_COUNT(cmd); 784 r = amdgpu_uvd_cs_reg(ctx, cb); 785 if (r) 786 return r; 787 break; 788 case PACKET_TYPE2: 789 ++ctx->idx; 790 break; 791 default: 792 DRM_ERROR("Unknown packet type %d !\n", type); 793 return -EINVAL; 794 } 795 } 796 return 0; 797 } 798 799 /** 800 * amdgpu_uvd_ring_parse_cs - UVD command submission parser 801 * 802 * @parser: Command submission parser context 803 * 804 * Parse the command stream, patch in addresses as necessary. 805 */ 806 int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx) 807 { 808 struct amdgpu_uvd_cs_ctx ctx = {}; 809 unsigned buf_sizes[] = { 810 [0x00000000] = 2048, 811 [0x00000001] = 0xFFFFFFFF, 812 [0x00000002] = 0xFFFFFFFF, 813 [0x00000003] = 2048, 814 [0x00000004] = 0xFFFFFFFF, 815 }; 816 struct amdgpu_ib *ib = &parser->job->ibs[ib_idx]; 817 int r; 818 819 if (ib->length_dw % 16) { 820 DRM_ERROR("UVD IB length (%d) not 16 dwords aligned!\n", 821 ib->length_dw); 822 return -EINVAL; 823 } 824 825 ctx.parser = parser; 826 ctx.buf_sizes = buf_sizes; 827 ctx.ib_idx = ib_idx; 828 829 /* first round, make sure the buffers are actually in the UVD segment */ 830 r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass1); 831 if (r) 832 return r; 833 834 /* second round, patch buffer addresses into the command stream */ 835 r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass2); 836 if (r) 837 return r; 838 839 if (!ctx.has_msg_cmd) { 840 DRM_ERROR("UVD-IBs need a msg command!\n"); 841 return -EINVAL; 842 } 843 844 amdgpu_uvd_note_usage(ctx.parser->adev); 845 846 return 0; 847 } 848 849 static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, 850 bool direct, struct fence **fence) 851 { 852 struct ttm_validate_buffer tv; 853 struct ww_acquire_ctx ticket; 854 struct list_head head; 855 struct amdgpu_job *job; 856 struct amdgpu_ib *ib; 857 struct fence *f = NULL; 858 struct amdgpu_device *adev = ring->adev; 859 uint64_t addr; 860 int i, r; 861 862 memset(&tv, 0, sizeof(tv)); 863 tv.bo = &bo->tbo; 864 865 INIT_LIST_HEAD(&head); 866 list_add(&tv.head, &head); 867 868 r = ttm_eu_reserve_buffers(&ticket, &head, true, NULL); 869 if (r) 870 return r; 871 872 if (!bo->adev->uvd.address_64_bit) { 873 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); 874 amdgpu_uvd_force_into_uvd_segment(bo); 875 } 876 877 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 878 if (r) 879 goto err; 880 881 r = amdgpu_job_alloc_with_ib(adev, 64, &job); 882 if (r) 883 goto err; 884 885 ib = &job->ibs[0]; 886 addr = amdgpu_bo_gpu_offset(bo); 887 ib->ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0); 888 ib->ptr[1] = addr; 889 ib->ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0); 890 ib->ptr[3] = addr >> 32; 891 ib->ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0); 892 ib->ptr[5] = 0; 893 for (i = 6; i < 16; ++i) 894 ib->ptr[i] = PACKET2(0); 895 ib->length_dw = 16; 896 897 if (direct) { 898 r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f); 899 job->fence = f; 900 if (r) 901 goto err_free; 902 903 amdgpu_job_free(job); 904 } else { 905 r = amdgpu_job_submit(job, ring, &adev->uvd.entity, 906 AMDGPU_FENCE_OWNER_UNDEFINED, &f); 907 if (r) 908 goto err_free; 909 } 910 911 ttm_eu_fence_buffer_objects(&ticket, &head, f); 912 913 if (fence) 914 *fence = fence_get(f); 915 amdgpu_bo_unref(&bo); 916 fence_put(f); 917 918 return 0; 919 920 err_free: 921 amdgpu_job_free(job); 922 923 err: 924 ttm_eu_backoff_reservation(&ticket, &head); 925 return r; 926 } 927 928 /* multiple fence commands without any stream commands in between can 929 crash the vcpu so just try to emmit a dummy create/destroy msg to 930 avoid this */ 931 int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, 932 struct fence **fence) 933 { 934 struct amdgpu_device *adev = ring->adev; 935 struct amdgpu_bo *bo; 936 uint32_t *msg; 937 int r, i; 938 939 r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true, 940 AMDGPU_GEM_DOMAIN_VRAM, 941 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, 942 NULL, NULL, &bo); 943 if (r) 944 return r; 945 946 r = amdgpu_bo_reserve(bo, false); 947 if (r) { 948 amdgpu_bo_unref(&bo); 949 return r; 950 } 951 952 r = amdgpu_bo_kmap(bo, (void **)&msg); 953 if (r) { 954 amdgpu_bo_unreserve(bo); 955 amdgpu_bo_unref(&bo); 956 return r; 957 } 958 959 /* stitch together an UVD create msg */ 960 msg[0] = cpu_to_le32(0x00000de4); 961 msg[1] = cpu_to_le32(0x00000000); 962 msg[2] = cpu_to_le32(handle); 963 msg[3] = cpu_to_le32(0x00000000); 964 msg[4] = cpu_to_le32(0x00000000); 965 msg[5] = cpu_to_le32(0x00000000); 966 msg[6] = cpu_to_le32(0x00000000); 967 msg[7] = cpu_to_le32(0x00000780); 968 msg[8] = cpu_to_le32(0x00000440); 969 msg[9] = cpu_to_le32(0x00000000); 970 msg[10] = cpu_to_le32(0x01b37000); 971 for (i = 11; i < 1024; ++i) 972 msg[i] = cpu_to_le32(0x0); 973 974 amdgpu_bo_kunmap(bo); 975 amdgpu_bo_unreserve(bo); 976 977 return amdgpu_uvd_send_msg(ring, bo, true, fence); 978 } 979 980 int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, 981 bool direct, struct fence **fence) 982 { 983 struct amdgpu_device *adev = ring->adev; 984 struct amdgpu_bo *bo; 985 uint32_t *msg; 986 int r, i; 987 988 r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true, 989 AMDGPU_GEM_DOMAIN_VRAM, 990 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, 991 NULL, NULL, &bo); 992 if (r) 993 return r; 994 995 r = amdgpu_bo_reserve(bo, false); 996 if (r) { 997 amdgpu_bo_unref(&bo); 998 return r; 999 } 1000 1001 r = amdgpu_bo_kmap(bo, (void **)&msg); 1002 if (r) { 1003 amdgpu_bo_unreserve(bo); 1004 amdgpu_bo_unref(&bo); 1005 return r; 1006 } 1007 1008 /* stitch together an UVD destroy msg */ 1009 msg[0] = cpu_to_le32(0x00000de4); 1010 msg[1] = cpu_to_le32(0x00000002); 1011 msg[2] = cpu_to_le32(handle); 1012 msg[3] = cpu_to_le32(0x00000000); 1013 for (i = 4; i < 1024; ++i) 1014 msg[i] = cpu_to_le32(0x0); 1015 1016 amdgpu_bo_kunmap(bo); 1017 amdgpu_bo_unreserve(bo); 1018 1019 return amdgpu_uvd_send_msg(ring, bo, direct, fence); 1020 } 1021 1022 static void amdgpu_uvd_idle_work_handler(struct work_struct *work) 1023 { 1024 struct amdgpu_device *adev = 1025 container_of(work, struct amdgpu_device, uvd.idle_work.work); 1026 unsigned i, fences, handles = 0; 1027 1028 fences = amdgpu_fence_count_emitted(&adev->uvd.ring); 1029 1030 for (i = 0; i < adev->uvd.max_handles; ++i) 1031 if (atomic_read(&adev->uvd.handles[i])) 1032 ++handles; 1033 1034 if (fences == 0 && handles == 0) { 1035 if (adev->pm.dpm_enabled) { 1036 amdgpu_dpm_enable_uvd(adev, false); 1037 } else { 1038 amdgpu_asic_set_uvd_clocks(adev, 0, 0); 1039 } 1040 } else { 1041 schedule_delayed_work(&adev->uvd.idle_work, 1042 msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); 1043 } 1044 } 1045 1046 static void amdgpu_uvd_note_usage(struct amdgpu_device *adev) 1047 { 1048 bool set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work); 1049 set_clocks &= schedule_delayed_work(&adev->uvd.idle_work, 1050 msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); 1051 1052 if (set_clocks) { 1053 if (adev->pm.dpm_enabled) { 1054 amdgpu_dpm_enable_uvd(adev, true); 1055 } else { 1056 amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); 1057 } 1058 } 1059 } 1060