1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2017 Linaro Ltd. 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "hfi_cmds.h" 17 #include "hfi_msgs.h" 18 #include "hfi_venus.h" 19 #include "hfi_venus_io.h" 20 #include "firmware.h" 21 22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000 23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000 24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00 25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff 26 27 #define HFI_HOST_TO_CTRL_CMD_Q 0 28 #define HFI_CTRL_TO_HOST_MSG_Q 1 29 #define HFI_CTRL_TO_HOST_DBG_Q 2 30 #define HFI_MASK_QHDR_STATUS 0x000000ff 31 32 #define IFACEQ_NUM 3 33 #define IFACEQ_CMD_IDX 0 34 #define IFACEQ_MSG_IDX 1 35 #define IFACEQ_DBG_IDX 2 36 #define IFACEQ_MAX_BUF_COUNT 50 37 #define IFACEQ_MAX_PARALLEL_CLNTS 16 38 #define IFACEQ_DFLT_QHDR 0x01010000 39 40 #define POLL_INTERVAL_US 50 41 42 #define IFACEQ_MAX_PKT_SIZE 1024 43 #define IFACEQ_MED_PKT_SIZE 768 44 #define IFACEQ_MIN_PKT_SIZE 8 45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100 46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512 47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12) 48 49 struct hfi_queue_table_header { 50 u32 version; 51 u32 size; 52 u32 qhdr0_offset; 53 u32 qhdr_size; 54 u32 num_q; 55 u32 num_active_q; 56 }; 57 58 struct hfi_queue_header { 59 u32 status; 60 u32 start_addr; 61 u32 type; 62 u32 q_size; 63 u32 pkt_size; 64 u32 pkt_drop_cnt; 65 u32 rx_wm; 66 u32 tx_wm; 67 u32 rx_req; 68 u32 tx_req; 69 u32 rx_irq_status; 70 u32 tx_irq_status; 71 u32 read_idx; 72 u32 write_idx; 73 }; 74 75 #define IFACEQ_TABLE_SIZE \ 76 (sizeof(struct hfi_queue_table_header) + \ 77 sizeof(struct hfi_queue_header) * IFACEQ_NUM) 78 79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \ 80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS) 81 82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \ 83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \ 84 ((i) * sizeof(struct hfi_queue_header))) 85 86 #define QDSS_SIZE SZ_4K 87 #define SFR_SIZE SZ_4K 88 #define QUEUE_SIZE \ 89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM)) 90 91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K) 92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K) 93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K) 94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \ 95 ALIGNED_QDSS_SIZE, SZ_1M) 96 97 struct mem_desc { 98 dma_addr_t da; /* device address */ 99 void *kva; /* kernel virtual address */ 100 u32 size; 101 unsigned long attrs; 102 }; 103 104 struct iface_queue { 105 struct hfi_queue_header *qhdr; 106 struct mem_desc qmem; 107 }; 108 109 enum venus_state { 110 VENUS_STATE_DEINIT = 1, 111 VENUS_STATE_INIT, 112 }; 113 114 struct venus_hfi_device { 115 struct venus_core *core; 116 u32 irq_status; 117 u32 last_packet_type; 118 bool power_enabled; 119 bool suspended; 120 enum venus_state state; 121 /* serialize read / write to the shared memory */ 122 struct mutex lock; 123 struct completion pwr_collapse_prep; 124 struct completion release_resource; 125 struct mem_desc ifaceq_table; 126 struct mem_desc sfr; 127 struct iface_queue queues[IFACEQ_NUM]; 128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 130 }; 131 132 static bool venus_pkt_debug; 133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL; 134 static bool venus_fw_low_power_mode = true; 135 static int venus_hw_rsp_timeout = 1000; 136 static bool venus_fw_coverage; 137 138 static void venus_set_state(struct venus_hfi_device *hdev, 139 enum venus_state state) 140 { 141 mutex_lock(&hdev->lock); 142 hdev->state = state; 143 mutex_unlock(&hdev->lock); 144 } 145 146 static bool venus_is_valid_state(struct venus_hfi_device *hdev) 147 { 148 return hdev->state != VENUS_STATE_DEINIT; 149 } 150 151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet) 152 { 153 size_t pkt_size = *(u32 *)packet; 154 155 if (!venus_pkt_debug) 156 return; 157 158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet, 159 pkt_size, true); 160 } 161 162 static int venus_write_queue(struct venus_hfi_device *hdev, 163 struct iface_queue *queue, 164 void *packet, u32 *rx_req) 165 { 166 struct hfi_queue_header *qhdr; 167 u32 dwords, new_wr_idx; 168 u32 empty_space, rd_idx, wr_idx, qsize; 169 u32 *wr_ptr; 170 171 if (!queue->qmem.kva) 172 return -EINVAL; 173 174 qhdr = queue->qhdr; 175 if (!qhdr) 176 return -EINVAL; 177 178 venus_dump_packet(hdev, packet); 179 180 dwords = (*(u32 *)packet) >> 2; 181 if (!dwords) 182 return -EINVAL; 183 184 rd_idx = qhdr->read_idx; 185 wr_idx = qhdr->write_idx; 186 qsize = qhdr->q_size; 187 /* ensure rd/wr indices's are read from memory */ 188 rmb(); 189 190 if (qsize > IFACEQ_QUEUE_SIZE / 4) 191 return -EINVAL; 192 193 if (wr_idx >= rd_idx) 194 empty_space = qsize - (wr_idx - rd_idx); 195 else 196 empty_space = rd_idx - wr_idx; 197 198 if (empty_space <= dwords) { 199 qhdr->tx_req = 1; 200 /* ensure tx_req is updated in memory */ 201 wmb(); 202 return -ENOSPC; 203 } 204 205 qhdr->tx_req = 0; 206 /* ensure tx_req is updated in memory */ 207 wmb(); 208 209 new_wr_idx = wr_idx + dwords; 210 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2)); 211 212 if (wr_ptr < (u32 *)queue->qmem.kva || 213 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr))) 214 return -EINVAL; 215 216 if (new_wr_idx < qsize) { 217 memcpy(wr_ptr, packet, dwords << 2); 218 } else { 219 size_t len; 220 221 new_wr_idx -= qsize; 222 len = (dwords - new_wr_idx) << 2; 223 memcpy(wr_ptr, packet, len); 224 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2); 225 } 226 227 /* make sure packet is written before updating the write index */ 228 wmb(); 229 230 qhdr->write_idx = new_wr_idx; 231 *rx_req = qhdr->rx_req ? 1 : 0; 232 233 /* make sure write index is updated before an interrupt is raised */ 234 mb(); 235 236 return 0; 237 } 238 239 static int venus_read_queue(struct venus_hfi_device *hdev, 240 struct iface_queue *queue, void *pkt, u32 *tx_req) 241 { 242 struct hfi_queue_header *qhdr; 243 u32 dwords, new_rd_idx; 244 u32 rd_idx, wr_idx, type, qsize; 245 u32 *rd_ptr; 246 u32 recv_request = 0; 247 int ret = 0; 248 249 if (!queue->qmem.kva) 250 return -EINVAL; 251 252 qhdr = queue->qhdr; 253 if (!qhdr) 254 return -EINVAL; 255 256 type = qhdr->type; 257 rd_idx = qhdr->read_idx; 258 wr_idx = qhdr->write_idx; 259 qsize = qhdr->q_size; 260 261 if (qsize > IFACEQ_QUEUE_SIZE / 4) 262 return -EINVAL; 263 264 /* make sure data is valid before using it */ 265 rmb(); 266 267 /* 268 * Do not set receive request for debug queue, if set, Venus generates 269 * interrupt for debug messages even when there is no response message 270 * available. In general debug queue will not become full as it is being 271 * emptied out for every interrupt from Venus. Venus will anyway 272 * generates interrupt if it is full. 273 */ 274 if (type & HFI_CTRL_TO_HOST_MSG_Q) 275 recv_request = 1; 276 277 if (rd_idx == wr_idx) { 278 qhdr->rx_req = recv_request; 279 *tx_req = 0; 280 /* update rx_req field in memory */ 281 wmb(); 282 return -ENODATA; 283 } 284 285 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2)); 286 287 if (rd_ptr < (u32 *)queue->qmem.kva || 288 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr))) 289 return -EINVAL; 290 291 dwords = *rd_ptr >> 2; 292 if (!dwords) 293 return -EINVAL; 294 295 new_rd_idx = rd_idx + dwords; 296 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) { 297 if (new_rd_idx < qsize) { 298 memcpy(pkt, rd_ptr, dwords << 2); 299 } else { 300 size_t len; 301 302 new_rd_idx -= qsize; 303 len = (dwords - new_rd_idx) << 2; 304 memcpy(pkt, rd_ptr, len); 305 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2); 306 } 307 } else { 308 /* bad packet received, dropping */ 309 new_rd_idx = qhdr->write_idx; 310 ret = -EBADMSG; 311 } 312 313 /* ensure the packet is read before updating read index */ 314 rmb(); 315 316 qhdr->read_idx = new_rd_idx; 317 /* ensure updating read index */ 318 wmb(); 319 320 rd_idx = qhdr->read_idx; 321 wr_idx = qhdr->write_idx; 322 /* ensure rd/wr indices are read from memory */ 323 rmb(); 324 325 if (rd_idx != wr_idx) 326 qhdr->rx_req = 0; 327 else 328 qhdr->rx_req = recv_request; 329 330 *tx_req = qhdr->tx_req ? 1 : 0; 331 332 /* ensure rx_req is stored to memory and tx_req is loaded from memory */ 333 mb(); 334 335 venus_dump_packet(hdev, pkt); 336 337 return ret; 338 } 339 340 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc, 341 u32 size) 342 { 343 struct device *dev = hdev->core->dev; 344 345 desc->attrs = DMA_ATTR_WRITE_COMBINE; 346 desc->size = ALIGN(size, SZ_4K); 347 348 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL, 349 desc->attrs); 350 if (!desc->kva) 351 return -ENOMEM; 352 353 return 0; 354 } 355 356 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem) 357 { 358 struct device *dev = hdev->core->dev; 359 360 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs); 361 } 362 363 static void venus_set_registers(struct venus_hfi_device *hdev) 364 { 365 const struct venus_resources *res = hdev->core->res; 366 const struct reg_val *tbl = res->reg_tbl; 367 unsigned int count = res->reg_tbl_size; 368 unsigned int i; 369 370 for (i = 0; i < count; i++) 371 writel(tbl[i].value, hdev->core->base + tbl[i].reg); 372 } 373 374 static void venus_soft_int(struct venus_hfi_device *hdev) 375 { 376 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base; 377 u32 clear_bit; 378 379 if (IS_V6(hdev->core)) 380 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6); 381 else 382 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT); 383 384 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT); 385 } 386 387 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev, 388 void *pkt, bool sync) 389 { 390 struct device *dev = hdev->core->dev; 391 struct hfi_pkt_hdr *cmd_packet; 392 struct iface_queue *queue; 393 u32 rx_req; 394 int ret; 395 396 if (!venus_is_valid_state(hdev)) 397 return -EINVAL; 398 399 cmd_packet = (struct hfi_pkt_hdr *)pkt; 400 hdev->last_packet_type = cmd_packet->pkt_type; 401 402 queue = &hdev->queues[IFACEQ_CMD_IDX]; 403 404 ret = venus_write_queue(hdev, queue, pkt, &rx_req); 405 if (ret) { 406 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret); 407 return ret; 408 } 409 410 if (sync) { 411 /* 412 * Inform video hardware to raise interrupt for synchronous 413 * commands 414 */ 415 queue = &hdev->queues[IFACEQ_MSG_IDX]; 416 queue->qhdr->rx_req = 1; 417 /* ensure rx_req is updated in memory */ 418 wmb(); 419 } 420 421 if (rx_req) 422 venus_soft_int(hdev); 423 424 return 0; 425 } 426 427 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync) 428 { 429 int ret; 430 431 mutex_lock(&hdev->lock); 432 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync); 433 mutex_unlock(&hdev->lock); 434 435 return ret; 436 } 437 438 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id, 439 u32 size, u32 addr, void *cookie) 440 { 441 struct venus_hfi_device *hdev = to_hfi_priv(core); 442 struct hfi_sys_set_resource_pkt *pkt; 443 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 444 int ret; 445 446 if (id == VIDC_RESOURCE_NONE) 447 return 0; 448 449 pkt = (struct hfi_sys_set_resource_pkt *)packet; 450 451 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie); 452 if (ret) 453 return ret; 454 455 ret = venus_iface_cmdq_write(hdev, pkt, false); 456 if (ret) 457 return ret; 458 459 return 0; 460 } 461 462 static int venus_boot_core(struct venus_hfi_device *hdev) 463 { 464 struct device *dev = hdev->core->dev; 465 static const unsigned int max_tries = 100; 466 u32 ctrl_status = 0, mask_val = 0; 467 unsigned int count = 0; 468 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 469 void __iomem *wrapper_base = hdev->core->wrapper_base; 470 int ret = 0; 471 472 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 473 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK); 474 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 | 475 WRAPPER_INTR_MASK_A2HCPU_MASK); 476 } else { 477 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK; 478 } 479 480 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK); 481 if (IS_V1(hdev->core)) 482 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3); 483 484 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT); 485 while (!ctrl_status && count < max_tries) { 486 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 487 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) { 488 dev_err(dev, "invalid setting for UC_REGION\n"); 489 ret = -EINVAL; 490 break; 491 } 492 493 usleep_range(500, 1000); 494 count++; 495 } 496 497 if (count >= max_tries) 498 ret = -ETIMEDOUT; 499 500 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 501 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6); 502 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6); 503 } 504 505 return ret; 506 } 507 508 static u32 venus_hwversion(struct venus_hfi_device *hdev) 509 { 510 struct device *dev = hdev->core->dev; 511 void __iomem *wrapper_base = hdev->core->wrapper_base; 512 u32 ver; 513 u32 major, minor, step; 514 515 ver = readl(wrapper_base + WRAPPER_HW_VERSION); 516 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK; 517 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT; 518 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK; 519 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT; 520 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK; 521 522 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step); 523 524 return major; 525 } 526 527 static int venus_run(struct venus_hfi_device *hdev) 528 { 529 struct device *dev = hdev->core->dev; 530 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 531 int ret; 532 533 /* 534 * Re-program all of the registers that get reset as a result of 535 * regulator_disable() and _enable() 536 */ 537 venus_set_registers(hdev); 538 539 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR); 540 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE); 541 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2); 542 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1); 543 if (hdev->sfr.da) 544 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR); 545 546 ret = venus_boot_core(hdev); 547 if (ret) { 548 dev_err(dev, "failed to reset venus core\n"); 549 return ret; 550 } 551 552 venus_hwversion(hdev); 553 554 return 0; 555 } 556 557 static int venus_halt_axi(struct venus_hfi_device *hdev) 558 { 559 void __iomem *wrapper_base = hdev->core->wrapper_base; 560 void __iomem *vbif_base = hdev->core->vbif_base; 561 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 562 void __iomem *aon_base = hdev->core->aon_base; 563 struct device *dev = hdev->core->dev; 564 u32 val; 565 u32 mask_val; 566 int ret; 567 568 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 569 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6); 570 571 if (IS_IRIS2_1(hdev->core)) 572 goto skip_aon_mvp_noc; 573 574 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); 575 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS, 576 val, 577 val & BIT(0), 578 POLL_INTERVAL_US, 579 VBIF_AXI_HALT_ACK_TIMEOUT_US); 580 if (ret) 581 return -ETIMEDOUT; 582 583 skip_aon_mvp_noc: 584 mask_val = (BIT(2) | BIT(1) | BIT(0)); 585 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 586 587 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 588 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6, 589 val, 590 val == 0, 591 POLL_INTERVAL_US, 592 VBIF_AXI_HALT_ACK_TIMEOUT_US); 593 594 if (ret) { 595 dev_err(dev, "DBLP Release: lpi_status %x\n", val); 596 return -ETIMEDOUT; 597 } 598 return 0; 599 } 600 601 if (IS_V4(hdev->core)) { 602 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT); 603 val |= WRAPPER_CPU_AXI_HALT_HALT; 604 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT); 605 606 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS, 607 val, 608 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE, 609 POLL_INTERVAL_US, 610 VBIF_AXI_HALT_ACK_TIMEOUT_US); 611 if (ret) { 612 dev_err(dev, "AXI bus port halt timeout\n"); 613 return ret; 614 } 615 616 return 0; 617 } 618 619 /* Halt AXI and AXI IMEM VBIF Access */ 620 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0); 621 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ; 622 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0); 623 624 /* Request for AXI bus port halt */ 625 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val, 626 val & VBIF_AXI_HALT_CTRL1_HALT_ACK, 627 POLL_INTERVAL_US, 628 VBIF_AXI_HALT_ACK_TIMEOUT_US); 629 if (ret) { 630 dev_err(dev, "AXI bus port halt timeout\n"); 631 return ret; 632 } 633 634 return 0; 635 } 636 637 static int venus_power_off(struct venus_hfi_device *hdev) 638 { 639 int ret; 640 641 if (!hdev->power_enabled) 642 return 0; 643 644 ret = venus_set_hw_state_suspend(hdev->core); 645 if (ret) 646 return ret; 647 648 ret = venus_halt_axi(hdev); 649 if (ret) 650 return ret; 651 652 hdev->power_enabled = false; 653 654 return 0; 655 } 656 657 static int venus_power_on(struct venus_hfi_device *hdev) 658 { 659 int ret; 660 661 if (hdev->power_enabled) 662 return 0; 663 664 ret = venus_set_hw_state_resume(hdev->core); 665 if (ret) 666 goto err; 667 668 ret = venus_run(hdev); 669 if (ret) 670 goto err_suspend; 671 672 hdev->power_enabled = true; 673 674 return 0; 675 676 err_suspend: 677 venus_set_hw_state_suspend(hdev->core); 678 err: 679 hdev->power_enabled = false; 680 return ret; 681 } 682 683 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev, 684 void *pkt) 685 { 686 struct iface_queue *queue; 687 u32 tx_req; 688 int ret; 689 690 if (!venus_is_valid_state(hdev)) 691 return -EINVAL; 692 693 queue = &hdev->queues[IFACEQ_MSG_IDX]; 694 695 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 696 if (ret) 697 return ret; 698 699 if (tx_req) 700 venus_soft_int(hdev); 701 702 return 0; 703 } 704 705 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt) 706 { 707 int ret; 708 709 mutex_lock(&hdev->lock); 710 ret = venus_iface_msgq_read_nolock(hdev, pkt); 711 mutex_unlock(&hdev->lock); 712 713 return ret; 714 } 715 716 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev, 717 void *pkt) 718 { 719 struct iface_queue *queue; 720 u32 tx_req; 721 int ret; 722 723 ret = venus_is_valid_state(hdev); 724 if (!ret) 725 return -EINVAL; 726 727 queue = &hdev->queues[IFACEQ_DBG_IDX]; 728 729 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 730 if (ret) 731 return ret; 732 733 if (tx_req) 734 venus_soft_int(hdev); 735 736 return 0; 737 } 738 739 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt) 740 { 741 int ret; 742 743 if (!pkt) 744 return -EINVAL; 745 746 mutex_lock(&hdev->lock); 747 ret = venus_iface_dbgq_read_nolock(hdev, pkt); 748 mutex_unlock(&hdev->lock); 749 750 return ret; 751 } 752 753 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr) 754 { 755 qhdr->status = 1; 756 qhdr->type = IFACEQ_DFLT_QHDR; 757 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4; 758 qhdr->pkt_size = 0; 759 qhdr->rx_wm = 1; 760 qhdr->tx_wm = 1; 761 qhdr->rx_req = 1; 762 qhdr->tx_req = 0; 763 qhdr->rx_irq_status = 0; 764 qhdr->tx_irq_status = 0; 765 qhdr->read_idx = 0; 766 qhdr->write_idx = 0; 767 } 768 769 static void venus_interface_queues_release(struct venus_hfi_device *hdev) 770 { 771 mutex_lock(&hdev->lock); 772 773 venus_free(hdev, &hdev->ifaceq_table); 774 venus_free(hdev, &hdev->sfr); 775 776 memset(hdev->queues, 0, sizeof(hdev->queues)); 777 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table)); 778 memset(&hdev->sfr, 0, sizeof(hdev->sfr)); 779 780 mutex_unlock(&hdev->lock); 781 } 782 783 static int venus_interface_queues_init(struct venus_hfi_device *hdev) 784 { 785 struct hfi_queue_table_header *tbl_hdr; 786 struct iface_queue *queue; 787 struct hfi_sfr *sfr; 788 struct mem_desc desc = {0}; 789 unsigned int offset; 790 unsigned int i; 791 int ret; 792 793 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE); 794 if (ret) 795 return ret; 796 797 hdev->ifaceq_table = desc; 798 offset = IFACEQ_TABLE_SIZE; 799 800 for (i = 0; i < IFACEQ_NUM; i++) { 801 queue = &hdev->queues[i]; 802 queue->qmem.da = desc.da + offset; 803 queue->qmem.kva = desc.kva + offset; 804 queue->qmem.size = IFACEQ_QUEUE_SIZE; 805 offset += queue->qmem.size; 806 queue->qhdr = 807 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 808 809 venus_set_qhdr_defaults(queue->qhdr); 810 811 queue->qhdr->start_addr = queue->qmem.da; 812 813 if (i == IFACEQ_CMD_IDX) 814 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 815 else if (i == IFACEQ_MSG_IDX) 816 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 817 else if (i == IFACEQ_DBG_IDX) 818 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 819 } 820 821 tbl_hdr = hdev->ifaceq_table.kva; 822 tbl_hdr->version = 0; 823 tbl_hdr->size = IFACEQ_TABLE_SIZE; 824 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 825 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 826 tbl_hdr->num_q = IFACEQ_NUM; 827 tbl_hdr->num_active_q = IFACEQ_NUM; 828 829 /* 830 * Set receive request to zero on debug queue as there is no 831 * need of interrupt from video hardware for debug messages 832 */ 833 queue = &hdev->queues[IFACEQ_DBG_IDX]; 834 queue->qhdr->rx_req = 0; 835 836 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE); 837 if (ret) { 838 hdev->sfr.da = 0; 839 } else { 840 hdev->sfr = desc; 841 sfr = hdev->sfr.kva; 842 sfr->buf_size = ALIGNED_SFR_SIZE; 843 } 844 845 /* ensure table and queue header structs are settled in memory */ 846 wmb(); 847 848 return 0; 849 } 850 851 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug) 852 { 853 struct hfi_sys_set_property_pkt *pkt; 854 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 855 856 pkt = (struct hfi_sys_set_property_pkt *)packet; 857 858 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug); 859 860 return venus_iface_cmdq_write(hdev, pkt, false); 861 } 862 863 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode) 864 { 865 struct hfi_sys_set_property_pkt *pkt; 866 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 867 868 pkt = (struct hfi_sys_set_property_pkt *)packet; 869 870 pkt_sys_coverage_config(pkt, mode); 871 872 return venus_iface_cmdq_write(hdev, pkt, false); 873 } 874 875 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev, 876 bool enable) 877 { 878 struct hfi_sys_set_property_pkt *pkt; 879 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 880 881 if (!enable) 882 return 0; 883 884 pkt = (struct hfi_sys_set_property_pkt *)packet; 885 886 pkt_sys_idle_indicator(pkt, enable); 887 888 return venus_iface_cmdq_write(hdev, pkt, false); 889 } 890 891 static int venus_sys_set_power_control(struct venus_hfi_device *hdev, 892 bool enable) 893 { 894 struct hfi_sys_set_property_pkt *pkt; 895 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 896 897 pkt = (struct hfi_sys_set_property_pkt *)packet; 898 899 pkt_sys_power_control(pkt, enable); 900 901 return venus_iface_cmdq_write(hdev, pkt, false); 902 } 903 904 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev) 905 { 906 struct hfi_sys_set_property_pkt *pkt; 907 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 908 const struct venus_resources *res = hdev->core->res; 909 int ret; 910 911 pkt = (struct hfi_sys_set_property_pkt *)packet; 912 913 pkt_sys_ubwc_config(pkt, res->ubwc_conf); 914 915 ret = venus_iface_cmdq_write(hdev, pkt, false); 916 if (ret) 917 return ret; 918 919 return 0; 920 } 921 922 static int venus_get_queue_size(struct venus_hfi_device *hdev, 923 unsigned int index) 924 { 925 struct hfi_queue_header *qhdr; 926 927 if (index >= IFACEQ_NUM) 928 return -EINVAL; 929 930 qhdr = hdev->queues[index].qhdr; 931 if (!qhdr) 932 return -EINVAL; 933 934 return abs(qhdr->read_idx - qhdr->write_idx); 935 } 936 937 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev) 938 { 939 struct device *dev = hdev->core->dev; 940 const struct venus_resources *res = hdev->core->res; 941 int ret; 942 943 ret = venus_sys_set_debug(hdev, venus_fw_debug); 944 if (ret) 945 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret); 946 947 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */ 948 if (IS_V1(hdev->core)) { 949 ret = venus_sys_set_idle_message(hdev, false); 950 if (ret) 951 dev_warn(dev, "setting idle response ON failed (%d)\n", ret); 952 } 953 954 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode); 955 if (ret) 956 dev_warn(dev, "setting hw power collapse ON failed (%d)\n", 957 ret); 958 959 /* For specific venus core, it is mandatory to set the UBWC configuration */ 960 if (res->ubwc_conf) { 961 ret = venus_sys_set_ubwc_config(hdev); 962 if (ret) 963 dev_warn(dev, "setting ubwc config failed (%d)\n", ret); 964 } 965 966 return ret; 967 } 968 969 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync) 970 { 971 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 972 struct hfi_session_pkt pkt; 973 974 pkt_session_cmd(&pkt, pkt_type, inst); 975 976 return venus_iface_cmdq_write(hdev, &pkt, sync); 977 } 978 979 static void venus_flush_debug_queue(struct venus_hfi_device *hdev) 980 { 981 struct device *dev = hdev->core->dev; 982 void *packet = hdev->dbg_buf; 983 984 while (!venus_iface_dbgq_read(hdev, packet)) { 985 struct hfi_msg_sys_coverage_pkt *pkt = packet; 986 987 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) { 988 struct hfi_msg_sys_debug_pkt *pkt = packet; 989 990 dev_dbg(dev, VDBGFW "%s", pkt->msg_data); 991 } 992 } 993 } 994 995 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev, 996 bool wait) 997 { 998 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout); 999 struct hfi_sys_pc_prep_pkt pkt; 1000 int ret; 1001 1002 init_completion(&hdev->pwr_collapse_prep); 1003 1004 pkt_sys_pc_prep(&pkt); 1005 1006 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1007 if (ret) 1008 return ret; 1009 1010 if (!wait) 1011 return 0; 1012 1013 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout); 1014 if (!ret) { 1015 venus_flush_debug_queue(hdev); 1016 return -ETIMEDOUT; 1017 } 1018 1019 return 0; 1020 } 1021 1022 static int venus_are_queues_empty(struct venus_hfi_device *hdev) 1023 { 1024 int ret1, ret2; 1025 1026 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX); 1027 if (ret1 < 0) 1028 return ret1; 1029 1030 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX); 1031 if (ret2 < 0) 1032 return ret2; 1033 1034 if (!ret1 && !ret2) 1035 return 1; 1036 1037 return 0; 1038 } 1039 1040 static void venus_sfr_print(struct venus_hfi_device *hdev) 1041 { 1042 struct device *dev = hdev->core->dev; 1043 struct hfi_sfr *sfr = hdev->sfr.kva; 1044 u32 size; 1045 void *p; 1046 1047 if (!sfr) 1048 return; 1049 1050 size = sfr->buf_size; 1051 if (!size) 1052 return; 1053 1054 if (size > ALIGNED_SFR_SIZE) 1055 size = ALIGNED_SFR_SIZE; 1056 1057 p = memchr(sfr->data, '\0', size); 1058 /* 1059 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates 1060 * that Venus is in the process of crashing. 1061 */ 1062 if (!p) 1063 sfr->data[size - 1] = '\0'; 1064 1065 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data); 1066 } 1067 1068 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev, 1069 void *packet) 1070 { 1071 struct hfi_msg_event_notify_pkt *event_pkt = packet; 1072 1073 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR) 1074 return; 1075 1076 venus_set_state(hdev, VENUS_STATE_DEINIT); 1077 1078 venus_sfr_print(hdev); 1079 } 1080 1081 static irqreturn_t venus_isr_thread(struct venus_core *core) 1082 { 1083 struct venus_hfi_device *hdev = to_hfi_priv(core); 1084 const struct venus_resources *res; 1085 void *pkt; 1086 u32 msg_ret; 1087 1088 if (!hdev) 1089 return IRQ_NONE; 1090 1091 res = hdev->core->res; 1092 pkt = hdev->pkt_buf; 1093 1094 1095 while (!venus_iface_msgq_read(hdev, pkt)) { 1096 msg_ret = hfi_process_msg_packet(core, pkt); 1097 switch (msg_ret) { 1098 case HFI_MSG_EVENT_NOTIFY: 1099 venus_process_msg_sys_error(hdev, pkt); 1100 break; 1101 case HFI_MSG_SYS_INIT: 1102 venus_hfi_core_set_resource(core, res->vmem_id, 1103 res->vmem_size, 1104 res->vmem_addr, 1105 hdev); 1106 break; 1107 case HFI_MSG_SYS_RELEASE_RESOURCE: 1108 complete(&hdev->release_resource); 1109 break; 1110 case HFI_MSG_SYS_PC_PREP: 1111 complete(&hdev->pwr_collapse_prep); 1112 break; 1113 default: 1114 break; 1115 } 1116 } 1117 1118 venus_flush_debug_queue(hdev); 1119 1120 return IRQ_HANDLED; 1121 } 1122 1123 static irqreturn_t venus_isr(struct venus_core *core) 1124 { 1125 struct venus_hfi_device *hdev = to_hfi_priv(core); 1126 u32 status; 1127 void __iomem *cpu_cs_base; 1128 void __iomem *wrapper_base; 1129 1130 if (!hdev) 1131 return IRQ_NONE; 1132 1133 cpu_cs_base = hdev->core->cpu_cs_base; 1134 wrapper_base = hdev->core->wrapper_base; 1135 1136 status = readl(wrapper_base + WRAPPER_INTR_STATUS); 1137 if (IS_IRIS2(core) || IS_IRIS2_1(core)) { 1138 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1139 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 || 1140 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1141 hdev->irq_status = status; 1142 } else { 1143 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1144 status & WRAPPER_INTR_STATUS_A2HWD_MASK || 1145 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1146 hdev->irq_status = status; 1147 } 1148 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR); 1149 if (!(IS_IRIS2(core) || IS_IRIS2_1(core))) 1150 writel(status, wrapper_base + WRAPPER_INTR_CLEAR); 1151 1152 return IRQ_WAKE_THREAD; 1153 } 1154 1155 static int venus_core_init(struct venus_core *core) 1156 { 1157 struct venus_hfi_device *hdev = to_hfi_priv(core); 1158 struct device *dev = core->dev; 1159 struct hfi_sys_get_property_pkt version_pkt; 1160 struct hfi_sys_init_pkt pkt; 1161 int ret; 1162 1163 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX); 1164 1165 venus_set_state(hdev, VENUS_STATE_INIT); 1166 1167 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1168 if (ret) 1169 return ret; 1170 1171 pkt_sys_image_version(&version_pkt); 1172 1173 ret = venus_iface_cmdq_write(hdev, &version_pkt, false); 1174 if (ret) 1175 dev_warn(dev, "failed to send image version pkt to fw\n"); 1176 1177 ret = venus_sys_set_default_properties(hdev); 1178 if (ret) 1179 return ret; 1180 1181 return 0; 1182 } 1183 1184 static int venus_core_deinit(struct venus_core *core) 1185 { 1186 struct venus_hfi_device *hdev = to_hfi_priv(core); 1187 1188 venus_set_state(hdev, VENUS_STATE_DEINIT); 1189 hdev->suspended = true; 1190 hdev->power_enabled = false; 1191 1192 return 0; 1193 } 1194 1195 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type) 1196 { 1197 struct venus_hfi_device *hdev = to_hfi_priv(core); 1198 struct hfi_sys_test_ssr_pkt pkt; 1199 int ret; 1200 1201 ret = pkt_sys_ssr_cmd(&pkt, trigger_type); 1202 if (ret) 1203 return ret; 1204 1205 return venus_iface_cmdq_write(hdev, &pkt, false); 1206 } 1207 1208 static int venus_session_init(struct venus_inst *inst, u32 session_type, 1209 u32 codec) 1210 { 1211 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1212 struct hfi_session_init_pkt pkt; 1213 int ret; 1214 1215 ret = venus_sys_set_debug(hdev, venus_fw_debug); 1216 if (ret) 1217 goto err; 1218 1219 ret = pkt_session_init(&pkt, inst, session_type, codec); 1220 if (ret) 1221 goto err; 1222 1223 ret = venus_iface_cmdq_write(hdev, &pkt, true); 1224 if (ret) 1225 goto err; 1226 1227 return 0; 1228 1229 err: 1230 venus_flush_debug_queue(hdev); 1231 return ret; 1232 } 1233 1234 static int venus_session_end(struct venus_inst *inst) 1235 { 1236 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1237 struct device *dev = hdev->core->dev; 1238 1239 if (venus_fw_coverage) { 1240 if (venus_sys_set_coverage(hdev, venus_fw_coverage)) 1241 dev_warn(dev, "fw coverage msg ON failed\n"); 1242 } 1243 1244 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true); 1245 } 1246 1247 static int venus_session_abort(struct venus_inst *inst) 1248 { 1249 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1250 1251 venus_flush_debug_queue(hdev); 1252 1253 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true); 1254 } 1255 1256 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode) 1257 { 1258 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1259 struct hfi_session_flush_pkt pkt; 1260 int ret; 1261 1262 ret = pkt_session_flush(&pkt, inst, flush_mode); 1263 if (ret) 1264 return ret; 1265 1266 return venus_iface_cmdq_write(hdev, &pkt, true); 1267 } 1268 1269 static int venus_session_start(struct venus_inst *inst) 1270 { 1271 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true); 1272 } 1273 1274 static int venus_session_stop(struct venus_inst *inst) 1275 { 1276 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true); 1277 } 1278 1279 static int venus_session_continue(struct venus_inst *inst) 1280 { 1281 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false); 1282 } 1283 1284 static int venus_session_etb(struct venus_inst *inst, 1285 struct hfi_frame_data *in_frame) 1286 { 1287 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1288 u32 session_type = inst->session_type; 1289 int ret; 1290 1291 if (session_type == VIDC_SESSION_TYPE_DEC) { 1292 struct hfi_session_empty_buffer_compressed_pkt pkt; 1293 1294 ret = pkt_session_etb_decoder(&pkt, inst, in_frame); 1295 if (ret) 1296 return ret; 1297 1298 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1299 } else if (session_type == VIDC_SESSION_TYPE_ENC) { 1300 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt; 1301 1302 ret = pkt_session_etb_encoder(&pkt, inst, in_frame); 1303 if (ret) 1304 return ret; 1305 1306 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1307 } else { 1308 ret = -EINVAL; 1309 } 1310 1311 return ret; 1312 } 1313 1314 static int venus_session_ftb(struct venus_inst *inst, 1315 struct hfi_frame_data *out_frame) 1316 { 1317 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1318 struct hfi_session_fill_buffer_pkt pkt; 1319 int ret; 1320 1321 ret = pkt_session_ftb(&pkt, inst, out_frame); 1322 if (ret) 1323 return ret; 1324 1325 return venus_iface_cmdq_write(hdev, &pkt, false); 1326 } 1327 1328 static int venus_session_set_buffers(struct venus_inst *inst, 1329 struct hfi_buffer_desc *bd) 1330 { 1331 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1332 struct hfi_session_set_buffers_pkt *pkt; 1333 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1334 int ret; 1335 1336 if (bd->buffer_type == HFI_BUFFER_INPUT) 1337 return 0; 1338 1339 pkt = (struct hfi_session_set_buffers_pkt *)packet; 1340 1341 ret = pkt_session_set_buffers(pkt, inst, bd); 1342 if (ret) 1343 return ret; 1344 1345 return venus_iface_cmdq_write(hdev, pkt, false); 1346 } 1347 1348 static int venus_session_unset_buffers(struct venus_inst *inst, 1349 struct hfi_buffer_desc *bd) 1350 { 1351 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1352 struct hfi_session_release_buffer_pkt *pkt; 1353 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1354 int ret; 1355 1356 if (bd->buffer_type == HFI_BUFFER_INPUT) 1357 return 0; 1358 1359 pkt = (struct hfi_session_release_buffer_pkt *)packet; 1360 1361 ret = pkt_session_unset_buffers(pkt, inst, bd); 1362 if (ret) 1363 return ret; 1364 1365 return venus_iface_cmdq_write(hdev, pkt, true); 1366 } 1367 1368 static int venus_session_load_res(struct venus_inst *inst) 1369 { 1370 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true); 1371 } 1372 1373 static int venus_session_release_res(struct venus_inst *inst) 1374 { 1375 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true); 1376 } 1377 1378 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1379 u32 seq_hdr_len) 1380 { 1381 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1382 struct hfi_session_parse_sequence_header_pkt *pkt; 1383 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1384 int ret; 1385 1386 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet; 1387 1388 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len); 1389 if (ret) 1390 return ret; 1391 1392 ret = venus_iface_cmdq_write(hdev, pkt, false); 1393 if (ret) 1394 return ret; 1395 1396 return 0; 1397 } 1398 1399 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1400 u32 seq_hdr_len) 1401 { 1402 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1403 struct hfi_session_get_sequence_header_pkt *pkt; 1404 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1405 int ret; 1406 1407 pkt = (struct hfi_session_get_sequence_header_pkt *)packet; 1408 1409 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len); 1410 if (ret) 1411 return ret; 1412 1413 return venus_iface_cmdq_write(hdev, pkt, false); 1414 } 1415 1416 static int venus_session_set_property(struct venus_inst *inst, u32 ptype, 1417 void *pdata) 1418 { 1419 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1420 struct hfi_session_set_property_pkt *pkt; 1421 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1422 int ret; 1423 1424 pkt = (struct hfi_session_set_property_pkt *)packet; 1425 1426 ret = pkt_session_set_property(pkt, inst, ptype, pdata); 1427 if (ret == -ENOTSUPP) 1428 return 0; 1429 if (ret) 1430 return ret; 1431 1432 return venus_iface_cmdq_write(hdev, pkt, false); 1433 } 1434 1435 static int venus_session_get_property(struct venus_inst *inst, u32 ptype) 1436 { 1437 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1438 struct hfi_session_get_property_pkt pkt; 1439 int ret; 1440 1441 ret = pkt_session_get_property(&pkt, inst, ptype); 1442 if (ret) 1443 return ret; 1444 1445 return venus_iface_cmdq_write(hdev, &pkt, true); 1446 } 1447 1448 static int venus_resume(struct venus_core *core) 1449 { 1450 struct venus_hfi_device *hdev = to_hfi_priv(core); 1451 int ret = 0; 1452 1453 mutex_lock(&hdev->lock); 1454 1455 if (!hdev->suspended) 1456 goto unlock; 1457 1458 ret = venus_power_on(hdev); 1459 1460 unlock: 1461 if (!ret) 1462 hdev->suspended = false; 1463 1464 mutex_unlock(&hdev->lock); 1465 1466 return ret; 1467 } 1468 1469 static int venus_suspend_1xx(struct venus_core *core) 1470 { 1471 struct venus_hfi_device *hdev = to_hfi_priv(core); 1472 struct device *dev = core->dev; 1473 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1474 u32 ctrl_status; 1475 int ret; 1476 1477 if (!hdev->power_enabled || hdev->suspended) 1478 return 0; 1479 1480 mutex_lock(&hdev->lock); 1481 ret = venus_is_valid_state(hdev); 1482 mutex_unlock(&hdev->lock); 1483 1484 if (!ret) { 1485 dev_err(dev, "bad state, cannot suspend\n"); 1486 return -EINVAL; 1487 } 1488 1489 ret = venus_prepare_power_collapse(hdev, true); 1490 if (ret) { 1491 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1492 return ret; 1493 } 1494 1495 mutex_lock(&hdev->lock); 1496 1497 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) { 1498 mutex_unlock(&hdev->lock); 1499 return -EINVAL; 1500 } 1501 1502 ret = venus_are_queues_empty(hdev); 1503 if (ret < 0 || !ret) { 1504 mutex_unlock(&hdev->lock); 1505 return -EINVAL; 1506 } 1507 1508 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1509 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) { 1510 mutex_unlock(&hdev->lock); 1511 return -EINVAL; 1512 } 1513 1514 ret = venus_power_off(hdev); 1515 if (ret) { 1516 mutex_unlock(&hdev->lock); 1517 return ret; 1518 } 1519 1520 hdev->suspended = true; 1521 1522 mutex_unlock(&hdev->lock); 1523 1524 return 0; 1525 } 1526 1527 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev) 1528 { 1529 void __iomem *wrapper_base = hdev->core->wrapper_base; 1530 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1531 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1532 u32 ctrl_status, cpu_status; 1533 1534 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1535 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1536 else 1537 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1538 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1539 1540 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1541 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1542 return true; 1543 1544 return false; 1545 } 1546 1547 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev) 1548 { 1549 void __iomem *wrapper_base = hdev->core->wrapper_base; 1550 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1551 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1552 u32 ctrl_status, cpu_status; 1553 1554 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1555 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1556 else 1557 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1558 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1559 1560 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1561 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1562 return true; 1563 1564 return false; 1565 } 1566 1567 static int venus_suspend_3xx(struct venus_core *core) 1568 { 1569 struct venus_hfi_device *hdev = to_hfi_priv(core); 1570 struct device *dev = core->dev; 1571 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1572 u32 ctrl_status; 1573 bool val; 1574 int ret; 1575 1576 if (!hdev->power_enabled || hdev->suspended) 1577 return 0; 1578 1579 mutex_lock(&hdev->lock); 1580 ret = venus_is_valid_state(hdev); 1581 mutex_unlock(&hdev->lock); 1582 1583 if (!ret) { 1584 dev_err(dev, "bad state, cannot suspend\n"); 1585 return -EINVAL; 1586 } 1587 1588 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1589 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1590 goto power_off; 1591 1592 /* 1593 * Power collapse sequence for Venus 3xx and 4xx versions: 1594 * 1. Check for ARM9 and video core to be idle by checking WFI bit 1595 * (bit 0) in CPU status register and by checking Idle (bit 30) in 1596 * Control status register for video core. 1597 * 2. Send a command to prepare for power collapse. 1598 * 3. Check for WFI and PC_READY bits. 1599 */ 1600 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val, 1601 1500, 100 * 1500); 1602 if (ret) { 1603 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret); 1604 return ret; 1605 } 1606 1607 ret = venus_prepare_power_collapse(hdev, false); 1608 if (ret) { 1609 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1610 return ret; 1611 } 1612 1613 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val, 1614 1500, 100 * 1500); 1615 if (ret) 1616 return ret; 1617 1618 power_off: 1619 mutex_lock(&hdev->lock); 1620 1621 ret = venus_power_off(hdev); 1622 if (ret) { 1623 dev_err(dev, "venus_power_off (%d)\n", ret); 1624 mutex_unlock(&hdev->lock); 1625 return ret; 1626 } 1627 1628 hdev->suspended = true; 1629 1630 mutex_unlock(&hdev->lock); 1631 1632 return 0; 1633 } 1634 1635 static int venus_suspend(struct venus_core *core) 1636 { 1637 if (IS_V3(core) || IS_V4(core) || IS_V6(core)) 1638 return venus_suspend_3xx(core); 1639 1640 return venus_suspend_1xx(core); 1641 } 1642 1643 static const struct hfi_ops venus_hfi_ops = { 1644 .core_init = venus_core_init, 1645 .core_deinit = venus_core_deinit, 1646 .core_trigger_ssr = venus_core_trigger_ssr, 1647 1648 .session_init = venus_session_init, 1649 .session_end = venus_session_end, 1650 .session_abort = venus_session_abort, 1651 .session_flush = venus_session_flush, 1652 .session_start = venus_session_start, 1653 .session_stop = venus_session_stop, 1654 .session_continue = venus_session_continue, 1655 .session_etb = venus_session_etb, 1656 .session_ftb = venus_session_ftb, 1657 .session_set_buffers = venus_session_set_buffers, 1658 .session_unset_buffers = venus_session_unset_buffers, 1659 .session_load_res = venus_session_load_res, 1660 .session_release_res = venus_session_release_res, 1661 .session_parse_seq_hdr = venus_session_parse_seq_hdr, 1662 .session_get_seq_hdr = venus_session_get_seq_hdr, 1663 .session_set_property = venus_session_set_property, 1664 .session_get_property = venus_session_get_property, 1665 1666 .resume = venus_resume, 1667 .suspend = venus_suspend, 1668 1669 .isr = venus_isr, 1670 .isr_thread = venus_isr_thread, 1671 }; 1672 1673 void venus_hfi_destroy(struct venus_core *core) 1674 { 1675 struct venus_hfi_device *hdev = to_hfi_priv(core); 1676 1677 core->priv = NULL; 1678 venus_interface_queues_release(hdev); 1679 mutex_destroy(&hdev->lock); 1680 kfree(hdev); 1681 core->ops = NULL; 1682 } 1683 1684 int venus_hfi_create(struct venus_core *core) 1685 { 1686 struct venus_hfi_device *hdev; 1687 int ret; 1688 1689 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL); 1690 if (!hdev) 1691 return -ENOMEM; 1692 1693 mutex_init(&hdev->lock); 1694 1695 hdev->core = core; 1696 hdev->suspended = true; 1697 core->priv = hdev; 1698 core->ops = &venus_hfi_ops; 1699 1700 ret = venus_interface_queues_init(hdev); 1701 if (ret) 1702 goto err_kfree; 1703 1704 return 0; 1705 1706 err_kfree: 1707 kfree(hdev); 1708 core->priv = NULL; 1709 core->ops = NULL; 1710 return ret; 1711 } 1712 1713 void venus_hfi_queues_reinit(struct venus_core *core) 1714 { 1715 struct venus_hfi_device *hdev = to_hfi_priv(core); 1716 struct hfi_queue_table_header *tbl_hdr; 1717 struct iface_queue *queue; 1718 struct hfi_sfr *sfr; 1719 unsigned int i; 1720 1721 mutex_lock(&hdev->lock); 1722 1723 for (i = 0; i < IFACEQ_NUM; i++) { 1724 queue = &hdev->queues[i]; 1725 queue->qhdr = 1726 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 1727 1728 venus_set_qhdr_defaults(queue->qhdr); 1729 1730 queue->qhdr->start_addr = queue->qmem.da; 1731 1732 if (i == IFACEQ_CMD_IDX) 1733 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 1734 else if (i == IFACEQ_MSG_IDX) 1735 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 1736 else if (i == IFACEQ_DBG_IDX) 1737 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 1738 } 1739 1740 tbl_hdr = hdev->ifaceq_table.kva; 1741 tbl_hdr->version = 0; 1742 tbl_hdr->size = IFACEQ_TABLE_SIZE; 1743 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 1744 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 1745 tbl_hdr->num_q = IFACEQ_NUM; 1746 tbl_hdr->num_active_q = IFACEQ_NUM; 1747 1748 /* 1749 * Set receive request to zero on debug queue as there is no 1750 * need of interrupt from video hardware for debug messages 1751 */ 1752 queue = &hdev->queues[IFACEQ_DBG_IDX]; 1753 queue->qhdr->rx_req = 0; 1754 1755 sfr = hdev->sfr.kva; 1756 sfr->buf_size = ALIGNED_SFR_SIZE; 1757 1758 /* ensure table and queue header structs are settled in memory */ 1759 wmb(); 1760 1761 mutex_unlock(&hdev->lock); 1762 } 1763