1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2017 Linaro Ltd. 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "hfi_cmds.h" 17 #include "hfi_msgs.h" 18 #include "hfi_venus.h" 19 #include "hfi_venus_io.h" 20 #include "firmware.h" 21 22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000 23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000 24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00 25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff 26 27 #define HFI_HOST_TO_CTRL_CMD_Q 0 28 #define HFI_CTRL_TO_HOST_MSG_Q 1 29 #define HFI_CTRL_TO_HOST_DBG_Q 2 30 #define HFI_MASK_QHDR_STATUS 0x000000ff 31 32 #define IFACEQ_NUM 3 33 #define IFACEQ_CMD_IDX 0 34 #define IFACEQ_MSG_IDX 1 35 #define IFACEQ_DBG_IDX 2 36 #define IFACEQ_MAX_BUF_COUNT 50 37 #define IFACEQ_MAX_PARALLEL_CLNTS 16 38 #define IFACEQ_DFLT_QHDR 0x01010000 39 40 #define POLL_INTERVAL_US 50 41 42 #define IFACEQ_MAX_PKT_SIZE 1024 43 #define IFACEQ_MED_PKT_SIZE 768 44 #define IFACEQ_MIN_PKT_SIZE 8 45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100 46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512 47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12) 48 49 struct hfi_queue_table_header { 50 u32 version; 51 u32 size; 52 u32 qhdr0_offset; 53 u32 qhdr_size; 54 u32 num_q; 55 u32 num_active_q; 56 }; 57 58 struct hfi_queue_header { 59 u32 status; 60 u32 start_addr; 61 u32 type; 62 u32 q_size; 63 u32 pkt_size; 64 u32 pkt_drop_cnt; 65 u32 rx_wm; 66 u32 tx_wm; 67 u32 rx_req; 68 u32 tx_req; 69 u32 rx_irq_status; 70 u32 tx_irq_status; 71 u32 read_idx; 72 u32 write_idx; 73 }; 74 75 #define IFACEQ_TABLE_SIZE \ 76 (sizeof(struct hfi_queue_table_header) + \ 77 sizeof(struct hfi_queue_header) * IFACEQ_NUM) 78 79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \ 80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS) 81 82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \ 83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \ 84 ((i) * sizeof(struct hfi_queue_header))) 85 86 #define QDSS_SIZE SZ_4K 87 #define SFR_SIZE SZ_4K 88 #define QUEUE_SIZE \ 89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM)) 90 91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K) 92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K) 93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K) 94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \ 95 ALIGNED_QDSS_SIZE, SZ_1M) 96 97 struct mem_desc { 98 dma_addr_t da; /* device address */ 99 void *kva; /* kernel virtual address */ 100 u32 size; 101 unsigned long attrs; 102 }; 103 104 struct iface_queue { 105 struct hfi_queue_header *qhdr; 106 struct mem_desc qmem; 107 }; 108 109 enum venus_state { 110 VENUS_STATE_DEINIT = 1, 111 VENUS_STATE_INIT, 112 }; 113 114 struct venus_hfi_device { 115 struct venus_core *core; 116 u32 irq_status; 117 u32 last_packet_type; 118 bool power_enabled; 119 bool suspended; 120 enum venus_state state; 121 /* serialize read / write to the shared memory */ 122 struct mutex lock; 123 struct completion pwr_collapse_prep; 124 struct completion release_resource; 125 struct mem_desc ifaceq_table; 126 struct mem_desc sfr; 127 struct iface_queue queues[IFACEQ_NUM]; 128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 130 }; 131 132 static bool venus_pkt_debug; 133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL; 134 static bool venus_fw_low_power_mode = true; 135 static int venus_hw_rsp_timeout = 1000; 136 static bool venus_fw_coverage; 137 138 static void venus_set_state(struct venus_hfi_device *hdev, 139 enum venus_state state) 140 { 141 mutex_lock(&hdev->lock); 142 hdev->state = state; 143 mutex_unlock(&hdev->lock); 144 } 145 146 static bool venus_is_valid_state(struct venus_hfi_device *hdev) 147 { 148 return hdev->state != VENUS_STATE_DEINIT; 149 } 150 151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet) 152 { 153 size_t pkt_size = *(u32 *)packet; 154 155 if (!venus_pkt_debug) 156 return; 157 158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet, 159 pkt_size, true); 160 } 161 162 static int venus_write_queue(struct venus_hfi_device *hdev, 163 struct iface_queue *queue, 164 void *packet, u32 *rx_req) 165 { 166 struct hfi_queue_header *qhdr; 167 u32 dwords, new_wr_idx; 168 u32 empty_space, rd_idx, wr_idx, qsize; 169 u32 *wr_ptr; 170 171 if (!queue->qmem.kva) 172 return -EINVAL; 173 174 qhdr = queue->qhdr; 175 if (!qhdr) 176 return -EINVAL; 177 178 venus_dump_packet(hdev, packet); 179 180 dwords = (*(u32 *)packet) >> 2; 181 if (!dwords) 182 return -EINVAL; 183 184 rd_idx = qhdr->read_idx; 185 wr_idx = qhdr->write_idx; 186 qsize = qhdr->q_size; 187 /* ensure rd/wr indices's are read from memory */ 188 rmb(); 189 190 if (qsize > IFACEQ_QUEUE_SIZE / 4) 191 return -EINVAL; 192 193 if (wr_idx >= rd_idx) 194 empty_space = qsize - (wr_idx - rd_idx); 195 else 196 empty_space = rd_idx - wr_idx; 197 198 if (empty_space <= dwords) { 199 qhdr->tx_req = 1; 200 /* ensure tx_req is updated in memory */ 201 wmb(); 202 return -ENOSPC; 203 } 204 205 qhdr->tx_req = 0; 206 /* ensure tx_req is updated in memory */ 207 wmb(); 208 209 new_wr_idx = wr_idx + dwords; 210 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2)); 211 212 if (wr_ptr < (u32 *)queue->qmem.kva || 213 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr))) 214 return -EINVAL; 215 216 if (new_wr_idx < qsize) { 217 memcpy(wr_ptr, packet, dwords << 2); 218 } else { 219 size_t len; 220 221 new_wr_idx -= qsize; 222 len = (dwords - new_wr_idx) << 2; 223 memcpy(wr_ptr, packet, len); 224 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2); 225 } 226 227 /* make sure packet is written before updating the write index */ 228 wmb(); 229 230 qhdr->write_idx = new_wr_idx; 231 *rx_req = qhdr->rx_req ? 1 : 0; 232 233 /* make sure write index is updated before an interrupt is raised */ 234 mb(); 235 236 return 0; 237 } 238 239 static int venus_read_queue(struct venus_hfi_device *hdev, 240 struct iface_queue *queue, void *pkt, u32 *tx_req) 241 { 242 struct hfi_pkt_hdr *pkt_hdr = NULL; 243 struct hfi_queue_header *qhdr; 244 u32 dwords, new_rd_idx; 245 u32 rd_idx, wr_idx, type, qsize; 246 u32 *rd_ptr; 247 u32 recv_request = 0; 248 int ret = 0; 249 250 if (!queue->qmem.kva) 251 return -EINVAL; 252 253 qhdr = queue->qhdr; 254 if (!qhdr) 255 return -EINVAL; 256 257 type = qhdr->type; 258 rd_idx = qhdr->read_idx; 259 wr_idx = qhdr->write_idx; 260 qsize = qhdr->q_size; 261 262 if (qsize > IFACEQ_QUEUE_SIZE / 4) 263 return -EINVAL; 264 265 /* make sure data is valid before using it */ 266 rmb(); 267 268 /* 269 * Do not set receive request for debug queue, if set, Venus generates 270 * interrupt for debug messages even when there is no response message 271 * available. In general debug queue will not become full as it is being 272 * emptied out for every interrupt from Venus. Venus will anyway 273 * generates interrupt if it is full. 274 */ 275 if (type & HFI_CTRL_TO_HOST_MSG_Q) 276 recv_request = 1; 277 278 if (rd_idx == wr_idx) { 279 qhdr->rx_req = recv_request; 280 *tx_req = 0; 281 /* update rx_req field in memory */ 282 wmb(); 283 return -ENODATA; 284 } 285 286 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2)); 287 288 if (rd_ptr < (u32 *)queue->qmem.kva || 289 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr))) 290 return -EINVAL; 291 292 dwords = *rd_ptr >> 2; 293 if (!dwords) 294 return -EINVAL; 295 296 new_rd_idx = rd_idx + dwords; 297 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) { 298 if (new_rd_idx < qsize) { 299 memcpy(pkt, rd_ptr, dwords << 2); 300 } else { 301 size_t len; 302 303 new_rd_idx -= qsize; 304 len = (dwords - new_rd_idx) << 2; 305 memcpy(pkt, rd_ptr, len); 306 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2); 307 } 308 pkt_hdr = (struct hfi_pkt_hdr *)(pkt); 309 if ((pkt_hdr->size >> 2) != dwords) 310 return -EINVAL; 311 } else { 312 /* bad packet received, dropping */ 313 new_rd_idx = qhdr->write_idx; 314 ret = -EBADMSG; 315 } 316 317 /* ensure the packet is read before updating read index */ 318 rmb(); 319 320 qhdr->read_idx = new_rd_idx; 321 /* ensure updating read index */ 322 wmb(); 323 324 rd_idx = qhdr->read_idx; 325 wr_idx = qhdr->write_idx; 326 /* ensure rd/wr indices are read from memory */ 327 rmb(); 328 329 if (rd_idx != wr_idx) 330 qhdr->rx_req = 0; 331 else 332 qhdr->rx_req = recv_request; 333 334 *tx_req = qhdr->tx_req ? 1 : 0; 335 336 /* ensure rx_req is stored to memory and tx_req is loaded from memory */ 337 mb(); 338 339 venus_dump_packet(hdev, pkt); 340 341 return ret; 342 } 343 344 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc, 345 u32 size) 346 { 347 struct device *dev = hdev->core->dev; 348 349 desc->attrs = DMA_ATTR_WRITE_COMBINE; 350 desc->size = ALIGN(size, SZ_4K); 351 352 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL, 353 desc->attrs); 354 if (!desc->kva) 355 return -ENOMEM; 356 357 return 0; 358 } 359 360 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem) 361 { 362 struct device *dev = hdev->core->dev; 363 364 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs); 365 } 366 367 static void venus_set_registers(struct venus_hfi_device *hdev) 368 { 369 const struct venus_resources *res = hdev->core->res; 370 const struct reg_val *tbl = res->reg_tbl; 371 unsigned int count = res->reg_tbl_size; 372 unsigned int i; 373 374 for (i = 0; i < count; i++) 375 writel(tbl[i].value, hdev->core->base + tbl[i].reg); 376 } 377 378 static void venus_soft_int(struct venus_hfi_device *hdev) 379 { 380 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base; 381 u32 clear_bit; 382 383 if (IS_V6(hdev->core)) 384 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6); 385 else 386 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT); 387 388 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT); 389 } 390 391 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev, 392 void *pkt, bool sync) 393 { 394 struct device *dev = hdev->core->dev; 395 struct hfi_pkt_hdr *cmd_packet; 396 struct iface_queue *queue; 397 u32 rx_req; 398 int ret; 399 400 if (!venus_is_valid_state(hdev)) 401 return -EINVAL; 402 403 cmd_packet = (struct hfi_pkt_hdr *)pkt; 404 hdev->last_packet_type = cmd_packet->pkt_type; 405 406 queue = &hdev->queues[IFACEQ_CMD_IDX]; 407 408 ret = venus_write_queue(hdev, queue, pkt, &rx_req); 409 if (ret) { 410 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret); 411 return ret; 412 } 413 414 if (sync) { 415 /* 416 * Inform video hardware to raise interrupt for synchronous 417 * commands 418 */ 419 queue = &hdev->queues[IFACEQ_MSG_IDX]; 420 queue->qhdr->rx_req = 1; 421 /* ensure rx_req is updated in memory */ 422 wmb(); 423 } 424 425 if (rx_req) 426 venus_soft_int(hdev); 427 428 return 0; 429 } 430 431 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync) 432 { 433 int ret; 434 435 mutex_lock(&hdev->lock); 436 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync); 437 mutex_unlock(&hdev->lock); 438 439 return ret; 440 } 441 442 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id, 443 u32 size, u32 addr, void *cookie) 444 { 445 struct venus_hfi_device *hdev = to_hfi_priv(core); 446 struct hfi_sys_set_resource_pkt *pkt; 447 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 448 int ret; 449 450 if (id == VIDC_RESOURCE_NONE) 451 return 0; 452 453 pkt = (struct hfi_sys_set_resource_pkt *)packet; 454 455 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie); 456 if (ret) 457 return ret; 458 459 ret = venus_iface_cmdq_write(hdev, pkt, false); 460 if (ret) 461 return ret; 462 463 return 0; 464 } 465 466 static int venus_boot_core(struct venus_hfi_device *hdev) 467 { 468 struct device *dev = hdev->core->dev; 469 static const unsigned int max_tries = 100; 470 u32 ctrl_status = 0, mask_val = 0; 471 unsigned int count = 0; 472 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 473 void __iomem *wrapper_base = hdev->core->wrapper_base; 474 int ret = 0; 475 476 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 477 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK); 478 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 | 479 WRAPPER_INTR_MASK_A2HCPU_MASK); 480 } else { 481 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK; 482 } 483 484 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK); 485 if (IS_V1(hdev->core)) 486 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3); 487 488 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT); 489 while (!ctrl_status && count < max_tries) { 490 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 491 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) { 492 dev_err(dev, "invalid setting for UC_REGION\n"); 493 ret = -EINVAL; 494 break; 495 } 496 497 usleep_range(500, 1000); 498 count++; 499 } 500 501 if (count >= max_tries) 502 ret = -ETIMEDOUT; 503 504 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 505 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6); 506 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6); 507 } 508 509 return ret; 510 } 511 512 static u32 venus_hwversion(struct venus_hfi_device *hdev) 513 { 514 struct device *dev = hdev->core->dev; 515 void __iomem *wrapper_base = hdev->core->wrapper_base; 516 u32 ver; 517 u32 major, minor, step; 518 519 ver = readl(wrapper_base + WRAPPER_HW_VERSION); 520 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK; 521 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT; 522 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK; 523 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT; 524 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK; 525 526 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step); 527 528 return major; 529 } 530 531 static int venus_run(struct venus_hfi_device *hdev) 532 { 533 struct device *dev = hdev->core->dev; 534 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 535 int ret; 536 537 /* 538 * Re-program all of the registers that get reset as a result of 539 * regulator_disable() and _enable() 540 */ 541 venus_set_registers(hdev); 542 543 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR); 544 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE); 545 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2); 546 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1); 547 if (hdev->sfr.da) 548 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR); 549 550 ret = venus_boot_core(hdev); 551 if (ret) { 552 dev_err(dev, "failed to reset venus core\n"); 553 return ret; 554 } 555 556 venus_hwversion(hdev); 557 558 return 0; 559 } 560 561 static int venus_halt_axi(struct venus_hfi_device *hdev) 562 { 563 void __iomem *wrapper_base = hdev->core->wrapper_base; 564 void __iomem *vbif_base = hdev->core->vbif_base; 565 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 566 void __iomem *aon_base = hdev->core->aon_base; 567 struct device *dev = hdev->core->dev; 568 u32 val; 569 u32 mask_val; 570 int ret; 571 572 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 573 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6); 574 575 if (IS_IRIS2_1(hdev->core)) 576 goto skip_aon_mvp_noc; 577 578 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); 579 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS, 580 val, 581 val & BIT(0), 582 POLL_INTERVAL_US, 583 VBIF_AXI_HALT_ACK_TIMEOUT_US); 584 if (ret) 585 return -ETIMEDOUT; 586 587 skip_aon_mvp_noc: 588 mask_val = (BIT(2) | BIT(1) | BIT(0)); 589 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 590 591 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 592 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6, 593 val, 594 val == 0, 595 POLL_INTERVAL_US, 596 VBIF_AXI_HALT_ACK_TIMEOUT_US); 597 598 if (ret) { 599 dev_err(dev, "DBLP Release: lpi_status %x\n", val); 600 return -ETIMEDOUT; 601 } 602 return 0; 603 } 604 605 if (IS_V4(hdev->core)) { 606 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT); 607 val |= WRAPPER_CPU_AXI_HALT_HALT; 608 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT); 609 610 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS, 611 val, 612 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE, 613 POLL_INTERVAL_US, 614 VBIF_AXI_HALT_ACK_TIMEOUT_US); 615 if (ret) { 616 dev_err(dev, "AXI bus port halt timeout\n"); 617 return ret; 618 } 619 620 return 0; 621 } 622 623 /* Halt AXI and AXI IMEM VBIF Access */ 624 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0); 625 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ; 626 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0); 627 628 /* Request for AXI bus port halt */ 629 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val, 630 val & VBIF_AXI_HALT_CTRL1_HALT_ACK, 631 POLL_INTERVAL_US, 632 VBIF_AXI_HALT_ACK_TIMEOUT_US); 633 if (ret) { 634 dev_err(dev, "AXI bus port halt timeout\n"); 635 return ret; 636 } 637 638 return 0; 639 } 640 641 static int venus_power_off(struct venus_hfi_device *hdev) 642 { 643 int ret; 644 645 if (!hdev->power_enabled) 646 return 0; 647 648 ret = venus_set_hw_state_suspend(hdev->core); 649 if (ret) 650 return ret; 651 652 ret = venus_halt_axi(hdev); 653 if (ret) 654 return ret; 655 656 hdev->power_enabled = false; 657 658 return 0; 659 } 660 661 static int venus_power_on(struct venus_hfi_device *hdev) 662 { 663 int ret; 664 665 if (hdev->power_enabled) 666 return 0; 667 668 ret = venus_set_hw_state_resume(hdev->core); 669 if (ret) 670 goto err; 671 672 ret = venus_run(hdev); 673 if (ret) 674 goto err_suspend; 675 676 hdev->power_enabled = true; 677 678 return 0; 679 680 err_suspend: 681 venus_set_hw_state_suspend(hdev->core); 682 err: 683 hdev->power_enabled = false; 684 return ret; 685 } 686 687 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev, 688 void *pkt) 689 { 690 struct iface_queue *queue; 691 u32 tx_req; 692 int ret; 693 694 if (!venus_is_valid_state(hdev)) 695 return -EINVAL; 696 697 queue = &hdev->queues[IFACEQ_MSG_IDX]; 698 699 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 700 if (ret) 701 return ret; 702 703 if (tx_req) 704 venus_soft_int(hdev); 705 706 return 0; 707 } 708 709 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt) 710 { 711 int ret; 712 713 mutex_lock(&hdev->lock); 714 ret = venus_iface_msgq_read_nolock(hdev, pkt); 715 mutex_unlock(&hdev->lock); 716 717 return ret; 718 } 719 720 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev, 721 void *pkt) 722 { 723 struct iface_queue *queue; 724 u32 tx_req; 725 int ret; 726 727 ret = venus_is_valid_state(hdev); 728 if (!ret) 729 return -EINVAL; 730 731 queue = &hdev->queues[IFACEQ_DBG_IDX]; 732 733 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 734 if (ret) 735 return ret; 736 737 if (tx_req) 738 venus_soft_int(hdev); 739 740 return 0; 741 } 742 743 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt) 744 { 745 int ret; 746 747 if (!pkt) 748 return -EINVAL; 749 750 mutex_lock(&hdev->lock); 751 ret = venus_iface_dbgq_read_nolock(hdev, pkt); 752 mutex_unlock(&hdev->lock); 753 754 return ret; 755 } 756 757 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr) 758 { 759 qhdr->status = 1; 760 qhdr->type = IFACEQ_DFLT_QHDR; 761 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4; 762 qhdr->pkt_size = 0; 763 qhdr->rx_wm = 1; 764 qhdr->tx_wm = 1; 765 qhdr->rx_req = 1; 766 qhdr->tx_req = 0; 767 qhdr->rx_irq_status = 0; 768 qhdr->tx_irq_status = 0; 769 qhdr->read_idx = 0; 770 qhdr->write_idx = 0; 771 } 772 773 static void venus_interface_queues_release(struct venus_hfi_device *hdev) 774 { 775 mutex_lock(&hdev->lock); 776 777 venus_free(hdev, &hdev->ifaceq_table); 778 venus_free(hdev, &hdev->sfr); 779 780 memset(hdev->queues, 0, sizeof(hdev->queues)); 781 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table)); 782 memset(&hdev->sfr, 0, sizeof(hdev->sfr)); 783 784 mutex_unlock(&hdev->lock); 785 } 786 787 static int venus_interface_queues_init(struct venus_hfi_device *hdev) 788 { 789 struct hfi_queue_table_header *tbl_hdr; 790 struct iface_queue *queue; 791 struct hfi_sfr *sfr; 792 struct mem_desc desc = {0}; 793 unsigned int offset; 794 unsigned int i; 795 int ret; 796 797 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE); 798 if (ret) 799 return ret; 800 801 hdev->ifaceq_table = desc; 802 offset = IFACEQ_TABLE_SIZE; 803 804 for (i = 0; i < IFACEQ_NUM; i++) { 805 queue = &hdev->queues[i]; 806 queue->qmem.da = desc.da + offset; 807 queue->qmem.kva = desc.kva + offset; 808 queue->qmem.size = IFACEQ_QUEUE_SIZE; 809 offset += queue->qmem.size; 810 queue->qhdr = 811 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 812 813 venus_set_qhdr_defaults(queue->qhdr); 814 815 queue->qhdr->start_addr = queue->qmem.da; 816 817 if (i == IFACEQ_CMD_IDX) 818 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 819 else if (i == IFACEQ_MSG_IDX) 820 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 821 else if (i == IFACEQ_DBG_IDX) 822 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 823 } 824 825 tbl_hdr = hdev->ifaceq_table.kva; 826 tbl_hdr->version = 0; 827 tbl_hdr->size = IFACEQ_TABLE_SIZE; 828 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 829 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 830 tbl_hdr->num_q = IFACEQ_NUM; 831 tbl_hdr->num_active_q = IFACEQ_NUM; 832 833 /* 834 * Set receive request to zero on debug queue as there is no 835 * need of interrupt from video hardware for debug messages 836 */ 837 queue = &hdev->queues[IFACEQ_DBG_IDX]; 838 queue->qhdr->rx_req = 0; 839 840 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE); 841 if (ret) { 842 hdev->sfr.da = 0; 843 } else { 844 hdev->sfr = desc; 845 sfr = hdev->sfr.kva; 846 sfr->buf_size = ALIGNED_SFR_SIZE; 847 } 848 849 /* ensure table and queue header structs are settled in memory */ 850 wmb(); 851 852 return 0; 853 } 854 855 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug) 856 { 857 struct hfi_sys_set_property_pkt *pkt; 858 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 859 860 pkt = (struct hfi_sys_set_property_pkt *)packet; 861 862 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug); 863 864 return venus_iface_cmdq_write(hdev, pkt, false); 865 } 866 867 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode) 868 { 869 struct hfi_sys_set_property_pkt *pkt; 870 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 871 872 pkt = (struct hfi_sys_set_property_pkt *)packet; 873 874 pkt_sys_coverage_config(pkt, mode); 875 876 return venus_iface_cmdq_write(hdev, pkt, false); 877 } 878 879 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev, 880 bool enable) 881 { 882 struct hfi_sys_set_property_pkt *pkt; 883 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 884 885 if (!enable) 886 return 0; 887 888 pkt = (struct hfi_sys_set_property_pkt *)packet; 889 890 pkt_sys_idle_indicator(pkt, enable); 891 892 return venus_iface_cmdq_write(hdev, pkt, false); 893 } 894 895 static int venus_sys_set_power_control(struct venus_hfi_device *hdev, 896 bool enable) 897 { 898 struct hfi_sys_set_property_pkt *pkt; 899 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 900 901 pkt = (struct hfi_sys_set_property_pkt *)packet; 902 903 pkt_sys_power_control(pkt, enable); 904 905 return venus_iface_cmdq_write(hdev, pkt, false); 906 } 907 908 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev) 909 { 910 struct hfi_sys_set_property_pkt *pkt; 911 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 912 const struct venus_resources *res = hdev->core->res; 913 int ret; 914 915 pkt = (struct hfi_sys_set_property_pkt *)packet; 916 917 pkt_sys_ubwc_config(pkt, res->ubwc_conf); 918 919 ret = venus_iface_cmdq_write(hdev, pkt, false); 920 if (ret) 921 return ret; 922 923 return 0; 924 } 925 926 static int venus_get_queue_size(struct venus_hfi_device *hdev, 927 unsigned int index) 928 { 929 struct hfi_queue_header *qhdr; 930 931 if (index >= IFACEQ_NUM) 932 return -EINVAL; 933 934 qhdr = hdev->queues[index].qhdr; 935 if (!qhdr) 936 return -EINVAL; 937 938 return abs(qhdr->read_idx - qhdr->write_idx); 939 } 940 941 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev) 942 { 943 struct device *dev = hdev->core->dev; 944 const struct venus_resources *res = hdev->core->res; 945 int ret; 946 947 ret = venus_sys_set_debug(hdev, venus_fw_debug); 948 if (ret) 949 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret); 950 951 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */ 952 if (IS_V1(hdev->core)) { 953 ret = venus_sys_set_idle_message(hdev, false); 954 if (ret) 955 dev_warn(dev, "setting idle response ON failed (%d)\n", ret); 956 } 957 958 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode); 959 if (ret) 960 dev_warn(dev, "setting hw power collapse ON failed (%d)\n", 961 ret); 962 963 /* For specific venus core, it is mandatory to set the UBWC configuration */ 964 if (res->ubwc_conf) { 965 ret = venus_sys_set_ubwc_config(hdev); 966 if (ret) 967 dev_warn(dev, "setting ubwc config failed (%d)\n", ret); 968 } 969 970 return ret; 971 } 972 973 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync) 974 { 975 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 976 struct hfi_session_pkt pkt; 977 978 pkt_session_cmd(&pkt, pkt_type, inst); 979 980 return venus_iface_cmdq_write(hdev, &pkt, sync); 981 } 982 983 static void venus_flush_debug_queue(struct venus_hfi_device *hdev) 984 { 985 struct device *dev = hdev->core->dev; 986 void *packet = hdev->dbg_buf; 987 988 while (!venus_iface_dbgq_read(hdev, packet)) { 989 struct hfi_msg_sys_coverage_pkt *pkt = packet; 990 991 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) { 992 struct hfi_msg_sys_debug_pkt *pkt = packet; 993 994 dev_dbg(dev, VDBGFW "%s", pkt->msg_data); 995 } 996 } 997 } 998 999 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev, 1000 bool wait) 1001 { 1002 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout); 1003 struct hfi_sys_pc_prep_pkt pkt; 1004 int ret; 1005 1006 init_completion(&hdev->pwr_collapse_prep); 1007 1008 pkt_sys_pc_prep(&pkt); 1009 1010 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1011 if (ret) 1012 return ret; 1013 1014 if (!wait) 1015 return 0; 1016 1017 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout); 1018 if (!ret) { 1019 venus_flush_debug_queue(hdev); 1020 return -ETIMEDOUT; 1021 } 1022 1023 return 0; 1024 } 1025 1026 static int venus_are_queues_empty(struct venus_hfi_device *hdev) 1027 { 1028 int ret1, ret2; 1029 1030 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX); 1031 if (ret1 < 0) 1032 return ret1; 1033 1034 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX); 1035 if (ret2 < 0) 1036 return ret2; 1037 1038 if (!ret1 && !ret2) 1039 return 1; 1040 1041 return 0; 1042 } 1043 1044 static void venus_sfr_print(struct venus_hfi_device *hdev) 1045 { 1046 struct device *dev = hdev->core->dev; 1047 struct hfi_sfr *sfr = hdev->sfr.kva; 1048 u32 size; 1049 void *p; 1050 1051 if (!sfr) 1052 return; 1053 1054 size = sfr->buf_size; 1055 if (!size) 1056 return; 1057 1058 if (size > ALIGNED_SFR_SIZE) 1059 size = ALIGNED_SFR_SIZE; 1060 1061 p = memchr(sfr->data, '\0', size); 1062 /* 1063 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates 1064 * that Venus is in the process of crashing. 1065 */ 1066 if (!p) 1067 sfr->data[size - 1] = '\0'; 1068 1069 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data); 1070 } 1071 1072 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev, 1073 void *packet) 1074 { 1075 struct hfi_msg_event_notify_pkt *event_pkt = packet; 1076 1077 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR) 1078 return; 1079 1080 venus_set_state(hdev, VENUS_STATE_DEINIT); 1081 1082 venus_sfr_print(hdev); 1083 } 1084 1085 static irqreturn_t venus_isr_thread(struct venus_core *core) 1086 { 1087 struct venus_hfi_device *hdev = to_hfi_priv(core); 1088 const struct venus_resources *res; 1089 void *pkt; 1090 u32 msg_ret; 1091 1092 if (!hdev) 1093 return IRQ_NONE; 1094 1095 res = hdev->core->res; 1096 pkt = hdev->pkt_buf; 1097 1098 1099 while (!venus_iface_msgq_read(hdev, pkt)) { 1100 msg_ret = hfi_process_msg_packet(core, pkt); 1101 switch (msg_ret) { 1102 case HFI_MSG_EVENT_NOTIFY: 1103 venus_process_msg_sys_error(hdev, pkt); 1104 break; 1105 case HFI_MSG_SYS_INIT: 1106 venus_hfi_core_set_resource(core, res->vmem_id, 1107 res->vmem_size, 1108 res->vmem_addr, 1109 hdev); 1110 break; 1111 case HFI_MSG_SYS_RELEASE_RESOURCE: 1112 complete(&hdev->release_resource); 1113 break; 1114 case HFI_MSG_SYS_PC_PREP: 1115 complete(&hdev->pwr_collapse_prep); 1116 break; 1117 default: 1118 break; 1119 } 1120 } 1121 1122 venus_flush_debug_queue(hdev); 1123 1124 return IRQ_HANDLED; 1125 } 1126 1127 static irqreturn_t venus_isr(struct venus_core *core) 1128 { 1129 struct venus_hfi_device *hdev = to_hfi_priv(core); 1130 u32 status; 1131 void __iomem *cpu_cs_base; 1132 void __iomem *wrapper_base; 1133 1134 if (!hdev) 1135 return IRQ_NONE; 1136 1137 cpu_cs_base = hdev->core->cpu_cs_base; 1138 wrapper_base = hdev->core->wrapper_base; 1139 1140 status = readl(wrapper_base + WRAPPER_INTR_STATUS); 1141 if (IS_IRIS2(core) || IS_IRIS2_1(core)) { 1142 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1143 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 || 1144 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1145 hdev->irq_status = status; 1146 } else { 1147 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1148 status & WRAPPER_INTR_STATUS_A2HWD_MASK || 1149 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1150 hdev->irq_status = status; 1151 } 1152 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR); 1153 if (!(IS_IRIS2(core) || IS_IRIS2_1(core))) 1154 writel(status, wrapper_base + WRAPPER_INTR_CLEAR); 1155 1156 return IRQ_WAKE_THREAD; 1157 } 1158 1159 static int venus_core_init(struct venus_core *core) 1160 { 1161 struct venus_hfi_device *hdev = to_hfi_priv(core); 1162 struct device *dev = core->dev; 1163 struct hfi_sys_get_property_pkt version_pkt; 1164 struct hfi_sys_init_pkt pkt; 1165 int ret; 1166 1167 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX); 1168 1169 venus_set_state(hdev, VENUS_STATE_INIT); 1170 1171 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1172 if (ret) 1173 return ret; 1174 1175 pkt_sys_image_version(&version_pkt); 1176 1177 ret = venus_iface_cmdq_write(hdev, &version_pkt, false); 1178 if (ret) 1179 dev_warn(dev, "failed to send image version pkt to fw\n"); 1180 1181 ret = venus_sys_set_default_properties(hdev); 1182 if (ret) 1183 return ret; 1184 1185 return 0; 1186 } 1187 1188 static int venus_core_deinit(struct venus_core *core) 1189 { 1190 struct venus_hfi_device *hdev = to_hfi_priv(core); 1191 1192 venus_set_state(hdev, VENUS_STATE_DEINIT); 1193 hdev->suspended = true; 1194 hdev->power_enabled = false; 1195 1196 return 0; 1197 } 1198 1199 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type) 1200 { 1201 struct venus_hfi_device *hdev = to_hfi_priv(core); 1202 struct hfi_sys_test_ssr_pkt pkt; 1203 int ret; 1204 1205 ret = pkt_sys_ssr_cmd(&pkt, trigger_type); 1206 if (ret) 1207 return ret; 1208 1209 return venus_iface_cmdq_write(hdev, &pkt, false); 1210 } 1211 1212 static int venus_session_init(struct venus_inst *inst, u32 session_type, 1213 u32 codec) 1214 { 1215 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1216 struct hfi_session_init_pkt pkt; 1217 int ret; 1218 1219 ret = venus_sys_set_debug(hdev, venus_fw_debug); 1220 if (ret) 1221 goto err; 1222 1223 ret = pkt_session_init(&pkt, inst, session_type, codec); 1224 if (ret) 1225 goto err; 1226 1227 ret = venus_iface_cmdq_write(hdev, &pkt, true); 1228 if (ret) 1229 goto err; 1230 1231 return 0; 1232 1233 err: 1234 venus_flush_debug_queue(hdev); 1235 return ret; 1236 } 1237 1238 static int venus_session_end(struct venus_inst *inst) 1239 { 1240 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1241 struct device *dev = hdev->core->dev; 1242 1243 if (venus_fw_coverage) { 1244 if (venus_sys_set_coverage(hdev, venus_fw_coverage)) 1245 dev_warn(dev, "fw coverage msg ON failed\n"); 1246 } 1247 1248 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true); 1249 } 1250 1251 static int venus_session_abort(struct venus_inst *inst) 1252 { 1253 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1254 1255 venus_flush_debug_queue(hdev); 1256 1257 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true); 1258 } 1259 1260 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode) 1261 { 1262 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1263 struct hfi_session_flush_pkt pkt; 1264 int ret; 1265 1266 ret = pkt_session_flush(&pkt, inst, flush_mode); 1267 if (ret) 1268 return ret; 1269 1270 return venus_iface_cmdq_write(hdev, &pkt, true); 1271 } 1272 1273 static int venus_session_start(struct venus_inst *inst) 1274 { 1275 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true); 1276 } 1277 1278 static int venus_session_stop(struct venus_inst *inst) 1279 { 1280 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true); 1281 } 1282 1283 static int venus_session_continue(struct venus_inst *inst) 1284 { 1285 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false); 1286 } 1287 1288 static int venus_session_etb(struct venus_inst *inst, 1289 struct hfi_frame_data *in_frame) 1290 { 1291 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1292 u32 session_type = inst->session_type; 1293 int ret; 1294 1295 if (session_type == VIDC_SESSION_TYPE_DEC) { 1296 struct hfi_session_empty_buffer_compressed_pkt pkt; 1297 1298 ret = pkt_session_etb_decoder(&pkt, inst, in_frame); 1299 if (ret) 1300 return ret; 1301 1302 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1303 } else if (session_type == VIDC_SESSION_TYPE_ENC) { 1304 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt; 1305 1306 ret = pkt_session_etb_encoder(&pkt, inst, in_frame); 1307 if (ret) 1308 return ret; 1309 1310 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1311 } else { 1312 ret = -EINVAL; 1313 } 1314 1315 return ret; 1316 } 1317 1318 static int venus_session_ftb(struct venus_inst *inst, 1319 struct hfi_frame_data *out_frame) 1320 { 1321 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1322 struct hfi_session_fill_buffer_pkt pkt; 1323 int ret; 1324 1325 ret = pkt_session_ftb(&pkt, inst, out_frame); 1326 if (ret) 1327 return ret; 1328 1329 return venus_iface_cmdq_write(hdev, &pkt, false); 1330 } 1331 1332 static int venus_session_set_buffers(struct venus_inst *inst, 1333 struct hfi_buffer_desc *bd) 1334 { 1335 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1336 struct hfi_session_set_buffers_pkt *pkt; 1337 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1338 int ret; 1339 1340 if (bd->buffer_type == HFI_BUFFER_INPUT) 1341 return 0; 1342 1343 pkt = (struct hfi_session_set_buffers_pkt *)packet; 1344 1345 ret = pkt_session_set_buffers(pkt, inst, bd); 1346 if (ret) 1347 return ret; 1348 1349 return venus_iface_cmdq_write(hdev, pkt, false); 1350 } 1351 1352 static int venus_session_unset_buffers(struct venus_inst *inst, 1353 struct hfi_buffer_desc *bd) 1354 { 1355 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1356 struct hfi_session_release_buffer_pkt *pkt; 1357 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1358 int ret; 1359 1360 if (bd->buffer_type == HFI_BUFFER_INPUT) 1361 return 0; 1362 1363 pkt = (struct hfi_session_release_buffer_pkt *)packet; 1364 1365 ret = pkt_session_unset_buffers(pkt, inst, bd); 1366 if (ret) 1367 return ret; 1368 1369 return venus_iface_cmdq_write(hdev, pkt, true); 1370 } 1371 1372 static int venus_session_load_res(struct venus_inst *inst) 1373 { 1374 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true); 1375 } 1376 1377 static int venus_session_release_res(struct venus_inst *inst) 1378 { 1379 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true); 1380 } 1381 1382 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1383 u32 seq_hdr_len) 1384 { 1385 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1386 struct hfi_session_parse_sequence_header_pkt *pkt; 1387 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1388 int ret; 1389 1390 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet; 1391 1392 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len); 1393 if (ret) 1394 return ret; 1395 1396 ret = venus_iface_cmdq_write(hdev, pkt, false); 1397 if (ret) 1398 return ret; 1399 1400 return 0; 1401 } 1402 1403 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1404 u32 seq_hdr_len) 1405 { 1406 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1407 struct hfi_session_get_sequence_header_pkt *pkt; 1408 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1409 int ret; 1410 1411 pkt = (struct hfi_session_get_sequence_header_pkt *)packet; 1412 1413 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len); 1414 if (ret) 1415 return ret; 1416 1417 return venus_iface_cmdq_write(hdev, pkt, false); 1418 } 1419 1420 static int venus_session_set_property(struct venus_inst *inst, u32 ptype, 1421 void *pdata) 1422 { 1423 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1424 struct hfi_session_set_property_pkt *pkt; 1425 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1426 int ret; 1427 1428 pkt = (struct hfi_session_set_property_pkt *)packet; 1429 1430 ret = pkt_session_set_property(pkt, inst, ptype, pdata); 1431 if (ret == -ENOTSUPP) 1432 return 0; 1433 if (ret) 1434 return ret; 1435 1436 return venus_iface_cmdq_write(hdev, pkt, false); 1437 } 1438 1439 static int venus_session_get_property(struct venus_inst *inst, u32 ptype) 1440 { 1441 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1442 struct hfi_session_get_property_pkt pkt; 1443 int ret; 1444 1445 ret = pkt_session_get_property(&pkt, inst, ptype); 1446 if (ret) 1447 return ret; 1448 1449 return venus_iface_cmdq_write(hdev, &pkt, true); 1450 } 1451 1452 static int venus_resume(struct venus_core *core) 1453 { 1454 struct venus_hfi_device *hdev = to_hfi_priv(core); 1455 int ret = 0; 1456 1457 mutex_lock(&hdev->lock); 1458 1459 if (!hdev->suspended) 1460 goto unlock; 1461 1462 ret = venus_power_on(hdev); 1463 1464 unlock: 1465 if (!ret) 1466 hdev->suspended = false; 1467 1468 mutex_unlock(&hdev->lock); 1469 1470 return ret; 1471 } 1472 1473 static int venus_suspend_1xx(struct venus_core *core) 1474 { 1475 struct venus_hfi_device *hdev = to_hfi_priv(core); 1476 struct device *dev = core->dev; 1477 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1478 u32 ctrl_status; 1479 int ret; 1480 1481 if (!hdev->power_enabled || hdev->suspended) 1482 return 0; 1483 1484 mutex_lock(&hdev->lock); 1485 ret = venus_is_valid_state(hdev); 1486 mutex_unlock(&hdev->lock); 1487 1488 if (!ret) { 1489 dev_err(dev, "bad state, cannot suspend\n"); 1490 return -EINVAL; 1491 } 1492 1493 ret = venus_prepare_power_collapse(hdev, true); 1494 if (ret) { 1495 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1496 return ret; 1497 } 1498 1499 mutex_lock(&hdev->lock); 1500 1501 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) { 1502 mutex_unlock(&hdev->lock); 1503 return -EINVAL; 1504 } 1505 1506 ret = venus_are_queues_empty(hdev); 1507 if (ret < 0 || !ret) { 1508 mutex_unlock(&hdev->lock); 1509 return -EINVAL; 1510 } 1511 1512 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1513 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) { 1514 mutex_unlock(&hdev->lock); 1515 return -EINVAL; 1516 } 1517 1518 ret = venus_power_off(hdev); 1519 if (ret) { 1520 mutex_unlock(&hdev->lock); 1521 return ret; 1522 } 1523 1524 hdev->suspended = true; 1525 1526 mutex_unlock(&hdev->lock); 1527 1528 return 0; 1529 } 1530 1531 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev) 1532 { 1533 void __iomem *wrapper_base = hdev->core->wrapper_base; 1534 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1535 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1536 u32 ctrl_status, cpu_status; 1537 1538 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1539 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1540 else 1541 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1542 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1543 1544 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1545 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1546 return true; 1547 1548 return false; 1549 } 1550 1551 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev) 1552 { 1553 void __iomem *wrapper_base = hdev->core->wrapper_base; 1554 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1555 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1556 u32 ctrl_status, cpu_status; 1557 1558 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1559 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1560 else 1561 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1562 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1563 1564 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1565 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1566 return true; 1567 1568 return false; 1569 } 1570 1571 static int venus_suspend_3xx(struct venus_core *core) 1572 { 1573 struct venus_hfi_device *hdev = to_hfi_priv(core); 1574 struct device *dev = core->dev; 1575 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1576 u32 ctrl_status; 1577 bool val; 1578 int ret; 1579 1580 if (!hdev->power_enabled || hdev->suspended) 1581 return 0; 1582 1583 mutex_lock(&hdev->lock); 1584 ret = venus_is_valid_state(hdev); 1585 mutex_unlock(&hdev->lock); 1586 1587 if (!ret) { 1588 dev_err(dev, "bad state, cannot suspend\n"); 1589 return -EINVAL; 1590 } 1591 1592 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1593 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1594 goto power_off; 1595 1596 /* 1597 * Power collapse sequence for Venus 3xx and 4xx versions: 1598 * 1. Check for ARM9 and video core to be idle by checking WFI bit 1599 * (bit 0) in CPU status register and by checking Idle (bit 30) in 1600 * Control status register for video core. 1601 * 2. Send a command to prepare for power collapse. 1602 * 3. Check for WFI and PC_READY bits. 1603 */ 1604 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val, 1605 1500, 100 * 1500); 1606 if (ret) { 1607 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret); 1608 return ret; 1609 } 1610 1611 ret = venus_prepare_power_collapse(hdev, false); 1612 if (ret) { 1613 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1614 return ret; 1615 } 1616 1617 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val, 1618 1500, 100 * 1500); 1619 if (ret) 1620 return ret; 1621 1622 power_off: 1623 mutex_lock(&hdev->lock); 1624 1625 ret = venus_power_off(hdev); 1626 if (ret) { 1627 dev_err(dev, "venus_power_off (%d)\n", ret); 1628 mutex_unlock(&hdev->lock); 1629 return ret; 1630 } 1631 1632 hdev->suspended = true; 1633 1634 mutex_unlock(&hdev->lock); 1635 1636 return 0; 1637 } 1638 1639 static int venus_suspend(struct venus_core *core) 1640 { 1641 if (IS_V3(core) || IS_V4(core) || IS_V6(core)) 1642 return venus_suspend_3xx(core); 1643 1644 return venus_suspend_1xx(core); 1645 } 1646 1647 static const struct hfi_ops venus_hfi_ops = { 1648 .core_init = venus_core_init, 1649 .core_deinit = venus_core_deinit, 1650 .core_trigger_ssr = venus_core_trigger_ssr, 1651 1652 .session_init = venus_session_init, 1653 .session_end = venus_session_end, 1654 .session_abort = venus_session_abort, 1655 .session_flush = venus_session_flush, 1656 .session_start = venus_session_start, 1657 .session_stop = venus_session_stop, 1658 .session_continue = venus_session_continue, 1659 .session_etb = venus_session_etb, 1660 .session_ftb = venus_session_ftb, 1661 .session_set_buffers = venus_session_set_buffers, 1662 .session_unset_buffers = venus_session_unset_buffers, 1663 .session_load_res = venus_session_load_res, 1664 .session_release_res = venus_session_release_res, 1665 .session_parse_seq_hdr = venus_session_parse_seq_hdr, 1666 .session_get_seq_hdr = venus_session_get_seq_hdr, 1667 .session_set_property = venus_session_set_property, 1668 .session_get_property = venus_session_get_property, 1669 1670 .resume = venus_resume, 1671 .suspend = venus_suspend, 1672 1673 .isr = venus_isr, 1674 .isr_thread = venus_isr_thread, 1675 }; 1676 1677 void venus_hfi_destroy(struct venus_core *core) 1678 { 1679 struct venus_hfi_device *hdev = to_hfi_priv(core); 1680 1681 core->priv = NULL; 1682 venus_interface_queues_release(hdev); 1683 mutex_destroy(&hdev->lock); 1684 kfree(hdev); 1685 disable_irq(core->irq); 1686 core->ops = NULL; 1687 } 1688 1689 int venus_hfi_create(struct venus_core *core) 1690 { 1691 struct venus_hfi_device *hdev; 1692 int ret; 1693 1694 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL); 1695 if (!hdev) 1696 return -ENOMEM; 1697 1698 mutex_init(&hdev->lock); 1699 1700 hdev->core = core; 1701 hdev->suspended = true; 1702 core->priv = hdev; 1703 core->ops = &venus_hfi_ops; 1704 1705 ret = venus_interface_queues_init(hdev); 1706 if (ret) 1707 goto err_kfree; 1708 1709 return 0; 1710 1711 err_kfree: 1712 kfree(hdev); 1713 core->priv = NULL; 1714 core->ops = NULL; 1715 return ret; 1716 } 1717 1718 void venus_hfi_queues_reinit(struct venus_core *core) 1719 { 1720 struct venus_hfi_device *hdev = to_hfi_priv(core); 1721 struct hfi_queue_table_header *tbl_hdr; 1722 struct iface_queue *queue; 1723 struct hfi_sfr *sfr; 1724 unsigned int i; 1725 1726 mutex_lock(&hdev->lock); 1727 1728 for (i = 0; i < IFACEQ_NUM; i++) { 1729 queue = &hdev->queues[i]; 1730 queue->qhdr = 1731 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 1732 1733 venus_set_qhdr_defaults(queue->qhdr); 1734 1735 queue->qhdr->start_addr = queue->qmem.da; 1736 1737 if (i == IFACEQ_CMD_IDX) 1738 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 1739 else if (i == IFACEQ_MSG_IDX) 1740 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 1741 else if (i == IFACEQ_DBG_IDX) 1742 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 1743 } 1744 1745 tbl_hdr = hdev->ifaceq_table.kva; 1746 tbl_hdr->version = 0; 1747 tbl_hdr->size = IFACEQ_TABLE_SIZE; 1748 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 1749 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 1750 tbl_hdr->num_q = IFACEQ_NUM; 1751 tbl_hdr->num_active_q = IFACEQ_NUM; 1752 1753 /* 1754 * Set receive request to zero on debug queue as there is no 1755 * need of interrupt from video hardware for debug messages 1756 */ 1757 queue = &hdev->queues[IFACEQ_DBG_IDX]; 1758 queue->qhdr->rx_req = 0; 1759 1760 sfr = hdev->sfr.kva; 1761 sfr->buf_size = ALIGNED_SFR_SIZE; 1762 1763 /* ensure table and queue header structs are settled in memory */ 1764 wmb(); 1765 1766 mutex_unlock(&hdev->lock); 1767 } 1768