1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2017 Linaro Ltd.
5 */
6
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14
15 #include "core.h"
16 #include "hfi_cmds.h"
17 #include "hfi_msgs.h"
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
20 #include "firmware.h"
21
22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff
26
27 #define HFI_HOST_TO_CTRL_CMD_Q 0
28 #define HFI_CTRL_TO_HOST_MSG_Q 1
29 #define HFI_CTRL_TO_HOST_DBG_Q 2
30 #define HFI_MASK_QHDR_STATUS 0x000000ff
31
32 #define IFACEQ_NUM 3
33 #define IFACEQ_CMD_IDX 0
34 #define IFACEQ_MSG_IDX 1
35 #define IFACEQ_DBG_IDX 2
36 #define IFACEQ_MAX_BUF_COUNT 50
37 #define IFACEQ_MAX_PARALLEL_CLNTS 16
38 #define IFACEQ_DFLT_QHDR 0x01010000
39
40 #define POLL_INTERVAL_US 50
41
42 #define IFACEQ_MAX_PKT_SIZE 1024
43 #define IFACEQ_MED_PKT_SIZE 768
44 #define IFACEQ_MIN_PKT_SIZE 8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)
48
49 struct hfi_queue_table_header {
50 u32 version;
51 u32 size;
52 u32 qhdr0_offset;
53 u32 qhdr_size;
54 u32 num_q;
55 u32 num_active_q;
56 };
57
58 struct hfi_queue_header {
59 u32 status;
60 u32 start_addr;
61 u32 type;
62 u32 q_size;
63 u32 pkt_size;
64 u32 pkt_drop_cnt;
65 u32 rx_wm;
66 u32 tx_wm;
67 u32 rx_req;
68 u32 tx_req;
69 u32 rx_irq_status;
70 u32 tx_irq_status;
71 u32 read_idx;
72 u32 write_idx;
73 };
74
75 #define IFACEQ_TABLE_SIZE \
76 (sizeof(struct hfi_queue_table_header) + \
77 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
78
79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \
80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
81
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \
83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \
84 ((i) * sizeof(struct hfi_queue_header)))
85
86 #define QDSS_SIZE SZ_4K
87 #define SFR_SIZE SZ_4K
88 #define QUEUE_SIZE \
89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
90
91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 ALIGNED_QDSS_SIZE, SZ_1M)
96
97 struct mem_desc {
98 dma_addr_t da; /* device address */
99 void *kva; /* kernel virtual address */
100 u32 size;
101 unsigned long attrs;
102 };
103
104 struct iface_queue {
105 struct hfi_queue_header *qhdr;
106 struct mem_desc qmem;
107 };
108
109 enum venus_state {
110 VENUS_STATE_DEINIT = 1,
111 VENUS_STATE_INIT,
112 };
113
114 struct venus_hfi_device {
115 struct venus_core *core;
116 u32 irq_status;
117 u32 last_packet_type;
118 bool power_enabled;
119 bool suspended;
120 enum venus_state state;
121 /* serialize read / write to the shared memory */
122 struct mutex lock;
123 struct completion pwr_collapse_prep;
124 struct completion release_resource;
125 struct mem_desc ifaceq_table;
126 struct mem_desc sfr;
127 struct iface_queue queues[IFACEQ_NUM];
128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
130 };
131
132 static bool venus_pkt_debug;
133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_fw_low_power_mode = true;
135 static int venus_hw_rsp_timeout = 1000;
136 static bool venus_fw_coverage;
137
venus_set_state(struct venus_hfi_device * hdev,enum venus_state state)138 static void venus_set_state(struct venus_hfi_device *hdev,
139 enum venus_state state)
140 {
141 mutex_lock(&hdev->lock);
142 hdev->state = state;
143 mutex_unlock(&hdev->lock);
144 }
145
venus_is_valid_state(struct venus_hfi_device * hdev)146 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
147 {
148 return hdev->state != VENUS_STATE_DEINIT;
149 }
150
venus_dump_packet(struct venus_hfi_device * hdev,const void * packet)151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
152 {
153 size_t pkt_size = *(u32 *)packet;
154
155 if (!venus_pkt_debug)
156 return;
157
158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
159 pkt_size, true);
160 }
161
venus_write_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * packet,u32 * rx_req)162 static int venus_write_queue(struct venus_hfi_device *hdev,
163 struct iface_queue *queue,
164 void *packet, u32 *rx_req)
165 {
166 struct hfi_queue_header *qhdr;
167 u32 dwords, new_wr_idx;
168 u32 empty_space, rd_idx, wr_idx, qsize;
169 u32 *wr_ptr;
170
171 if (!queue->qmem.kva)
172 return -EINVAL;
173
174 qhdr = queue->qhdr;
175 if (!qhdr)
176 return -EINVAL;
177
178 venus_dump_packet(hdev, packet);
179
180 dwords = (*(u32 *)packet) >> 2;
181 if (!dwords)
182 return -EINVAL;
183
184 rd_idx = qhdr->read_idx;
185 wr_idx = qhdr->write_idx;
186 qsize = qhdr->q_size;
187 /* ensure rd/wr indices's are read from memory */
188 rmb();
189
190 if (wr_idx >= rd_idx)
191 empty_space = qsize - (wr_idx - rd_idx);
192 else
193 empty_space = rd_idx - wr_idx;
194
195 if (empty_space <= dwords) {
196 qhdr->tx_req = 1;
197 /* ensure tx_req is updated in memory */
198 wmb();
199 return -ENOSPC;
200 }
201
202 qhdr->tx_req = 0;
203 /* ensure tx_req is updated in memory */
204 wmb();
205
206 new_wr_idx = wr_idx + dwords;
207 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
208
209 if (wr_ptr < (u32 *)queue->qmem.kva ||
210 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))
211 return -EINVAL;
212
213 if (new_wr_idx < qsize) {
214 memcpy(wr_ptr, packet, dwords << 2);
215 } else {
216 size_t len;
217
218 new_wr_idx -= qsize;
219 len = (dwords - new_wr_idx) << 2;
220 memcpy(wr_ptr, packet, len);
221 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
222 }
223
224 /* make sure packet is written before updating the write index */
225 wmb();
226
227 qhdr->write_idx = new_wr_idx;
228 *rx_req = qhdr->rx_req ? 1 : 0;
229
230 /* make sure write index is updated before an interrupt is raised */
231 mb();
232
233 return 0;
234 }
235
venus_read_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * pkt,u32 * tx_req)236 static int venus_read_queue(struct venus_hfi_device *hdev,
237 struct iface_queue *queue, void *pkt, u32 *tx_req)
238 {
239 struct hfi_queue_header *qhdr;
240 u32 dwords, new_rd_idx;
241 u32 rd_idx, wr_idx, type, qsize;
242 u32 *rd_ptr;
243 u32 recv_request = 0;
244 int ret = 0;
245
246 if (!queue->qmem.kva)
247 return -EINVAL;
248
249 qhdr = queue->qhdr;
250 if (!qhdr)
251 return -EINVAL;
252
253 type = qhdr->type;
254 rd_idx = qhdr->read_idx;
255 wr_idx = qhdr->write_idx;
256 qsize = qhdr->q_size;
257
258 /* make sure data is valid before using it */
259 rmb();
260
261 /*
262 * Do not set receive request for debug queue, if set, Venus generates
263 * interrupt for debug messages even when there is no response message
264 * available. In general debug queue will not become full as it is being
265 * emptied out for every interrupt from Venus. Venus will anyway
266 * generates interrupt if it is full.
267 */
268 if (type & HFI_CTRL_TO_HOST_MSG_Q)
269 recv_request = 1;
270
271 if (rd_idx == wr_idx) {
272 qhdr->rx_req = recv_request;
273 *tx_req = 0;
274 /* update rx_req field in memory */
275 wmb();
276 return -ENODATA;
277 }
278
279 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
280
281 if (rd_ptr < (u32 *)queue->qmem.kva ||
282 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))
283 return -EINVAL;
284
285 dwords = *rd_ptr >> 2;
286 if (!dwords)
287 return -EINVAL;
288
289 new_rd_idx = rd_idx + dwords;
290 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
291 if (new_rd_idx < qsize) {
292 memcpy(pkt, rd_ptr, dwords << 2);
293 } else {
294 size_t len;
295
296 new_rd_idx -= qsize;
297 len = (dwords - new_rd_idx) << 2;
298 memcpy(pkt, rd_ptr, len);
299 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
300 }
301 } else {
302 /* bad packet received, dropping */
303 new_rd_idx = qhdr->write_idx;
304 ret = -EBADMSG;
305 }
306
307 /* ensure the packet is read before updating read index */
308 rmb();
309
310 qhdr->read_idx = new_rd_idx;
311 /* ensure updating read index */
312 wmb();
313
314 rd_idx = qhdr->read_idx;
315 wr_idx = qhdr->write_idx;
316 /* ensure rd/wr indices are read from memory */
317 rmb();
318
319 if (rd_idx != wr_idx)
320 qhdr->rx_req = 0;
321 else
322 qhdr->rx_req = recv_request;
323
324 *tx_req = qhdr->tx_req ? 1 : 0;
325
326 /* ensure rx_req is stored to memory and tx_req is loaded from memory */
327 mb();
328
329 venus_dump_packet(hdev, pkt);
330
331 return ret;
332 }
333
venus_alloc(struct venus_hfi_device * hdev,struct mem_desc * desc,u32 size)334 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
335 u32 size)
336 {
337 struct device *dev = hdev->core->dev;
338
339 desc->attrs = DMA_ATTR_WRITE_COMBINE;
340 desc->size = ALIGN(size, SZ_4K);
341
342 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
343 desc->attrs);
344 if (!desc->kva)
345 return -ENOMEM;
346
347 return 0;
348 }
349
venus_free(struct venus_hfi_device * hdev,struct mem_desc * mem)350 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
351 {
352 struct device *dev = hdev->core->dev;
353
354 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
355 }
356
venus_set_registers(struct venus_hfi_device * hdev)357 static void venus_set_registers(struct venus_hfi_device *hdev)
358 {
359 const struct venus_resources *res = hdev->core->res;
360 const struct reg_val *tbl = res->reg_tbl;
361 unsigned int count = res->reg_tbl_size;
362 unsigned int i;
363
364 for (i = 0; i < count; i++)
365 writel(tbl[i].value, hdev->core->base + tbl[i].reg);
366 }
367
venus_soft_int(struct venus_hfi_device * hdev)368 static void venus_soft_int(struct venus_hfi_device *hdev)
369 {
370 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
371 u32 clear_bit;
372
373 if (IS_V6(hdev->core))
374 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
375 else
376 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
377
378 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
379 }
380
venus_iface_cmdq_write_nolock(struct venus_hfi_device * hdev,void * pkt,bool sync)381 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
382 void *pkt, bool sync)
383 {
384 struct device *dev = hdev->core->dev;
385 struct hfi_pkt_hdr *cmd_packet;
386 struct iface_queue *queue;
387 u32 rx_req;
388 int ret;
389
390 if (!venus_is_valid_state(hdev))
391 return -EINVAL;
392
393 cmd_packet = (struct hfi_pkt_hdr *)pkt;
394 hdev->last_packet_type = cmd_packet->pkt_type;
395
396 queue = &hdev->queues[IFACEQ_CMD_IDX];
397
398 ret = venus_write_queue(hdev, queue, pkt, &rx_req);
399 if (ret) {
400 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
401 return ret;
402 }
403
404 if (sync) {
405 /*
406 * Inform video hardware to raise interrupt for synchronous
407 * commands
408 */
409 queue = &hdev->queues[IFACEQ_MSG_IDX];
410 queue->qhdr->rx_req = 1;
411 /* ensure rx_req is updated in memory */
412 wmb();
413 }
414
415 if (rx_req)
416 venus_soft_int(hdev);
417
418 return 0;
419 }
420
venus_iface_cmdq_write(struct venus_hfi_device * hdev,void * pkt,bool sync)421 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
422 {
423 int ret;
424
425 mutex_lock(&hdev->lock);
426 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
427 mutex_unlock(&hdev->lock);
428
429 return ret;
430 }
431
venus_hfi_core_set_resource(struct venus_core * core,u32 id,u32 size,u32 addr,void * cookie)432 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
433 u32 size, u32 addr, void *cookie)
434 {
435 struct venus_hfi_device *hdev = to_hfi_priv(core);
436 struct hfi_sys_set_resource_pkt *pkt;
437 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
438 int ret;
439
440 if (id == VIDC_RESOURCE_NONE)
441 return 0;
442
443 pkt = (struct hfi_sys_set_resource_pkt *)packet;
444
445 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
446 if (ret)
447 return ret;
448
449 ret = venus_iface_cmdq_write(hdev, pkt, false);
450 if (ret)
451 return ret;
452
453 return 0;
454 }
455
venus_boot_core(struct venus_hfi_device * hdev)456 static int venus_boot_core(struct venus_hfi_device *hdev)
457 {
458 struct device *dev = hdev->core->dev;
459 static const unsigned int max_tries = 100;
460 u32 ctrl_status = 0, mask_val = 0;
461 unsigned int count = 0;
462 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
463 void __iomem *wrapper_base = hdev->core->wrapper_base;
464 int ret = 0;
465
466 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
467 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
468 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
469 WRAPPER_INTR_MASK_A2HCPU_MASK);
470 } else {
471 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
472 }
473
474 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
475 if (IS_V1(hdev->core))
476 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
477
478 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
479 while (!ctrl_status && count < max_tries) {
480 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
481 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
482 dev_err(dev, "invalid setting for UC_REGION\n");
483 ret = -EINVAL;
484 break;
485 }
486
487 usleep_range(500, 1000);
488 count++;
489 }
490
491 if (count >= max_tries)
492 ret = -ETIMEDOUT;
493
494 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
495 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
496 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
497 }
498
499 return ret;
500 }
501
venus_hwversion(struct venus_hfi_device * hdev)502 static u32 venus_hwversion(struct venus_hfi_device *hdev)
503 {
504 struct device *dev = hdev->core->dev;
505 void __iomem *wrapper_base = hdev->core->wrapper_base;
506 u32 ver;
507 u32 major, minor, step;
508
509 ver = readl(wrapper_base + WRAPPER_HW_VERSION);
510 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
511 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
512 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
513 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
514 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
515
516 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
517
518 return major;
519 }
520
venus_run(struct venus_hfi_device * hdev)521 static int venus_run(struct venus_hfi_device *hdev)
522 {
523 struct device *dev = hdev->core->dev;
524 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
525 int ret;
526
527 /*
528 * Re-program all of the registers that get reset as a result of
529 * regulator_disable() and _enable()
530 */
531 venus_set_registers(hdev);
532
533 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
534 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
535 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
536 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
537 if (hdev->sfr.da)
538 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
539
540 ret = venus_boot_core(hdev);
541 if (ret) {
542 dev_err(dev, "failed to reset venus core\n");
543 return ret;
544 }
545
546 venus_hwversion(hdev);
547
548 return 0;
549 }
550
venus_halt_axi(struct venus_hfi_device * hdev)551 static int venus_halt_axi(struct venus_hfi_device *hdev)
552 {
553 void __iomem *wrapper_base = hdev->core->wrapper_base;
554 void __iomem *vbif_base = hdev->core->vbif_base;
555 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
556 void __iomem *aon_base = hdev->core->aon_base;
557 struct device *dev = hdev->core->dev;
558 u32 val;
559 u32 mask_val;
560 int ret;
561
562 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
563 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
564
565 if (IS_IRIS2_1(hdev->core))
566 goto skip_aon_mvp_noc;
567
568 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
569 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
570 val,
571 val & BIT(0),
572 POLL_INTERVAL_US,
573 VBIF_AXI_HALT_ACK_TIMEOUT_US);
574 if (ret)
575 return -ETIMEDOUT;
576
577 skip_aon_mvp_noc:
578 mask_val = (BIT(2) | BIT(1) | BIT(0));
579 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
580
581 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
582 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
583 val,
584 val == 0,
585 POLL_INTERVAL_US,
586 VBIF_AXI_HALT_ACK_TIMEOUT_US);
587
588 if (ret) {
589 dev_err(dev, "DBLP Release: lpi_status %x\n", val);
590 return -ETIMEDOUT;
591 }
592 return 0;
593 }
594
595 if (IS_V4(hdev->core)) {
596 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
597 val |= WRAPPER_CPU_AXI_HALT_HALT;
598 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
599
600 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
601 val,
602 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
603 POLL_INTERVAL_US,
604 VBIF_AXI_HALT_ACK_TIMEOUT_US);
605 if (ret) {
606 dev_err(dev, "AXI bus port halt timeout\n");
607 return ret;
608 }
609
610 return 0;
611 }
612
613 /* Halt AXI and AXI IMEM VBIF Access */
614 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
615 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
616 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
617
618 /* Request for AXI bus port halt */
619 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
620 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
621 POLL_INTERVAL_US,
622 VBIF_AXI_HALT_ACK_TIMEOUT_US);
623 if (ret) {
624 dev_err(dev, "AXI bus port halt timeout\n");
625 return ret;
626 }
627
628 return 0;
629 }
630
venus_power_off(struct venus_hfi_device * hdev)631 static int venus_power_off(struct venus_hfi_device *hdev)
632 {
633 int ret;
634
635 if (!hdev->power_enabled)
636 return 0;
637
638 ret = venus_set_hw_state_suspend(hdev->core);
639 if (ret)
640 return ret;
641
642 ret = venus_halt_axi(hdev);
643 if (ret)
644 return ret;
645
646 hdev->power_enabled = false;
647
648 return 0;
649 }
650
venus_power_on(struct venus_hfi_device * hdev)651 static int venus_power_on(struct venus_hfi_device *hdev)
652 {
653 int ret;
654
655 if (hdev->power_enabled)
656 return 0;
657
658 ret = venus_set_hw_state_resume(hdev->core);
659 if (ret)
660 goto err;
661
662 ret = venus_run(hdev);
663 if (ret)
664 goto err_suspend;
665
666 hdev->power_enabled = true;
667
668 return 0;
669
670 err_suspend:
671 venus_set_hw_state_suspend(hdev->core);
672 err:
673 hdev->power_enabled = false;
674 return ret;
675 }
676
venus_iface_msgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)677 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
678 void *pkt)
679 {
680 struct iface_queue *queue;
681 u32 tx_req;
682 int ret;
683
684 if (!venus_is_valid_state(hdev))
685 return -EINVAL;
686
687 queue = &hdev->queues[IFACEQ_MSG_IDX];
688
689 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
690 if (ret)
691 return ret;
692
693 if (tx_req)
694 venus_soft_int(hdev);
695
696 return 0;
697 }
698
venus_iface_msgq_read(struct venus_hfi_device * hdev,void * pkt)699 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
700 {
701 int ret;
702
703 mutex_lock(&hdev->lock);
704 ret = venus_iface_msgq_read_nolock(hdev, pkt);
705 mutex_unlock(&hdev->lock);
706
707 return ret;
708 }
709
venus_iface_dbgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)710 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
711 void *pkt)
712 {
713 struct iface_queue *queue;
714 u32 tx_req;
715 int ret;
716
717 ret = venus_is_valid_state(hdev);
718 if (!ret)
719 return -EINVAL;
720
721 queue = &hdev->queues[IFACEQ_DBG_IDX];
722
723 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
724 if (ret)
725 return ret;
726
727 if (tx_req)
728 venus_soft_int(hdev);
729
730 return 0;
731 }
732
venus_iface_dbgq_read(struct venus_hfi_device * hdev,void * pkt)733 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
734 {
735 int ret;
736
737 if (!pkt)
738 return -EINVAL;
739
740 mutex_lock(&hdev->lock);
741 ret = venus_iface_dbgq_read_nolock(hdev, pkt);
742 mutex_unlock(&hdev->lock);
743
744 return ret;
745 }
746
venus_set_qhdr_defaults(struct hfi_queue_header * qhdr)747 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
748 {
749 qhdr->status = 1;
750 qhdr->type = IFACEQ_DFLT_QHDR;
751 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
752 qhdr->pkt_size = 0;
753 qhdr->rx_wm = 1;
754 qhdr->tx_wm = 1;
755 qhdr->rx_req = 1;
756 qhdr->tx_req = 0;
757 qhdr->rx_irq_status = 0;
758 qhdr->tx_irq_status = 0;
759 qhdr->read_idx = 0;
760 qhdr->write_idx = 0;
761 }
762
venus_interface_queues_release(struct venus_hfi_device * hdev)763 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
764 {
765 mutex_lock(&hdev->lock);
766
767 venus_free(hdev, &hdev->ifaceq_table);
768 venus_free(hdev, &hdev->sfr);
769
770 memset(hdev->queues, 0, sizeof(hdev->queues));
771 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
772 memset(&hdev->sfr, 0, sizeof(hdev->sfr));
773
774 mutex_unlock(&hdev->lock);
775 }
776
venus_interface_queues_init(struct venus_hfi_device * hdev)777 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
778 {
779 struct hfi_queue_table_header *tbl_hdr;
780 struct iface_queue *queue;
781 struct hfi_sfr *sfr;
782 struct mem_desc desc = {0};
783 unsigned int offset;
784 unsigned int i;
785 int ret;
786
787 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
788 if (ret)
789 return ret;
790
791 hdev->ifaceq_table = desc;
792 offset = IFACEQ_TABLE_SIZE;
793
794 for (i = 0; i < IFACEQ_NUM; i++) {
795 queue = &hdev->queues[i];
796 queue->qmem.da = desc.da + offset;
797 queue->qmem.kva = desc.kva + offset;
798 queue->qmem.size = IFACEQ_QUEUE_SIZE;
799 offset += queue->qmem.size;
800 queue->qhdr =
801 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
802
803 venus_set_qhdr_defaults(queue->qhdr);
804
805 queue->qhdr->start_addr = queue->qmem.da;
806
807 if (i == IFACEQ_CMD_IDX)
808 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
809 else if (i == IFACEQ_MSG_IDX)
810 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
811 else if (i == IFACEQ_DBG_IDX)
812 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
813 }
814
815 tbl_hdr = hdev->ifaceq_table.kva;
816 tbl_hdr->version = 0;
817 tbl_hdr->size = IFACEQ_TABLE_SIZE;
818 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
819 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
820 tbl_hdr->num_q = IFACEQ_NUM;
821 tbl_hdr->num_active_q = IFACEQ_NUM;
822
823 /*
824 * Set receive request to zero on debug queue as there is no
825 * need of interrupt from video hardware for debug messages
826 */
827 queue = &hdev->queues[IFACEQ_DBG_IDX];
828 queue->qhdr->rx_req = 0;
829
830 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
831 if (ret) {
832 hdev->sfr.da = 0;
833 } else {
834 hdev->sfr = desc;
835 sfr = hdev->sfr.kva;
836 sfr->buf_size = ALIGNED_SFR_SIZE;
837 }
838
839 /* ensure table and queue header structs are settled in memory */
840 wmb();
841
842 return 0;
843 }
844
venus_sys_set_debug(struct venus_hfi_device * hdev,u32 debug)845 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
846 {
847 struct hfi_sys_set_property_pkt *pkt;
848 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
849
850 pkt = (struct hfi_sys_set_property_pkt *)packet;
851
852 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
853
854 return venus_iface_cmdq_write(hdev, pkt, false);
855 }
856
venus_sys_set_coverage(struct venus_hfi_device * hdev,u32 mode)857 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
858 {
859 struct hfi_sys_set_property_pkt *pkt;
860 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
861
862 pkt = (struct hfi_sys_set_property_pkt *)packet;
863
864 pkt_sys_coverage_config(pkt, mode);
865
866 return venus_iface_cmdq_write(hdev, pkt, false);
867 }
868
venus_sys_set_idle_message(struct venus_hfi_device * hdev,bool enable)869 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
870 bool enable)
871 {
872 struct hfi_sys_set_property_pkt *pkt;
873 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
874
875 if (!enable)
876 return 0;
877
878 pkt = (struct hfi_sys_set_property_pkt *)packet;
879
880 pkt_sys_idle_indicator(pkt, enable);
881
882 return venus_iface_cmdq_write(hdev, pkt, false);
883 }
884
venus_sys_set_power_control(struct venus_hfi_device * hdev,bool enable)885 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
886 bool enable)
887 {
888 struct hfi_sys_set_property_pkt *pkt;
889 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
890
891 pkt = (struct hfi_sys_set_property_pkt *)packet;
892
893 pkt_sys_power_control(pkt, enable);
894
895 return venus_iface_cmdq_write(hdev, pkt, false);
896 }
897
venus_sys_set_ubwc_config(struct venus_hfi_device * hdev)898 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
899 {
900 struct hfi_sys_set_property_pkt *pkt;
901 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
902 const struct venus_resources *res = hdev->core->res;
903 int ret;
904
905 pkt = (struct hfi_sys_set_property_pkt *)packet;
906
907 pkt_sys_ubwc_config(pkt, res->ubwc_conf);
908
909 ret = venus_iface_cmdq_write(hdev, pkt, false);
910 if (ret)
911 return ret;
912
913 return 0;
914 }
915
venus_get_queue_size(struct venus_hfi_device * hdev,unsigned int index)916 static int venus_get_queue_size(struct venus_hfi_device *hdev,
917 unsigned int index)
918 {
919 struct hfi_queue_header *qhdr;
920
921 if (index >= IFACEQ_NUM)
922 return -EINVAL;
923
924 qhdr = hdev->queues[index].qhdr;
925 if (!qhdr)
926 return -EINVAL;
927
928 return abs(qhdr->read_idx - qhdr->write_idx);
929 }
930
venus_sys_set_default_properties(struct venus_hfi_device * hdev)931 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
932 {
933 struct device *dev = hdev->core->dev;
934 const struct venus_resources *res = hdev->core->res;
935 int ret;
936
937 ret = venus_sys_set_debug(hdev, venus_fw_debug);
938 if (ret)
939 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
940
941 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */
942 if (IS_V1(hdev->core)) {
943 ret = venus_sys_set_idle_message(hdev, false);
944 if (ret)
945 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
946 }
947
948 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
949 if (ret)
950 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
951 ret);
952
953 /* For specific venus core, it is mandatory to set the UBWC configuration */
954 if (res->ubwc_conf) {
955 ret = venus_sys_set_ubwc_config(hdev);
956 if (ret)
957 dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
958 }
959
960 return ret;
961 }
962
venus_session_cmd(struct venus_inst * inst,u32 pkt_type,bool sync)963 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
964 {
965 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
966 struct hfi_session_pkt pkt;
967
968 pkt_session_cmd(&pkt, pkt_type, inst);
969
970 return venus_iface_cmdq_write(hdev, &pkt, sync);
971 }
972
venus_flush_debug_queue(struct venus_hfi_device * hdev)973 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
974 {
975 struct device *dev = hdev->core->dev;
976 void *packet = hdev->dbg_buf;
977
978 while (!venus_iface_dbgq_read(hdev, packet)) {
979 struct hfi_msg_sys_coverage_pkt *pkt = packet;
980
981 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
982 struct hfi_msg_sys_debug_pkt *pkt = packet;
983
984 dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
985 }
986 }
987 }
988
venus_prepare_power_collapse(struct venus_hfi_device * hdev,bool wait)989 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
990 bool wait)
991 {
992 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
993 struct hfi_sys_pc_prep_pkt pkt;
994 int ret;
995
996 init_completion(&hdev->pwr_collapse_prep);
997
998 pkt_sys_pc_prep(&pkt);
999
1000 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1001 if (ret)
1002 return ret;
1003
1004 if (!wait)
1005 return 0;
1006
1007 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1008 if (!ret) {
1009 venus_flush_debug_queue(hdev);
1010 return -ETIMEDOUT;
1011 }
1012
1013 return 0;
1014 }
1015
venus_are_queues_empty(struct venus_hfi_device * hdev)1016 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1017 {
1018 int ret1, ret2;
1019
1020 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1021 if (ret1 < 0)
1022 return ret1;
1023
1024 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1025 if (ret2 < 0)
1026 return ret2;
1027
1028 if (!ret1 && !ret2)
1029 return 1;
1030
1031 return 0;
1032 }
1033
venus_sfr_print(struct venus_hfi_device * hdev)1034 static void venus_sfr_print(struct venus_hfi_device *hdev)
1035 {
1036 struct device *dev = hdev->core->dev;
1037 struct hfi_sfr *sfr = hdev->sfr.kva;
1038 void *p;
1039
1040 if (!sfr)
1041 return;
1042
1043 p = memchr(sfr->data, '\0', sfr->buf_size);
1044 /*
1045 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1046 * that Venus is in the process of crashing.
1047 */
1048 if (!p)
1049 sfr->data[sfr->buf_size - 1] = '\0';
1050
1051 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1052 }
1053
venus_process_msg_sys_error(struct venus_hfi_device * hdev,void * packet)1054 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1055 void *packet)
1056 {
1057 struct hfi_msg_event_notify_pkt *event_pkt = packet;
1058
1059 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1060 return;
1061
1062 venus_set_state(hdev, VENUS_STATE_DEINIT);
1063
1064 venus_sfr_print(hdev);
1065 }
1066
venus_isr_thread(struct venus_core * core)1067 static irqreturn_t venus_isr_thread(struct venus_core *core)
1068 {
1069 struct venus_hfi_device *hdev = to_hfi_priv(core);
1070 const struct venus_resources *res;
1071 void *pkt;
1072 u32 msg_ret;
1073
1074 if (!hdev)
1075 return IRQ_NONE;
1076
1077 res = hdev->core->res;
1078 pkt = hdev->pkt_buf;
1079
1080
1081 while (!venus_iface_msgq_read(hdev, pkt)) {
1082 msg_ret = hfi_process_msg_packet(core, pkt);
1083 switch (msg_ret) {
1084 case HFI_MSG_EVENT_NOTIFY:
1085 venus_process_msg_sys_error(hdev, pkt);
1086 break;
1087 case HFI_MSG_SYS_INIT:
1088 venus_hfi_core_set_resource(core, res->vmem_id,
1089 res->vmem_size,
1090 res->vmem_addr,
1091 hdev);
1092 break;
1093 case HFI_MSG_SYS_RELEASE_RESOURCE:
1094 complete(&hdev->release_resource);
1095 break;
1096 case HFI_MSG_SYS_PC_PREP:
1097 complete(&hdev->pwr_collapse_prep);
1098 break;
1099 default:
1100 break;
1101 }
1102 }
1103
1104 venus_flush_debug_queue(hdev);
1105
1106 return IRQ_HANDLED;
1107 }
1108
venus_isr(struct venus_core * core)1109 static irqreturn_t venus_isr(struct venus_core *core)
1110 {
1111 struct venus_hfi_device *hdev = to_hfi_priv(core);
1112 u32 status;
1113 void __iomem *cpu_cs_base;
1114 void __iomem *wrapper_base;
1115
1116 if (!hdev)
1117 return IRQ_NONE;
1118
1119 cpu_cs_base = hdev->core->cpu_cs_base;
1120 wrapper_base = hdev->core->wrapper_base;
1121
1122 status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1123 if (IS_IRIS2(core) || IS_IRIS2_1(core)) {
1124 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1125 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1126 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1127 hdev->irq_status = status;
1128 } else {
1129 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1130 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1131 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1132 hdev->irq_status = status;
1133 }
1134 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1135 if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))
1136 writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1137
1138 return IRQ_WAKE_THREAD;
1139 }
1140
venus_core_init(struct venus_core * core)1141 static int venus_core_init(struct venus_core *core)
1142 {
1143 struct venus_hfi_device *hdev = to_hfi_priv(core);
1144 struct device *dev = core->dev;
1145 struct hfi_sys_get_property_pkt version_pkt;
1146 struct hfi_sys_init_pkt pkt;
1147 int ret;
1148
1149 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1150
1151 venus_set_state(hdev, VENUS_STATE_INIT);
1152
1153 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1154 if (ret)
1155 return ret;
1156
1157 pkt_sys_image_version(&version_pkt);
1158
1159 ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1160 if (ret)
1161 dev_warn(dev, "failed to send image version pkt to fw\n");
1162
1163 ret = venus_sys_set_default_properties(hdev);
1164 if (ret)
1165 return ret;
1166
1167 return 0;
1168 }
1169
venus_core_deinit(struct venus_core * core)1170 static int venus_core_deinit(struct venus_core *core)
1171 {
1172 struct venus_hfi_device *hdev = to_hfi_priv(core);
1173
1174 venus_set_state(hdev, VENUS_STATE_DEINIT);
1175 hdev->suspended = true;
1176 hdev->power_enabled = false;
1177
1178 return 0;
1179 }
1180
venus_core_ping(struct venus_core * core,u32 cookie)1181 static int venus_core_ping(struct venus_core *core, u32 cookie)
1182 {
1183 struct venus_hfi_device *hdev = to_hfi_priv(core);
1184 struct hfi_sys_ping_pkt pkt;
1185
1186 pkt_sys_ping(&pkt, cookie);
1187
1188 return venus_iface_cmdq_write(hdev, &pkt, false);
1189 }
1190
venus_core_trigger_ssr(struct venus_core * core,u32 trigger_type)1191 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1192 {
1193 struct venus_hfi_device *hdev = to_hfi_priv(core);
1194 struct hfi_sys_test_ssr_pkt pkt;
1195 int ret;
1196
1197 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1198 if (ret)
1199 return ret;
1200
1201 return venus_iface_cmdq_write(hdev, &pkt, false);
1202 }
1203
venus_session_init(struct venus_inst * inst,u32 session_type,u32 codec)1204 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1205 u32 codec)
1206 {
1207 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1208 struct hfi_session_init_pkt pkt;
1209 int ret;
1210
1211 ret = venus_sys_set_debug(hdev, venus_fw_debug);
1212 if (ret)
1213 goto err;
1214
1215 ret = pkt_session_init(&pkt, inst, session_type, codec);
1216 if (ret)
1217 goto err;
1218
1219 ret = venus_iface_cmdq_write(hdev, &pkt, true);
1220 if (ret)
1221 goto err;
1222
1223 return 0;
1224
1225 err:
1226 venus_flush_debug_queue(hdev);
1227 return ret;
1228 }
1229
venus_session_end(struct venus_inst * inst)1230 static int venus_session_end(struct venus_inst *inst)
1231 {
1232 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1233 struct device *dev = hdev->core->dev;
1234
1235 if (venus_fw_coverage) {
1236 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1237 dev_warn(dev, "fw coverage msg ON failed\n");
1238 }
1239
1240 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1241 }
1242
venus_session_abort(struct venus_inst * inst)1243 static int venus_session_abort(struct venus_inst *inst)
1244 {
1245 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1246
1247 venus_flush_debug_queue(hdev);
1248
1249 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1250 }
1251
venus_session_flush(struct venus_inst * inst,u32 flush_mode)1252 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1253 {
1254 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1255 struct hfi_session_flush_pkt pkt;
1256 int ret;
1257
1258 ret = pkt_session_flush(&pkt, inst, flush_mode);
1259 if (ret)
1260 return ret;
1261
1262 return venus_iface_cmdq_write(hdev, &pkt, true);
1263 }
1264
venus_session_start(struct venus_inst * inst)1265 static int venus_session_start(struct venus_inst *inst)
1266 {
1267 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1268 }
1269
venus_session_stop(struct venus_inst * inst)1270 static int venus_session_stop(struct venus_inst *inst)
1271 {
1272 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1273 }
1274
venus_session_continue(struct venus_inst * inst)1275 static int venus_session_continue(struct venus_inst *inst)
1276 {
1277 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1278 }
1279
venus_session_etb(struct venus_inst * inst,struct hfi_frame_data * in_frame)1280 static int venus_session_etb(struct venus_inst *inst,
1281 struct hfi_frame_data *in_frame)
1282 {
1283 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1284 u32 session_type = inst->session_type;
1285 int ret;
1286
1287 if (session_type == VIDC_SESSION_TYPE_DEC) {
1288 struct hfi_session_empty_buffer_compressed_pkt pkt;
1289
1290 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1291 if (ret)
1292 return ret;
1293
1294 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1295 } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1296 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1297
1298 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1299 if (ret)
1300 return ret;
1301
1302 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1303 } else {
1304 ret = -EINVAL;
1305 }
1306
1307 return ret;
1308 }
1309
venus_session_ftb(struct venus_inst * inst,struct hfi_frame_data * out_frame)1310 static int venus_session_ftb(struct venus_inst *inst,
1311 struct hfi_frame_data *out_frame)
1312 {
1313 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1314 struct hfi_session_fill_buffer_pkt pkt;
1315 int ret;
1316
1317 ret = pkt_session_ftb(&pkt, inst, out_frame);
1318 if (ret)
1319 return ret;
1320
1321 return venus_iface_cmdq_write(hdev, &pkt, false);
1322 }
1323
venus_session_set_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1324 static int venus_session_set_buffers(struct venus_inst *inst,
1325 struct hfi_buffer_desc *bd)
1326 {
1327 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1328 struct hfi_session_set_buffers_pkt *pkt;
1329 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1330 int ret;
1331
1332 if (bd->buffer_type == HFI_BUFFER_INPUT)
1333 return 0;
1334
1335 pkt = (struct hfi_session_set_buffers_pkt *)packet;
1336
1337 ret = pkt_session_set_buffers(pkt, inst, bd);
1338 if (ret)
1339 return ret;
1340
1341 return venus_iface_cmdq_write(hdev, pkt, false);
1342 }
1343
venus_session_unset_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1344 static int venus_session_unset_buffers(struct venus_inst *inst,
1345 struct hfi_buffer_desc *bd)
1346 {
1347 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1348 struct hfi_session_release_buffer_pkt *pkt;
1349 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1350 int ret;
1351
1352 if (bd->buffer_type == HFI_BUFFER_INPUT)
1353 return 0;
1354
1355 pkt = (struct hfi_session_release_buffer_pkt *)packet;
1356
1357 ret = pkt_session_unset_buffers(pkt, inst, bd);
1358 if (ret)
1359 return ret;
1360
1361 return venus_iface_cmdq_write(hdev, pkt, true);
1362 }
1363
venus_session_load_res(struct venus_inst * inst)1364 static int venus_session_load_res(struct venus_inst *inst)
1365 {
1366 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1367 }
1368
venus_session_release_res(struct venus_inst * inst)1369 static int venus_session_release_res(struct venus_inst *inst)
1370 {
1371 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1372 }
1373
venus_session_parse_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1374 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1375 u32 seq_hdr_len)
1376 {
1377 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1378 struct hfi_session_parse_sequence_header_pkt *pkt;
1379 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1380 int ret;
1381
1382 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1383
1384 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1385 if (ret)
1386 return ret;
1387
1388 ret = venus_iface_cmdq_write(hdev, pkt, false);
1389 if (ret)
1390 return ret;
1391
1392 return 0;
1393 }
1394
venus_session_get_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1395 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1396 u32 seq_hdr_len)
1397 {
1398 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1399 struct hfi_session_get_sequence_header_pkt *pkt;
1400 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1401 int ret;
1402
1403 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1404
1405 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1406 if (ret)
1407 return ret;
1408
1409 return venus_iface_cmdq_write(hdev, pkt, false);
1410 }
1411
venus_session_set_property(struct venus_inst * inst,u32 ptype,void * pdata)1412 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1413 void *pdata)
1414 {
1415 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1416 struct hfi_session_set_property_pkt *pkt;
1417 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1418 int ret;
1419
1420 pkt = (struct hfi_session_set_property_pkt *)packet;
1421
1422 ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1423 if (ret == -ENOTSUPP)
1424 return 0;
1425 if (ret)
1426 return ret;
1427
1428 return venus_iface_cmdq_write(hdev, pkt, false);
1429 }
1430
venus_session_get_property(struct venus_inst * inst,u32 ptype)1431 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1432 {
1433 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1434 struct hfi_session_get_property_pkt pkt;
1435 int ret;
1436
1437 ret = pkt_session_get_property(&pkt, inst, ptype);
1438 if (ret)
1439 return ret;
1440
1441 return venus_iface_cmdq_write(hdev, &pkt, true);
1442 }
1443
venus_resume(struct venus_core * core)1444 static int venus_resume(struct venus_core *core)
1445 {
1446 struct venus_hfi_device *hdev = to_hfi_priv(core);
1447 int ret = 0;
1448
1449 mutex_lock(&hdev->lock);
1450
1451 if (!hdev->suspended)
1452 goto unlock;
1453
1454 ret = venus_power_on(hdev);
1455
1456 unlock:
1457 if (!ret)
1458 hdev->suspended = false;
1459
1460 mutex_unlock(&hdev->lock);
1461
1462 return ret;
1463 }
1464
venus_suspend_1xx(struct venus_core * core)1465 static int venus_suspend_1xx(struct venus_core *core)
1466 {
1467 struct venus_hfi_device *hdev = to_hfi_priv(core);
1468 struct device *dev = core->dev;
1469 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1470 u32 ctrl_status;
1471 int ret;
1472
1473 if (!hdev->power_enabled || hdev->suspended)
1474 return 0;
1475
1476 mutex_lock(&hdev->lock);
1477 ret = venus_is_valid_state(hdev);
1478 mutex_unlock(&hdev->lock);
1479
1480 if (!ret) {
1481 dev_err(dev, "bad state, cannot suspend\n");
1482 return -EINVAL;
1483 }
1484
1485 ret = venus_prepare_power_collapse(hdev, true);
1486 if (ret) {
1487 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1488 return ret;
1489 }
1490
1491 mutex_lock(&hdev->lock);
1492
1493 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1494 mutex_unlock(&hdev->lock);
1495 return -EINVAL;
1496 }
1497
1498 ret = venus_are_queues_empty(hdev);
1499 if (ret < 0 || !ret) {
1500 mutex_unlock(&hdev->lock);
1501 return -EINVAL;
1502 }
1503
1504 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1505 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1506 mutex_unlock(&hdev->lock);
1507 return -EINVAL;
1508 }
1509
1510 ret = venus_power_off(hdev);
1511 if (ret) {
1512 mutex_unlock(&hdev->lock);
1513 return ret;
1514 }
1515
1516 hdev->suspended = true;
1517
1518 mutex_unlock(&hdev->lock);
1519
1520 return 0;
1521 }
1522
venus_cpu_and_video_core_idle(struct venus_hfi_device * hdev)1523 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1524 {
1525 void __iomem *wrapper_base = hdev->core->wrapper_base;
1526 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1527 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1528 u32 ctrl_status, cpu_status;
1529
1530 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1531 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1532 else
1533 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1534 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1535
1536 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1537 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1538 return true;
1539
1540 return false;
1541 }
1542
venus_cpu_idle_and_pc_ready(struct venus_hfi_device * hdev)1543 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1544 {
1545 void __iomem *wrapper_base = hdev->core->wrapper_base;
1546 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1547 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1548 u32 ctrl_status, cpu_status;
1549
1550 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1551 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1552 else
1553 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1554 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1555
1556 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1557 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1558 return true;
1559
1560 return false;
1561 }
1562
venus_suspend_3xx(struct venus_core * core)1563 static int venus_suspend_3xx(struct venus_core *core)
1564 {
1565 struct venus_hfi_device *hdev = to_hfi_priv(core);
1566 struct device *dev = core->dev;
1567 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1568 u32 ctrl_status;
1569 bool val;
1570 int ret;
1571
1572 if (!hdev->power_enabled || hdev->suspended)
1573 return 0;
1574
1575 mutex_lock(&hdev->lock);
1576 ret = venus_is_valid_state(hdev);
1577 mutex_unlock(&hdev->lock);
1578
1579 if (!ret) {
1580 dev_err(dev, "bad state, cannot suspend\n");
1581 return -EINVAL;
1582 }
1583
1584 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1585 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1586 goto power_off;
1587
1588 /*
1589 * Power collapse sequence for Venus 3xx and 4xx versions:
1590 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1591 * (bit 0) in CPU status register and by checking Idle (bit 30) in
1592 * Control status register for video core.
1593 * 2. Send a command to prepare for power collapse.
1594 * 3. Check for WFI and PC_READY bits.
1595 */
1596 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1597 1500, 100 * 1500);
1598 if (ret) {
1599 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1600 return ret;
1601 }
1602
1603 ret = venus_prepare_power_collapse(hdev, false);
1604 if (ret) {
1605 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1606 return ret;
1607 }
1608
1609 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1610 1500, 100 * 1500);
1611 if (ret)
1612 return ret;
1613
1614 power_off:
1615 mutex_lock(&hdev->lock);
1616
1617 ret = venus_power_off(hdev);
1618 if (ret) {
1619 dev_err(dev, "venus_power_off (%d)\n", ret);
1620 mutex_unlock(&hdev->lock);
1621 return ret;
1622 }
1623
1624 hdev->suspended = true;
1625
1626 mutex_unlock(&hdev->lock);
1627
1628 return 0;
1629 }
1630
venus_suspend(struct venus_core * core)1631 static int venus_suspend(struct venus_core *core)
1632 {
1633 if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1634 return venus_suspend_3xx(core);
1635
1636 return venus_suspend_1xx(core);
1637 }
1638
1639 static const struct hfi_ops venus_hfi_ops = {
1640 .core_init = venus_core_init,
1641 .core_deinit = venus_core_deinit,
1642 .core_ping = venus_core_ping,
1643 .core_trigger_ssr = venus_core_trigger_ssr,
1644
1645 .session_init = venus_session_init,
1646 .session_end = venus_session_end,
1647 .session_abort = venus_session_abort,
1648 .session_flush = venus_session_flush,
1649 .session_start = venus_session_start,
1650 .session_stop = venus_session_stop,
1651 .session_continue = venus_session_continue,
1652 .session_etb = venus_session_etb,
1653 .session_ftb = venus_session_ftb,
1654 .session_set_buffers = venus_session_set_buffers,
1655 .session_unset_buffers = venus_session_unset_buffers,
1656 .session_load_res = venus_session_load_res,
1657 .session_release_res = venus_session_release_res,
1658 .session_parse_seq_hdr = venus_session_parse_seq_hdr,
1659 .session_get_seq_hdr = venus_session_get_seq_hdr,
1660 .session_set_property = venus_session_set_property,
1661 .session_get_property = venus_session_get_property,
1662
1663 .resume = venus_resume,
1664 .suspend = venus_suspend,
1665
1666 .isr = venus_isr,
1667 .isr_thread = venus_isr_thread,
1668 };
1669
venus_hfi_destroy(struct venus_core * core)1670 void venus_hfi_destroy(struct venus_core *core)
1671 {
1672 struct venus_hfi_device *hdev = to_hfi_priv(core);
1673
1674 core->priv = NULL;
1675 venus_interface_queues_release(hdev);
1676 mutex_destroy(&hdev->lock);
1677 kfree(hdev);
1678 core->ops = NULL;
1679 }
1680
venus_hfi_create(struct venus_core * core)1681 int venus_hfi_create(struct venus_core *core)
1682 {
1683 struct venus_hfi_device *hdev;
1684 int ret;
1685
1686 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1687 if (!hdev)
1688 return -ENOMEM;
1689
1690 mutex_init(&hdev->lock);
1691
1692 hdev->core = core;
1693 hdev->suspended = true;
1694 core->priv = hdev;
1695 core->ops = &venus_hfi_ops;
1696
1697 ret = venus_interface_queues_init(hdev);
1698 if (ret)
1699 goto err_kfree;
1700
1701 return 0;
1702
1703 err_kfree:
1704 kfree(hdev);
1705 core->priv = NULL;
1706 core->ops = NULL;
1707 return ret;
1708 }
1709
venus_hfi_queues_reinit(struct venus_core * core)1710 void venus_hfi_queues_reinit(struct venus_core *core)
1711 {
1712 struct venus_hfi_device *hdev = to_hfi_priv(core);
1713 struct hfi_queue_table_header *tbl_hdr;
1714 struct iface_queue *queue;
1715 struct hfi_sfr *sfr;
1716 unsigned int i;
1717
1718 mutex_lock(&hdev->lock);
1719
1720 for (i = 0; i < IFACEQ_NUM; i++) {
1721 queue = &hdev->queues[i];
1722 queue->qhdr =
1723 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1724
1725 venus_set_qhdr_defaults(queue->qhdr);
1726
1727 queue->qhdr->start_addr = queue->qmem.da;
1728
1729 if (i == IFACEQ_CMD_IDX)
1730 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1731 else if (i == IFACEQ_MSG_IDX)
1732 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1733 else if (i == IFACEQ_DBG_IDX)
1734 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1735 }
1736
1737 tbl_hdr = hdev->ifaceq_table.kva;
1738 tbl_hdr->version = 0;
1739 tbl_hdr->size = IFACEQ_TABLE_SIZE;
1740 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1741 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1742 tbl_hdr->num_q = IFACEQ_NUM;
1743 tbl_hdr->num_active_q = IFACEQ_NUM;
1744
1745 /*
1746 * Set receive request to zero on debug queue as there is no
1747 * need of interrupt from video hardware for debug messages
1748 */
1749 queue = &hdev->queues[IFACEQ_DBG_IDX];
1750 queue->qhdr->rx_req = 0;
1751
1752 sfr = hdev->sfr.kva;
1753 sfr->buf_size = ALIGNED_SFR_SIZE;
1754
1755 /* ensure table and queue header structs are settled in memory */
1756 wmb();
1757
1758 mutex_unlock(&hdev->lock);
1759 }
1760