1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2018 Intel Corporation
3
4 #include <linux/device.h>
5 #include <linux/iopoll.h>
6 #include <linux/slab.h>
7 #include <linux/string_choices.h>
8
9 #include "ipu3.h"
10 #include "ipu3-css.h"
11 #include "ipu3-css-fw.h"
12 #include "ipu3-css-params.h"
13 #include "ipu3-dmamap.h"
14 #include "ipu3-tables.h"
15
16 /* IRQ configuration */
17 #define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \
18 IMGU_IRQCTRL_IRQ_SP2 | \
19 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
20 IMGU_IRQCTRL_IRQ_SW_PIN(1))
21
22 #define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */
23
24 /* Some sane limits for resolutions */
25 #define IPU3_CSS_MIN_RES 32
26 #define IPU3_CSS_MAX_H 3136
27 #define IPU3_CSS_MAX_W 4224
28
29 /* minimal envelope size(GDC in - out) should be 4 */
30 #define MIN_ENVELOPE 4
31
32 /*
33 * pre-allocated buffer size for CSS ABI, auxiliary frames
34 * after BDS and before GDC. Those values should be tuned
35 * to big enough to avoid buffer re-allocation when
36 * streaming to lower streaming latency.
37 */
38 #define CSS_ABI_SIZE 136
39 #define CSS_BDS_SIZE (4480 * 3200 * 3)
40 #define CSS_GDC_SIZE (4224 * 3200 * 12 / 8)
41
42 #define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q))
43 #define IPU3_CSS_FORMAT_FL_IN \
44 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
45 #define IPU3_CSS_FORMAT_FL_OUT \
46 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
47 #define IPU3_CSS_FORMAT_FL_VF \
48 IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
49
50 /* Formats supported by IPU3 Camera Sub System */
51 static const struct imgu_css_format imgu_css_formats[] = {
52 {
53 .pixelformat = V4L2_PIX_FMT_NV12,
54 .colorspace = V4L2_COLORSPACE_SRGB,
55 .frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
56 .osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
57 .osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
58 .chroma_decim = 4,
59 .width_align = IPU3_UAPI_ISP_VEC_ELEMS,
60 .flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
61 }, {
62 /* Each 32 bytes contains 25 10-bit pixels */
63 .pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
64 .colorspace = V4L2_COLORSPACE_RAW,
65 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
66 .bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
67 .bit_depth = 10,
68 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
69 .flags = IPU3_CSS_FORMAT_FL_IN,
70 }, {
71 .pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
72 .colorspace = V4L2_COLORSPACE_RAW,
73 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
74 .bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
75 .bit_depth = 10,
76 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
77 .flags = IPU3_CSS_FORMAT_FL_IN,
78 }, {
79 .pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
80 .colorspace = V4L2_COLORSPACE_RAW,
81 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
82 .bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
83 .bit_depth = 10,
84 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
85 .flags = IPU3_CSS_FORMAT_FL_IN,
86 }, {
87 .pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
88 .colorspace = V4L2_COLORSPACE_RAW,
89 .frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
90 .bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
91 .bit_depth = 10,
92 .width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
93 .flags = IPU3_CSS_FORMAT_FL_IN,
94 },
95 };
96
97 static const struct {
98 enum imgu_abi_queue_id qid;
99 size_t ptr_ofs;
100 } imgu_css_queues[IPU3_CSS_QUEUES] = {
101 [IPU3_CSS_QUEUE_IN] = {
102 IMGU_ABI_QUEUE_C_ID,
103 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
104 },
105 [IPU3_CSS_QUEUE_OUT] = {
106 IMGU_ABI_QUEUE_D_ID,
107 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
108 },
109 [IPU3_CSS_QUEUE_VF] = {
110 IMGU_ABI_QUEUE_E_ID,
111 offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
112 },
113 [IPU3_CSS_QUEUE_STAT_3A] = {
114 IMGU_ABI_QUEUE_F_ID,
115 offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
116 },
117 };
118
119 /* Initialize queue based on given format, adjust format as needed */
imgu_css_queue_init(struct imgu_css_queue * queue,struct v4l2_pix_format_mplane * fmt,u32 flags)120 static int imgu_css_queue_init(struct imgu_css_queue *queue,
121 struct v4l2_pix_format_mplane *fmt, u32 flags)
122 {
123 struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
124 unsigned int i;
125 u32 sizeimage;
126
127 INIT_LIST_HEAD(&queue->bufs);
128
129 queue->css_fmt = NULL; /* Disable */
130 if (!fmt)
131 return 0;
132
133 for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
134 if (!(imgu_css_formats[i].flags & flags))
135 continue;
136 queue->css_fmt = &imgu_css_formats[i];
137 if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
138 break;
139 }
140 if (!queue->css_fmt)
141 return -EINVAL; /* Could not find any suitable format */
142
143 queue->fmt.mpix = *fmt;
144
145 f->width = ALIGN(clamp_t(u32, f->width,
146 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
147 f->height = ALIGN(clamp_t(u32, f->height,
148 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
149 queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
150 f->plane_fmt[0].bytesperline =
151 imgu_bytesperline(f->width, queue->css_fmt->frame_format);
152 sizeimage = f->height * f->plane_fmt[0].bytesperline;
153 if (queue->css_fmt->chroma_decim)
154 sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
155
156 f->plane_fmt[0].sizeimage = sizeimage;
157 f->field = V4L2_FIELD_NONE;
158 f->num_planes = 1;
159 f->colorspace = queue->css_fmt->colorspace;
160 f->flags = 0;
161 f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
162 f->quantization = V4L2_QUANTIZATION_DEFAULT;
163 f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
164 memset(f->reserved, 0, sizeof(f->reserved));
165
166 return 0;
167 }
168
imgu_css_queue_enabled(struct imgu_css_queue * q)169 static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
170 {
171 return q->css_fmt;
172 }
173
174 /******************* css hw *******************/
175
176 /* In the style of writesl() defined in include/asm-generic/io.h */
writes(const void * mem,ssize_t count,void __iomem * addr)177 static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
178 {
179 if (count >= 4) {
180 const u32 *buf = mem;
181
182 count /= 4;
183 do {
184 writel(*buf++, addr);
185 addr += 4;
186 } while (--count);
187 }
188 }
189
190 /* Wait until register `reg', masked with `mask', becomes `cmp' */
imgu_hw_wait(void __iomem * base,int reg,u32 mask,u32 cmp)191 static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
192 {
193 u32 val;
194
195 return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
196 1000, 100 * 1000);
197 }
198
199 /* Initialize the IPU3 CSS hardware and associated h/w blocks */
200
imgu_css_set_powerup(struct device * dev,void __iomem * base,unsigned int freq)201 int imgu_css_set_powerup(struct device *dev, void __iomem *base,
202 unsigned int freq)
203 {
204 u32 pm_ctrl, state, val;
205
206 dev_dbg(dev, "%s with freq %u\n", __func__, freq);
207 /* Clear the CSS busy signal */
208 readl(base + IMGU_REG_GP_BUSY);
209 writel(0, base + IMGU_REG_GP_BUSY);
210
211 /* Wait for idle signal */
212 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
213 IMGU_STATE_IDLE_STS)) {
214 dev_err(dev, "failed to set CSS idle\n");
215 goto fail;
216 }
217
218 /* Reset the css */
219 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
220 base + IMGU_REG_PM_CTRL);
221
222 usleep_range(200, 300);
223
224 /** Prepare CSS */
225
226 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
227 state = readl(base + IMGU_REG_STATE);
228
229 dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
230 pm_ctrl, state, str_down_up(state & IMGU_STATE_POWER_DOWN));
231
232 /* Power up CSS using wrapper */
233 if (state & IMGU_STATE_POWER_DOWN) {
234 writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
235 base + IMGU_REG_PM_CTRL);
236 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
237 IMGU_PM_CTRL_START, 0)) {
238 dev_err(dev, "failed to power up CSS\n");
239 goto fail;
240 }
241 usleep_range(2000, 3000);
242 } else {
243 writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
244 }
245
246 /* Set the busy bit */
247 writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
248
249 /* Set CSS clock frequency */
250 pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
251 val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
252 writel(val, base + IMGU_REG_PM_CTRL);
253 writel(0, base + IMGU_REG_GP_BUSY);
254 if (imgu_hw_wait(base, IMGU_REG_STATE,
255 IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
256 dev_err(dev, "failed to pwrdn CSS\n");
257 goto fail;
258 }
259 val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
260 writel(val, base + IMGU_REG_SYSTEM_REQ);
261 writel(1, base + IMGU_REG_GP_BUSY);
262 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
263 base + IMGU_REG_PM_CTRL);
264 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
265 IMGU_STATE_HALT_STS)) {
266 dev_err(dev, "failed to halt CSS\n");
267 goto fail;
268 }
269
270 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
271 base + IMGU_REG_PM_CTRL);
272 if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
273 dev_err(dev, "failed to start CSS\n");
274 goto fail;
275 }
276 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
277 base + IMGU_REG_PM_CTRL);
278
279 val = readl(base + IMGU_REG_PM_CTRL); /* get pm_ctrl */
280 val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
281 val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
282 writel(val, base + IMGU_REG_PM_CTRL);
283
284 return 0;
285
286 fail:
287 imgu_css_set_powerdown(dev, base);
288 return -EIO;
289 }
290
imgu_css_set_powerdown(struct device * dev,void __iomem * base)291 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
292 {
293 dev_dbg(dev, "%s\n", __func__);
294 /* wait for cio idle signal */
295 if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
296 IMGU_CIO_GATE_BURST_MASK, 0))
297 dev_warn(dev, "wait cio gate idle timeout");
298
299 /* wait for css idle signal */
300 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
301 IMGU_STATE_IDLE_STS))
302 dev_warn(dev, "wait css idle timeout\n");
303
304 /* do halt-halted handshake with css */
305 writel(1, base + IMGU_REG_GP_HALT);
306 if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
307 IMGU_STATE_HALT_STS))
308 dev_warn(dev, "failed to halt css");
309
310 /* de-assert the busy bit */
311 writel(0, base + IMGU_REG_GP_BUSY);
312 }
313
imgu_css_hw_enable_irq(struct imgu_css * css)314 static void imgu_css_hw_enable_irq(struct imgu_css *css)
315 {
316 void __iomem *const base = css->base;
317 u32 val, i;
318
319 /* Set up interrupts */
320
321 /*
322 * Enable IRQ on the SP which signals that SP goes to idle
323 * (aka ready state) and set trigger to pulse
324 */
325 val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
326 writel(val, base + IMGU_REG_SP_CTRL(0));
327 writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
328
329 /* Enable IRQs from the IMGU wrapper */
330 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
331 /* Clear */
332 writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
333
334 /* Enable IRQs from main IRQ controller */
335 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
336 writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
337 writel(IMGU_IRQCTRL_IRQ_MASK,
338 base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
339 writel(IMGU_IRQCTRL_IRQ_MASK,
340 base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
341 writel(IMGU_IRQCTRL_IRQ_MASK,
342 base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
343 writel(IMGU_IRQCTRL_IRQ_MASK,
344 base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
345 /* Wait for write complete */
346 readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
347
348 /* Enable IRQs from SP0 and SP1 controllers */
349 for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
350 writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
351 writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
352 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
353 writel(IMGU_IRQCTRL_IRQ_MASK,
354 base + IMGU_REG_IRQCTRL_ENABLE(i));
355 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
356 writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
357 /* Wait for write complete */
358 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
359 }
360 }
361
imgu_css_hw_init(struct imgu_css * css)362 static int imgu_css_hw_init(struct imgu_css *css)
363 {
364 /* For checking that streaming monitor statuses are valid */
365 static const struct {
366 u32 reg;
367 u32 mask;
368 const char *name;
369 } stream_monitors[] = {
370 {
371 IMGU_REG_GP_SP1_STRMON_STAT,
372 IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
373 "ISP0 to SP0"
374 }, {
375 IMGU_REG_GP_ISP_STRMON_STAT,
376 IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
377 "SP0 to ISP0"
378 }, {
379 IMGU_REG_GP_MOD_STRMON_STAT,
380 IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
381 "ISP0 to DMA0"
382 }, {
383 IMGU_REG_GP_ISP_STRMON_STAT,
384 IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
385 "DMA0 to ISP0"
386 }, {
387 IMGU_REG_GP_MOD_STRMON_STAT,
388 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
389 "ISP0 to GDC0"
390 }, {
391 IMGU_REG_GP_MOD_STRMON_STAT,
392 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
393 "GDC0 to ISP0"
394 }, {
395 IMGU_REG_GP_MOD_STRMON_STAT,
396 IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
397 "SP0 to DMA0"
398 }, {
399 IMGU_REG_GP_SP1_STRMON_STAT,
400 IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
401 "DMA0 to SP0"
402 }, {
403 IMGU_REG_GP_MOD_STRMON_STAT,
404 IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
405 "SP0 to GDC0"
406 }, {
407 IMGU_REG_GP_MOD_STRMON_STAT,
408 IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
409 "GDC0 to SP0"
410 },
411 };
412
413 struct device *dev = css->dev;
414 void __iomem *const base = css->base;
415 u32 val, i;
416
417 /* Set instruction cache address and inv bit for ISP, SP, and SP1 */
418 for (i = 0; i < IMGU_NUM_SP; i++) {
419 struct imgu_fw_info *bi =
420 &css->fwp->binary_header[css->fw_sp[i]];
421
422 writel(css->binary[css->fw_sp[i]].daddr,
423 base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
424 writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
425 IMGU_CTRL_ICACHE_INV,
426 base + IMGU_REG_SP_CTRL(bi->type));
427 }
428 writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
429 writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
430 base + IMGU_REG_ISP_CTRL);
431
432 /* Check that IMGU hardware is ready */
433
434 if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
435 dev_err(dev, "SP is not idle\n");
436 return -EIO;
437 }
438 if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
439 dev_err(dev, "ISP is not idle\n");
440 return -EIO;
441 }
442
443 for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
444 val = readl(base + stream_monitors[i].reg);
445 if (val & stream_monitors[i].mask) {
446 dev_err(dev, "error: Stream monitor %s is valid\n",
447 stream_monitors[i].name);
448 return -EIO;
449 }
450 }
451
452 /* Initialize GDC with default values */
453
454 for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
455 u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
456 u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
457 u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
458 u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
459
460 writel(val0 | (val1 << 16),
461 base + IMGU_REG_GDC_LUT_BASE + i * 8);
462 writel(val2 | (val3 << 16),
463 base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
464 }
465
466 return 0;
467 }
468
469 /* Boot the given IPU3 CSS SP */
imgu_css_hw_start_sp(struct imgu_css * css,int sp)470 static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
471 {
472 void __iomem *const base = css->base;
473 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
474 struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
475 .ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
476 + bi->blob.data_source,
477 .dmem_data_addr = bi->blob.data_target,
478 .dmem_bss_addr = bi->blob.bss_target,
479 .data_size = bi->blob.data_size,
480 .bss_size = bi->blob.bss_size,
481 .sp_id = sp,
482 };
483
484 writes(&dmem_cfg, sizeof(dmem_cfg), base +
485 IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
486
487 writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
488
489 writel(readl(base + IMGU_REG_SP_CTRL(sp))
490 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
491
492 if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
493 + bi->info.sp.sw_state,
494 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
495 return -EIO;
496
497 return 0;
498 }
499
500 /* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
imgu_css_hw_start(struct imgu_css * css)501 static int imgu_css_hw_start(struct imgu_css *css)
502 {
503 static const u32 event_mask =
504 ((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
505 (1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
506 (1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
507 (1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
508 (1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
509 (1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
510 (1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
511 (1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
512 (1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
513 (1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
514 (1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
515 << IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
516
517 void __iomem *const base = css->base;
518 struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
519 unsigned int i;
520
521 writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
522
523 /* Start bootloader */
524
525 writel(IMGU_ABI_BL_SWSTATE_BUSY,
526 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
527 writel(IMGU_NUM_SP,
528 base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
529
530 for (i = 0; i < IMGU_NUM_SP; i++) {
531 int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */
532 struct imgu_fw_info *sp =
533 &css->fwp->binary_header[css->fw_sp[j]];
534 struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
535 .src_addr = css->binary[css->fw_sp[j]].daddr
536 + sp->blob.text_source,
537 .size = sp->blob.text_size,
538 .dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
539 .dst_addr = IMGU_SP_PMEM_BASE(j),
540 };
541
542 writes(&dma_cmd, sizeof(dma_cmd),
543 base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
544 bl->info.bl.dma_cmd_list);
545 }
546
547 writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
548
549 writel(readl(base + IMGU_REG_ISP_CTRL)
550 | IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
551 if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
552 + bl->info.bl.sw_state, ~0,
553 IMGU_ABI_BL_SWSTATE_OK)) {
554 dev_err(css->dev, "failed to start bootloader\n");
555 return -EIO;
556 }
557
558 /* Start ISP */
559
560 memset(css->xmem_sp_group_ptrs.vaddr, 0,
561 sizeof(struct imgu_abi_sp_group));
562
563 bi = &css->fwp->binary_header[css->fw_sp[0]];
564
565 writel(css->xmem_sp_group_ptrs.daddr,
566 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
567
568 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
569 base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
570 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
571
572 if (imgu_css_hw_start_sp(css, 0))
573 return -EIO;
574
575 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
576 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
577 bi->info.sp.host_sp_queues_initialized);
578 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
579 writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
580 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
581 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
582
583 /* Enable all events for all queues */
584
585 for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
586 writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
587 + bi->info.sp.host_sp_com
588 + IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
589 writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
590 bi->info.sp.host_sp_queues_initialized);
591
592 /* Start SP1 */
593
594 bi = &css->fwp->binary_header[css->fw_sp[1]];
595
596 writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
597 base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
598
599 if (imgu_css_hw_start_sp(css, 1))
600 return -EIO;
601
602 writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
603 + bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
604
605 return 0;
606 }
607
imgu_css_hw_stop(struct imgu_css * css)608 static void imgu_css_hw_stop(struct imgu_css *css)
609 {
610 void __iomem *const base = css->base;
611 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
612
613 /* Stop fw */
614 writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
615 base + IMGU_REG_SP_DMEM_BASE(0) +
616 bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
617 if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
618 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
619 dev_err(css->dev, "wait sp0 idle timeout.\n");
620 if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
621 IMGU_ABI_SP_SWSTATE_TERMINATED)
622 dev_err(css->dev, "sp0 is not terminated.\n");
623 if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
624 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
625 dev_err(css->dev, "wait isp idle timeout\n");
626 }
627
imgu_css_hw_cleanup(struct imgu_css * css)628 static void imgu_css_hw_cleanup(struct imgu_css *css)
629 {
630 void __iomem *const base = css->base;
631
632 /** Reset CSS **/
633
634 /* Clear the CSS busy signal */
635 readl(base + IMGU_REG_GP_BUSY);
636 writel(0, base + IMGU_REG_GP_BUSY);
637
638 /* Wait for idle signal */
639 if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
640 IMGU_STATE_IDLE_STS))
641 dev_err(css->dev, "failed to shut down hw cleanly\n");
642
643 /* Reset the css */
644 writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
645 base + IMGU_REG_PM_CTRL);
646
647 usleep_range(200, 300);
648 }
649
imgu_css_pipeline_cleanup(struct imgu_css * css,unsigned int pipe)650 static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
651 {
652 struct imgu_device *imgu = dev_get_drvdata(css->dev);
653 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
654 unsigned int i;
655
656 imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
657 imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
658 imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
659 imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
660
661 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
662 imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
663 }
664
665 /*
666 * This function initializes various stages of the
667 * IPU3 CSS ISP pipeline
668 */
imgu_css_pipeline_init(struct imgu_css * css,unsigned int pipe)669 static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
670 {
671 static const int BYPC = 2; /* Bytes per component */
672 static const struct imgu_abi_buffer_sp buffer_sp_init = {
673 .buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
674 .buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
675 };
676
677 struct imgu_abi_isp_iterator_config *cfg_iter;
678 struct imgu_abi_isp_ref_config *cfg_ref;
679 struct imgu_abi_isp_dvs_config *cfg_dvs;
680 struct imgu_abi_isp_tnr3_config *cfg_tnr;
681 struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
682 struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
683
684 const int stage = 0;
685 unsigned int i, j;
686
687 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
688 struct imgu_css_queue *css_queue_in =
689 &css_pipe->queue[IPU3_CSS_QUEUE_IN];
690 struct imgu_css_queue *css_queue_out =
691 &css_pipe->queue[IPU3_CSS_QUEUE_OUT];
692 struct imgu_css_queue *css_queue_vf =
693 &css_pipe->queue[IPU3_CSS_QUEUE_VF];
694 const struct imgu_fw_info *bi =
695 &css->fwp->binary_header[css_pipe->bindex];
696 const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
697
698 struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
699 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
700 struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
701 bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
702
703 struct imgu_abi_isp_stage *isp_stage;
704 struct imgu_abi_sp_stage *sp_stage;
705 struct imgu_abi_sp_group *sp_group;
706 struct imgu_abi_frames_sp *frames_sp;
707 struct imgu_abi_frame_sp *frame_sp;
708 struct imgu_abi_frame_sp_info *frame_sp_info;
709
710 const unsigned int bds_width_pad =
711 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
712 2 * IPU3_UAPI_ISP_VEC_ELEMS);
713
714 const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
715 enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
716 void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
717
718 struct imgu_device *imgu = dev_get_drvdata(css->dev);
719
720 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
721
722 /* Configure iterator */
723
724 cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
725 &cofs->dmem.iterator,
726 sizeof(*cfg_iter), vaddr);
727 if (!cfg_iter)
728 goto bad_firmware;
729
730 frame_sp_info = &cfg_iter->input_info;
731 frame_sp_info->res.width = css_queue_in->fmt.mpix.width;
732 frame_sp_info->res.height = css_queue_in->fmt.mpix.height;
733 frame_sp_info->padded_width = css_queue_in->width_pad;
734 frame_sp_info->format = css_queue_in->css_fmt->frame_format;
735 frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth;
736 frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order;
737 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
738
739 frame_sp_info = &cfg_iter->internal_info;
740 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
741 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
742 frame_sp_info->padded_width = bds_width_pad;
743 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
744 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
745 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
746 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
747
748 frame_sp_info = &cfg_iter->output_info;
749 frame_sp_info->res.width = css_queue_out->fmt.mpix.width;
750 frame_sp_info->res.height = css_queue_out->fmt.mpix.height;
751 frame_sp_info->padded_width = css_queue_out->width_pad;
752 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
753 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
754 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
755 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
756
757 frame_sp_info = &cfg_iter->vf_info;
758 frame_sp_info->res.width = css_queue_vf->fmt.mpix.width;
759 frame_sp_info->res.height = css_queue_vf->fmt.mpix.height;
760 frame_sp_info->padded_width = css_queue_vf->width_pad;
761 frame_sp_info->format = css_queue_vf->css_fmt->frame_format;
762 frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
763 frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
764 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
765
766 cfg_iter->dvs_envelope.width =
767 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
768 cfg_iter->dvs_envelope.height =
769 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
770
771 /* Configure reference (delay) frames */
772
773 cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
774 &cofs->dmem.ref,
775 sizeof(*cfg_ref), vaddr);
776 if (!cfg_ref)
777 goto bad_firmware;
778
779 cfg_ref->port_b.crop = 0;
780 cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
781 cfg_ref->port_b.width =
782 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
783 cfg_ref->port_b.stride =
784 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
785 cfg_ref->width_a_over_b =
786 IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
787 cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
788 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
789 cfg_ref->ref_frame_addr_y[i] =
790 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
791 cfg_ref->ref_frame_addr_c[i] =
792 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
793 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
794 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
795 }
796 for (; i < IMGU_ABI_FRAMES_REF; i++) {
797 cfg_ref->ref_frame_addr_y[i] = 0;
798 cfg_ref->ref_frame_addr_c[i] = 0;
799 }
800
801 /* Configure DVS (digital video stabilization) */
802
803 cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
804 &cofs->dmem.dvs, sizeof(*cfg_dvs),
805 vaddr);
806 if (!cfg_dvs)
807 goto bad_firmware;
808
809 cfg_dvs->num_horizontal_blocks =
810 ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
811 IMGU_DVS_BLOCK_W), 2);
812 cfg_dvs->num_vertical_blocks =
813 DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
814 IMGU_DVS_BLOCK_H);
815
816 /* Configure TNR (temporal noise reduction) */
817
818 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
819 cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
820 &cofs->dmem.tnr3,
821 sizeof(*cfg_tnr),
822 vaddr);
823 if (!cfg_tnr)
824 goto bad_firmware;
825
826 cfg_tnr->port_b.crop = 0;
827 cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
828 cfg_tnr->port_b.width =
829 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
830 cfg_tnr->port_b.stride =
831 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
832 cfg_tnr->width_a_over_b =
833 IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
834 cfg_tnr->frame_height =
835 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
836 cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
837 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
838 cfg_tnr->frame_addr[i] =
839 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
840 .mem[i].daddr;
841 for (; i < IMGU_ABI_FRAMES_TNR; i++)
842 cfg_tnr->frame_addr[i] = 0;
843 }
844
845 /* Configure ref dmem state parameters */
846
847 cfg = IMGU_ABI_PARAM_CLASS_STATE;
848 vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
849
850 cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
851 &sofs->dmem.ref,
852 sizeof(*cfg_ref_state),
853 vaddr);
854 if (!cfg_ref_state)
855 goto bad_firmware;
856
857 cfg_ref_state->ref_in_buf_idx = 0;
858 cfg_ref_state->ref_out_buf_idx = 1;
859
860 /* Configure tnr dmem state parameters */
861 if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
862 cfg_tnr_state =
863 imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
864 &sofs->dmem.tnr3,
865 sizeof(*cfg_tnr_state),
866 vaddr);
867 if (!cfg_tnr_state)
868 goto bad_firmware;
869
870 cfg_tnr_state->in_bufidx = 0;
871 cfg_tnr_state->out_bufidx = 1;
872 cfg_tnr_state->bypass_filter = 0;
873 cfg_tnr_state->total_frame_counter = 0;
874 for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
875 cfg_tnr_state->buffer_frame_counter[i] = 0;
876 }
877
878 /* Configure ISP stage */
879
880 isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
881 memset(isp_stage, 0, sizeof(*isp_stage));
882 isp_stage->blob_info = bi->blob;
883 isp_stage->binary_info = bi->info.isp.sp;
884 strscpy(isp_stage->binary_name,
885 (char *)css->fwp + bi->blob.prog_name_offset,
886 sizeof(isp_stage->binary_name));
887 isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
888 for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
889 for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
890 isp_stage->mem_initializers.params[i][j].address =
891 css_pipe->binary_params_cs[i - 1][j].daddr;
892
893 /* Configure SP stage */
894
895 sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
896 memset(sp_stage, 0, sizeof(*sp_stage));
897
898 frames_sp = &sp_stage->frames;
899 frames_sp->in.buf_attr = buffer_sp_init;
900 for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
901 frames_sp->out[i].buf_attr = buffer_sp_init;
902 frames_sp->out_vf.buf_attr = buffer_sp_init;
903 frames_sp->s3a_buf = buffer_sp_init;
904 frames_sp->dvs_buf = buffer_sp_init;
905
906 sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
907 sp_stage->num = stage;
908 sp_stage->isp_online = 0;
909 sp_stage->isp_copy_vf = 0;
910 sp_stage->isp_copy_output = 0;
911
912 sp_stage->enable.vf_output = css_pipe->vf_output_en;
913
914 frames_sp->effective_in_res.width =
915 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
916 frames_sp->effective_in_res.height =
917 css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
918
919 frame_sp = &frames_sp->in;
920 frame_sp->info.res.width = css_queue_in->fmt.mpix.width;
921 frame_sp->info.res.height = css_queue_in->fmt.mpix.height;
922 frame_sp->info.padded_width = css_queue_in->width_pad;
923 frame_sp->info.format = css_queue_in->css_fmt->frame_format;
924 frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth;
925 frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order;
926 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
927 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
928 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
929
930 frame_sp = &frames_sp->out[0];
931 frame_sp->info.res.width = css_queue_out->fmt.mpix.width;
932 frame_sp->info.res.height = css_queue_out->fmt.mpix.height;
933 frame_sp->info.padded_width = css_queue_out->width_pad;
934 frame_sp->info.format = css_queue_out->css_fmt->frame_format;
935 frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth;
936 frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order;
937 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
938 frame_sp->planes.nv.uv.offset = css_queue_out->width_pad *
939 css_queue_out->fmt.mpix.height;
940 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
941 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
942
943 frame_sp = &frames_sp->out[1];
944 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
945
946 frame_sp_info = &frames_sp->internal_frame_info;
947 frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
948 frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
949 frame_sp_info->padded_width = bds_width_pad;
950 frame_sp_info->format = css_queue_out->css_fmt->frame_format;
951 frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
952 frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
953 frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
954
955 frame_sp = &frames_sp->out_vf;
956 frame_sp->info.res.width = css_queue_vf->fmt.mpix.width;
957 frame_sp->info.res.height = css_queue_vf->fmt.mpix.height;
958 frame_sp->info.padded_width = css_queue_vf->width_pad;
959 frame_sp->info.format = css_queue_vf->css_fmt->frame_format;
960 frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
961 frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
962 frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
963 frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad *
964 css_queue_vf->fmt.mpix.height;
965 frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad *
966 css_queue_vf->fmt.mpix.height * 5 / 4;
967 frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
968 frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
969
970 frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
971 frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
972
973 frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
974 frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
975
976 sp_stage->dvs_envelope.width =
977 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
978 sp_stage->dvs_envelope.height =
979 css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
980
981 sp_stage->isp_pipe_version =
982 bi->info.isp.sp.pipeline.isp_pipe_version;
983 sp_stage->isp_deci_log_factor =
984 clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
985 IMGU_MAX_BQ_GRID_WIDTH),
986 fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
987 IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
988 sp_stage->isp_vf_downscale_bits = 0;
989 sp_stage->if_config_index = 255;
990 sp_stage->sp_enable_xnr = 0;
991 sp_stage->num_stripes = stripes;
992 sp_stage->enable.s3a = 1;
993 sp_stage->enable.dvs_stats = 0;
994
995 sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
996 sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
997 sp_stage->isp_stage_addr =
998 css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
999
1000 /* Configure SP group */
1001
1002 sp_group = css->xmem_sp_group_ptrs.vaddr;
1003 memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1004
1005 sp_group->pipe[pipe].num_stages = 1;
1006 sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1007 sp_group->pipe[pipe].thread_id = pipe;
1008 sp_group->pipe[pipe].pipe_num = pipe;
1009 sp_group->pipe[pipe].num_execs = -1;
1010 sp_group->pipe[pipe].pipe_qos_config = -1;
1011 sp_group->pipe[pipe].required_bds_factor = 0;
1012 sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1013 sp_group->pipe[pipe].inout_port_config =
1014 IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1015 IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1016 sp_group->pipe[pipe].scaler_pp_lut = 0;
1017 sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1018 sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1019 sp_group->pipe[pipe].sp_stage_addr[stage] =
1020 css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1021 sp_group->pipe[pipe].pipe_config =
1022 bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1023 sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1024
1025 /* Initialize parameter pools */
1026
1027 if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1028 sizeof(struct imgu_abi_parameter_set_info)) ||
1029 imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1030 sizeof(struct imgu_abi_acc_param)) ||
1031 imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1032 sizeof(struct imgu_abi_gdc_warp_param) *
1033 3 * cfg_dvs->num_horizontal_blocks / 2 *
1034 cfg_dvs->num_vertical_blocks) ||
1035 imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1036 imgu_css_fw_obgrid_size(
1037 &css->fwp->binary_header[css_pipe->bindex])))
1038 goto out_of_memory;
1039
1040 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1041 if (imgu_css_pool_init(imgu,
1042 &css_pipe->pool.binary_params_p[i],
1043 bi->info.isp.sp.mem_initializers.params
1044 [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1045 goto out_of_memory;
1046
1047 return 0;
1048
1049 bad_firmware:
1050 imgu_css_pipeline_cleanup(css, pipe);
1051 return -EPROTO;
1052
1053 out_of_memory:
1054 imgu_css_pipeline_cleanup(css, pipe);
1055 return -ENOMEM;
1056 }
1057
imgu_css_queue_pos(struct imgu_css * css,int queue,int thread)1058 static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1059 {
1060 static const unsigned int sp;
1061 void __iomem *const base = css->base;
1062 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1063 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1064 bi->info.sp.host_sp_queue;
1065
1066 return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1067 readb(&q->host2sp_evtq_info.end);
1068 }
1069
1070 /* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
imgu_css_queue_data(struct imgu_css * css,int queue,int thread,u32 data)1071 static int imgu_css_queue_data(struct imgu_css *css,
1072 int queue, int thread, u32 data)
1073 {
1074 static const unsigned int sp;
1075 void __iomem *const base = css->base;
1076 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1077 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1078 bi->info.sp.host_sp_queue;
1079 u8 size, start, end, end2;
1080
1081 if (queue >= 0) {
1082 size = readb(&q->host2sp_bufq_info[thread][queue].size);
1083 start = readb(&q->host2sp_bufq_info[thread][queue].start);
1084 end = readb(&q->host2sp_bufq_info[thread][queue].end);
1085 } else {
1086 size = readb(&q->host2sp_evtq_info.size);
1087 start = readb(&q->host2sp_evtq_info.start);
1088 end = readb(&q->host2sp_evtq_info.end);
1089 }
1090
1091 if (size == 0)
1092 return -EIO;
1093
1094 end2 = (end + 1) % size;
1095 if (end2 == start)
1096 return -EBUSY; /* Queue full */
1097
1098 if (queue >= 0) {
1099 writel(data, &q->host2sp_bufq[thread][queue][end]);
1100 writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1101 } else {
1102 writel(data, &q->host2sp_evtq[end]);
1103 writeb(end2, &q->host2sp_evtq_info.end);
1104 }
1105
1106 return 0;
1107 }
1108
1109 /* Receive data using given buffer queue, or if queue < 0, event queue. */
imgu_css_dequeue_data(struct imgu_css * css,int queue,u32 * data)1110 static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1111 {
1112 static const unsigned int sp;
1113 void __iomem *const base = css->base;
1114 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1115 struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1116 bi->info.sp.host_sp_queue;
1117 u8 size, start, end, start2;
1118
1119 if (queue >= 0) {
1120 size = readb(&q->sp2host_bufq_info[queue].size);
1121 start = readb(&q->sp2host_bufq_info[queue].start);
1122 end = readb(&q->sp2host_bufq_info[queue].end);
1123 } else {
1124 size = readb(&q->sp2host_evtq_info.size);
1125 start = readb(&q->sp2host_evtq_info.start);
1126 end = readb(&q->sp2host_evtq_info.end);
1127 }
1128
1129 if (size == 0)
1130 return -EIO;
1131
1132 if (end == start)
1133 return -EBUSY; /* Queue empty */
1134
1135 start2 = (start + 1) % size;
1136
1137 if (queue >= 0) {
1138 *data = readl(&q->sp2host_bufq[queue][start]);
1139 writeb(start2, &q->sp2host_bufq_info[queue].start);
1140 } else {
1141 int r;
1142
1143 *data = readl(&q->sp2host_evtq[start]);
1144 writeb(start2, &q->sp2host_evtq_info.start);
1145
1146 /* Acknowledge events dequeued from event queue */
1147 r = imgu_css_queue_data(css, queue, 0,
1148 IMGU_ABI_EVENT_EVENT_DEQUEUED);
1149 if (r < 0)
1150 return r;
1151 }
1152
1153 return 0;
1154 }
1155
1156 /* Free binary-specific resources */
imgu_css_binary_cleanup(struct imgu_css * css,unsigned int pipe)1157 static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1158 {
1159 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1160 unsigned int i, j;
1161
1162 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1163
1164 for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1165 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1166 imgu_dmamap_free(imgu,
1167 &css_pipe->binary_params_cs[j][i]);
1168
1169 j = IPU3_CSS_AUX_FRAME_REF;
1170 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1171 imgu_dmamap_free(imgu,
1172 &css_pipe->aux_frames[j].mem[i]);
1173
1174 j = IPU3_CSS_AUX_FRAME_TNR;
1175 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1176 imgu_dmamap_free(imgu,
1177 &css_pipe->aux_frames[j].mem[i]);
1178 }
1179
imgu_css_binary_preallocate(struct imgu_css * css,unsigned int pipe)1180 static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1181 {
1182 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1183 unsigned int i, j;
1184
1185 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1186
1187 for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1188 j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1189 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1190 if (!imgu_dmamap_alloc(imgu,
1191 &css_pipe->binary_params_cs[j - 1][i],
1192 CSS_ABI_SIZE))
1193 goto out_of_memory;
1194
1195 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1196 if (!imgu_dmamap_alloc(imgu,
1197 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1198 CSS_BDS_SIZE))
1199 goto out_of_memory;
1200
1201 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1202 if (!imgu_dmamap_alloc(imgu,
1203 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1204 CSS_GDC_SIZE))
1205 goto out_of_memory;
1206
1207 return 0;
1208
1209 out_of_memory:
1210 imgu_css_binary_cleanup(css, pipe);
1211 return -ENOMEM;
1212 }
1213
1214 /* allocate binary-specific resources */
imgu_css_binary_setup(struct imgu_css * css,unsigned int pipe)1215 static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1216 {
1217 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1218 struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1219 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1220 int i, j, size;
1221 static const int BYPC = 2; /* Bytes per component */
1222 unsigned int w, h;
1223
1224 /* Allocate parameter memory blocks for this binary */
1225
1226 for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1227 for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1228 if (imgu_css_dma_buffer_resize(
1229 imgu,
1230 &css_pipe->binary_params_cs[j - 1][i],
1231 bi->info.isp.sp.mem_initializers.params[j][i].size))
1232 goto out_of_memory;
1233 }
1234
1235 /* Allocate internal frame buffers */
1236
1237 /* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1238 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1239 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1240 css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1241 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1242 ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1243 IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1244 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1245 w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1246 2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1247 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1248 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1249 size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1250 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1251 if (imgu_css_dma_buffer_resize(
1252 imgu,
1253 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1254 size))
1255 goto out_of_memory;
1256
1257 /* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1258 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1259 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1260 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1261 bi->info.isp.sp.block.block_width *
1262 IPU3_UAPI_ISP_VEC_ELEMS);
1263 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1264 roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1265 bi->info.isp.sp.block.output_block_height);
1266
1267 w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1268 css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1269 h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1270 size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */
1271 for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1272 if (imgu_css_dma_buffer_resize(
1273 imgu,
1274 &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1275 size))
1276 goto out_of_memory;
1277
1278 return 0;
1279
1280 out_of_memory:
1281 imgu_css_binary_cleanup(css, pipe);
1282 return -ENOMEM;
1283 }
1284
imgu_css_start_streaming(struct imgu_css * css)1285 int imgu_css_start_streaming(struct imgu_css *css)
1286 {
1287 u32 data;
1288 int r, pipe;
1289
1290 if (css->streaming)
1291 return -EPROTO;
1292
1293 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1294 r = imgu_css_binary_setup(css, pipe);
1295 if (r < 0)
1296 return r;
1297 }
1298
1299 r = imgu_css_hw_init(css);
1300 if (r < 0)
1301 return r;
1302
1303 r = imgu_css_hw_start(css);
1304 if (r < 0)
1305 goto fail;
1306
1307 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1308 r = imgu_css_pipeline_init(css, pipe);
1309 if (r < 0)
1310 goto fail;
1311 }
1312
1313 css->streaming = true;
1314
1315 imgu_css_hw_enable_irq(css);
1316
1317 /* Initialize parameters to default */
1318 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1319 r = imgu_css_set_parameters(css, pipe, NULL);
1320 if (r < 0)
1321 goto fail;
1322 }
1323
1324 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1325 ;
1326 if (r != -EBUSY)
1327 goto fail;
1328
1329 while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1330 ;
1331 if (r != -EBUSY)
1332 goto fail;
1333
1334 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1335 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1336 IMGU_ABI_EVENT_START_STREAM |
1337 pipe << 16);
1338 if (r < 0)
1339 goto fail;
1340 }
1341
1342 return 0;
1343
1344 fail:
1345 css->streaming = false;
1346 imgu_css_hw_cleanup(css);
1347 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1348 imgu_css_pipeline_cleanup(css, pipe);
1349 imgu_css_binary_cleanup(css, pipe);
1350 }
1351
1352 return r;
1353 }
1354
imgu_css_stop_streaming(struct imgu_css * css)1355 void imgu_css_stop_streaming(struct imgu_css *css)
1356 {
1357 struct imgu_css_buffer *b, *b0;
1358 int q, r, pipe;
1359
1360 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1361 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1362 IMGU_ABI_EVENT_STOP_STREAM);
1363 if (r < 0)
1364 dev_warn(css->dev, "failed on stop stream event\n");
1365 }
1366
1367 if (!css->streaming)
1368 return;
1369
1370 imgu_css_hw_stop(css);
1371
1372 imgu_css_hw_cleanup(css);
1373
1374 for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1375 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1376
1377 imgu_css_pipeline_cleanup(css, pipe);
1378
1379 spin_lock(&css_pipe->qlock);
1380 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1381 list_for_each_entry_safe(b, b0,
1382 &css_pipe->queue[q].bufs,
1383 list) {
1384 b->state = IPU3_CSS_BUFFER_FAILED;
1385 list_del(&b->list);
1386 }
1387 spin_unlock(&css_pipe->qlock);
1388 }
1389
1390 css->streaming = false;
1391 }
1392
imgu_css_pipe_queue_empty(struct imgu_css * css,unsigned int pipe)1393 bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1394 {
1395 int q;
1396 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1397
1398 spin_lock(&css_pipe->qlock);
1399 for (q = 0; q < IPU3_CSS_QUEUES; q++)
1400 if (!list_empty(&css_pipe->queue[q].bufs))
1401 break;
1402 spin_unlock(&css_pipe->qlock);
1403 return (q == IPU3_CSS_QUEUES);
1404 }
1405
imgu_css_queue_empty(struct imgu_css * css)1406 bool imgu_css_queue_empty(struct imgu_css *css)
1407 {
1408 unsigned int pipe;
1409 bool ret = false;
1410
1411 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1412 ret &= imgu_css_pipe_queue_empty(css, pipe);
1413
1414 return ret;
1415 }
1416
imgu_css_is_streaming(struct imgu_css * css)1417 bool imgu_css_is_streaming(struct imgu_css *css)
1418 {
1419 return css->streaming;
1420 }
1421
imgu_css_map_init(struct imgu_css * css,unsigned int pipe)1422 static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1423 {
1424 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1425 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1426 unsigned int p, q, i;
1427
1428 /* Allocate and map common structures with imgu hardware */
1429 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1430 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1431 if (!imgu_dmamap_alloc(imgu,
1432 &css_pipe->xmem_sp_stage_ptrs[p][i],
1433 sizeof(struct imgu_abi_sp_stage)))
1434 return -ENOMEM;
1435 if (!imgu_dmamap_alloc(imgu,
1436 &css_pipe->xmem_isp_stage_ptrs[p][i],
1437 sizeof(struct imgu_abi_isp_stage)))
1438 return -ENOMEM;
1439 }
1440
1441 if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1442 ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1443 IMGU_ABI_ISP_DDR_WORD_BYTES)))
1444 return -ENOMEM;
1445
1446 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1447 unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1448
1449 for (i = 0; i < abi_buf_num; i++)
1450 if (!imgu_dmamap_alloc(imgu,
1451 &css_pipe->abi_buffers[q][i],
1452 sizeof(struct imgu_abi_buffer)))
1453 return -ENOMEM;
1454 }
1455
1456 if (imgu_css_binary_preallocate(css, pipe)) {
1457 imgu_css_binary_cleanup(css, pipe);
1458 return -ENOMEM;
1459 }
1460
1461 return 0;
1462 }
1463
imgu_css_pipe_cleanup(struct imgu_css * css,unsigned int pipe)1464 static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1465 {
1466 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1467 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1468 unsigned int p, q, i, abi_buf_num;
1469
1470 imgu_css_binary_cleanup(css, pipe);
1471
1472 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1473 abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1474 for (i = 0; i < abi_buf_num; i++)
1475 imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1476 }
1477
1478 for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1479 for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1480 imgu_dmamap_free(imgu,
1481 &css_pipe->xmem_sp_stage_ptrs[p][i]);
1482 imgu_dmamap_free(imgu,
1483 &css_pipe->xmem_isp_stage_ptrs[p][i]);
1484 }
1485
1486 imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1487 }
1488
imgu_css_cleanup(struct imgu_css * css)1489 void imgu_css_cleanup(struct imgu_css *css)
1490 {
1491 struct imgu_device *imgu = dev_get_drvdata(css->dev);
1492 unsigned int pipe;
1493
1494 imgu_css_stop_streaming(css);
1495 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1496 imgu_css_pipe_cleanup(css, pipe);
1497 imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1498 imgu_css_fw_cleanup(css);
1499 }
1500
imgu_css_init(struct device * dev,struct imgu_css * css,void __iomem * base,int length)1501 int imgu_css_init(struct device *dev, struct imgu_css *css,
1502 void __iomem *base, int length)
1503 {
1504 struct imgu_device *imgu = dev_get_drvdata(dev);
1505 int r, q, pipe;
1506
1507 /* Initialize main data structure */
1508 css->dev = dev;
1509 css->base = base;
1510 css->iomem_length = length;
1511
1512 for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1513 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1514
1515 css_pipe->vf_output_en = false;
1516 spin_lock_init(&css_pipe->qlock);
1517 css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1518 css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1519 for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1520 r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1521 if (r)
1522 return r;
1523 }
1524 r = imgu_css_map_init(css, pipe);
1525 if (r) {
1526 imgu_css_cleanup(css);
1527 return r;
1528 }
1529 }
1530 if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1531 sizeof(struct imgu_abi_sp_group)))
1532 return -ENOMEM;
1533
1534 r = imgu_css_fw_init(css);
1535 if (r)
1536 return r;
1537
1538 return 0;
1539 }
1540
imgu_css_adjust(u32 res,u32 align)1541 static u32 imgu_css_adjust(u32 res, u32 align)
1542 {
1543 u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1544
1545 return DIV_ROUND_CLOSEST(val, align) * align;
1546 }
1547
1548 /* Select a binary matching the required resolutions and formats */
imgu_css_find_binary(struct imgu_css * css,unsigned int pipe,struct imgu_css_queue queue[IPU3_CSS_QUEUES],struct v4l2_rect rects[IPU3_CSS_RECTS])1549 static int imgu_css_find_binary(struct imgu_css *css,
1550 unsigned int pipe,
1551 struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1552 struct v4l2_rect rects[IPU3_CSS_RECTS])
1553 {
1554 const int binary_nr = css->fwp->file_header.binary_nr;
1555 unsigned int binary_mode =
1556 (css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1557 IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1558 const struct v4l2_pix_format_mplane *in =
1559 &queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1560 const struct v4l2_pix_format_mplane *out =
1561 &queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1562 const struct v4l2_pix_format_mplane *vf =
1563 &queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1564 u32 stripe_w = 0, stripe_h = 0;
1565 const char *name;
1566 int i, j;
1567
1568 if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1569 return -EINVAL;
1570
1571 /* Find out the strip size boundary */
1572 for (i = 0; i < binary_nr; i++) {
1573 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1574
1575 u32 max_width = bi->info.isp.sp.output.max_width;
1576 u32 max_height = bi->info.isp.sp.output.max_height;
1577
1578 if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1579 stripe_w = stripe_w ?
1580 min(stripe_w, max_width) : max_width;
1581 stripe_h = stripe_h ?
1582 min(stripe_h, max_height) : max_height;
1583 }
1584 }
1585
1586 for (i = 0; i < binary_nr; i++) {
1587 struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1588 enum imgu_abi_frame_format q_fmt;
1589
1590 name = (void *)css->fwp + bi->blob.prog_name_offset;
1591
1592 /* Check that binary supports memory-to-memory processing */
1593 if (bi->info.isp.sp.input.source !=
1594 IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1595 continue;
1596
1597 /* Check that binary supports raw10 input */
1598 if (!bi->info.isp.sp.enable.input_feeder &&
1599 !bi->info.isp.sp.enable.input_raw)
1600 continue;
1601
1602 /* Check binary mode */
1603 if (bi->info.isp.sp.pipeline.mode != binary_mode)
1604 continue;
1605
1606 /* Since input is RGGB bayer, need to process colors */
1607 if (bi->info.isp.sp.enable.luma_only)
1608 continue;
1609
1610 if (in->width < bi->info.isp.sp.input.min_width ||
1611 in->width > bi->info.isp.sp.input.max_width ||
1612 in->height < bi->info.isp.sp.input.min_height ||
1613 in->height > bi->info.isp.sp.input.max_height)
1614 continue;
1615
1616 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1617 if (bi->info.isp.num_output_pins <= 0)
1618 continue;
1619
1620 q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1621 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1622 if (bi->info.isp.output_formats[j] == q_fmt)
1623 break;
1624 if (j >= bi->info.isp.num_output_formats)
1625 continue;
1626
1627 if (out->width < bi->info.isp.sp.output.min_width ||
1628 out->width > bi->info.isp.sp.output.max_width ||
1629 out->height < bi->info.isp.sp.output.min_height ||
1630 out->height > bi->info.isp.sp.output.max_height)
1631 continue;
1632
1633 if (out->width > bi->info.isp.sp.internal.max_width ||
1634 out->height > bi->info.isp.sp.internal.max_height)
1635 continue;
1636 }
1637
1638 if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1639 if (bi->info.isp.num_output_pins <= 1)
1640 continue;
1641
1642 q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1643 for (j = 0; j < bi->info.isp.num_output_formats; j++)
1644 if (bi->info.isp.output_formats[j] == q_fmt)
1645 break;
1646 if (j >= bi->info.isp.num_output_formats)
1647 continue;
1648
1649 if (vf->width < bi->info.isp.sp.output.min_width ||
1650 vf->width > bi->info.isp.sp.output.max_width ||
1651 vf->height < bi->info.isp.sp.output.min_height ||
1652 vf->height > bi->info.isp.sp.output.max_height)
1653 continue;
1654 }
1655
1656 /* All checks passed, select the binary */
1657 dev_dbg(css->dev, "using binary %s id = %u\n", name,
1658 bi->info.isp.sp.id);
1659 return i;
1660 }
1661
1662 /* Can not find suitable binary for these parameters */
1663 return -EINVAL;
1664 }
1665
1666 /*
1667 * Check that there is a binary matching requirements. Parameters may be
1668 * NULL indicating disabled input/output. Return negative if given
1669 * parameters can not be supported or on error, zero or positive indicating
1670 * found binary number. May modify the given parameters if not exact match
1671 * is found.
1672 */
imgu_css_fmt_try(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1673 int imgu_css_fmt_try(struct imgu_css *css,
1674 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1675 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1676 unsigned int pipe)
1677 {
1678 static const u32 EFF_ALIGN_W = 2;
1679 static const u32 BDS_ALIGN_W = 4;
1680 static const u32 OUT_ALIGN_W = 8;
1681 static const u32 OUT_ALIGN_H = 4;
1682 static const u32 VF_ALIGN_W = 2;
1683 static const char *qnames[IPU3_CSS_QUEUES] = {
1684 [IPU3_CSS_QUEUE_IN] = "in",
1685 [IPU3_CSS_QUEUE_PARAMS] = "params",
1686 [IPU3_CSS_QUEUE_OUT] = "out",
1687 [IPU3_CSS_QUEUE_VF] = "vf",
1688 [IPU3_CSS_QUEUE_STAT_3A] = "3a",
1689 };
1690 static const char *rnames[IPU3_CSS_RECTS] = {
1691 [IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1692 [IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution",
1693 [IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size",
1694 [IPU3_CSS_RECT_GDC] = "GDC output res",
1695 };
1696 struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1697 struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1698 struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1699 struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1700 struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1701 struct imgu_css_queue *q;
1702 struct v4l2_pix_format_mplane *in, *out, *vf;
1703 int i, s, ret;
1704
1705 q = kzalloc_objs(struct imgu_css_queue, IPU3_CSS_QUEUES);
1706 if (!q)
1707 return -ENOMEM;
1708
1709 in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1710 out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1711 vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1712
1713 /* Adjust all formats, get statistics buffer sizes and formats */
1714 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1715 if (fmts[i])
1716 dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1717 qnames[i], fmts[i]->width, fmts[i]->height,
1718 fmts[i]->pixelformat);
1719 else
1720 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1721 qnames[i]);
1722 if (imgu_css_queue_init(&q[i], fmts[i],
1723 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1724 dev_notice(css->dev, "can not initialize queue %s\n",
1725 qnames[i]);
1726 ret = -EINVAL;
1727 goto out;
1728 }
1729 }
1730 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1731 if (rects[i]) {
1732 dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1733 rnames[i], rects[i]->width, rects[i]->height);
1734 r[i].width = rects[i]->width;
1735 r[i].height = rects[i]->height;
1736 } else {
1737 dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1738 rnames[i]);
1739 }
1740 /* For now, force known good resolutions */
1741 r[i].left = 0;
1742 r[i].top = 0;
1743 }
1744
1745 /* Always require one input and vf only if out is also enabled */
1746 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1747 !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1748 dev_warn(css->dev, "required queues are disabled\n");
1749 ret = -EINVAL;
1750 goto out;
1751 }
1752
1753 if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1754 out->width = in->width;
1755 out->height = in->height;
1756 }
1757 if (eff->width <= 0 || eff->height <= 0) {
1758 eff->width = in->width;
1759 eff->height = in->height;
1760 }
1761 if (bds->width <= 0 || bds->height <= 0) {
1762 bds->width = out->width;
1763 bds->height = out->height;
1764 }
1765 if (gdc->width <= 0 || gdc->height <= 0) {
1766 gdc->width = out->width;
1767 gdc->height = out->height;
1768 }
1769
1770 in->width = imgu_css_adjust(in->width, 1);
1771 in->height = imgu_css_adjust(in->height, 1);
1772 eff->width = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1773 eff->height = imgu_css_adjust(eff->height, 1);
1774 bds->width = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1775 bds->height = imgu_css_adjust(bds->height, 1);
1776 gdc->width = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1777 gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1778 out->width = imgu_css_adjust(out->width, OUT_ALIGN_W);
1779 out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1780 vf->width = imgu_css_adjust(vf->width, VF_ALIGN_W);
1781 vf->height = imgu_css_adjust(vf->height, 1);
1782
1783 s = (bds->width - gdc->width) / 2;
1784 env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1785 s = (bds->height - gdc->height) / 2;
1786 env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1787
1788 ret = imgu_css_find_binary(css, pipe, q, r);
1789 if (ret < 0) {
1790 dev_err(css->dev, "failed to find suitable binary\n");
1791 ret = -EINVAL;
1792 goto out;
1793 }
1794 css->pipes[pipe].bindex = ret;
1795
1796 dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1797 css->pipes[pipe].bindex, pipe);
1798
1799 /* Final adjustment and set back the queried formats */
1800 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1801 if (fmts[i]) {
1802 if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1803 IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1804 dev_err(css->dev,
1805 "final resolution adjustment failed\n");
1806 ret = -EINVAL;
1807 goto out;
1808 }
1809 *fmts[i] = q[i].fmt.mpix;
1810 }
1811 }
1812
1813 for (i = 0; i < IPU3_CSS_RECTS; i++)
1814 if (rects[i])
1815 *rects[i] = r[i];
1816
1817 dev_dbg(css->dev,
1818 "in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1819 in->width, in->height, eff->width, eff->height,
1820 bds->width, bds->height, gdc->width, gdc->height,
1821 out->width, out->height, vf->width, vf->height);
1822
1823 ret = 0;
1824 out:
1825 kfree(q);
1826 return ret;
1827 }
1828
imgu_css_fmt_set(struct imgu_css * css,struct v4l2_pix_format_mplane * fmts[IPU3_CSS_QUEUES],struct v4l2_rect * rects[IPU3_CSS_RECTS],unsigned int pipe)1829 int imgu_css_fmt_set(struct imgu_css *css,
1830 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1831 struct v4l2_rect *rects[IPU3_CSS_RECTS],
1832 unsigned int pipe)
1833 {
1834 struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1835 struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1836 int i, r;
1837 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1838
1839 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1840 if (rects[i])
1841 rect_data[i] = *rects[i];
1842 else
1843 memset(&rect_data[i], 0, sizeof(rect_data[i]));
1844 all_rects[i] = &rect_data[i];
1845 }
1846 r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1847 if (r < 0)
1848 return r;
1849
1850 for (i = 0; i < IPU3_CSS_QUEUES; i++)
1851 if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1852 IPU3_CSS_QUEUE_TO_FLAGS(i)))
1853 return -EINVAL;
1854 for (i = 0; i < IPU3_CSS_RECTS; i++) {
1855 css_pipe->rect[i] = rect_data[i];
1856 if (rects[i])
1857 *rects[i] = rect_data[i];
1858 }
1859
1860 return 0;
1861 }
1862
imgu_css_meta_fmt_set(struct v4l2_meta_format * fmt)1863 int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1864 {
1865 switch (fmt->dataformat) {
1866 case V4L2_META_FMT_IPU3_PARAMS:
1867 fmt->buffersize = sizeof(struct ipu3_uapi_params);
1868
1869 /*
1870 * Sanity check for the parameter struct size. This must
1871 * not change!
1872 */
1873 BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1874
1875 break;
1876 case V4L2_META_FMT_IPU3_STAT_3A:
1877 fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1878 break;
1879 default:
1880 return -EINVAL;
1881 }
1882
1883 return 0;
1884 }
1885
1886 /*
1887 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1888 * called for the buffer. May be called from interrupt context.
1889 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1890 * code on error conditions.
1891 */
imgu_css_buf_queue(struct imgu_css * css,unsigned int pipe,struct imgu_css_buffer * b)1892 int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1893 struct imgu_css_buffer *b)
1894 {
1895 struct imgu_abi_buffer *abi_buf;
1896 struct imgu_addr_t *buf_addr;
1897 u32 data;
1898 int r;
1899 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1900
1901 if (!css->streaming)
1902 return -EPROTO; /* CSS or buffer in wrong state */
1903
1904 if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1905 return -EINVAL;
1906
1907 b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1908 pipe);
1909
1910 if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1911 return -EIO;
1912 abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1913
1914 /* Fill struct abi_buffer for firmware */
1915 memset(abi_buf, 0, sizeof(*abi_buf));
1916
1917 buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1918 *(imgu_addr_t *)buf_addr = b->daddr;
1919
1920 if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1921 abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1922
1923 if (b->queue == IPU3_CSS_QUEUE_OUT)
1924 abi_buf->payload.frame.padded_width =
1925 css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1926
1927 if (b->queue == IPU3_CSS_QUEUE_VF)
1928 abi_buf->payload.frame.padded_width =
1929 css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1930
1931 spin_lock(&css_pipe->qlock);
1932 list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1933 spin_unlock(&css_pipe->qlock);
1934 b->state = IPU3_CSS_BUFFER_QUEUED;
1935
1936 data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1937 r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1938 pipe, data);
1939 if (r < 0)
1940 goto queueing_failed;
1941
1942 data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1943 imgu_css_queues[b->queue].qid);
1944 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1945 if (r < 0)
1946 goto queueing_failed;
1947
1948 dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1949 b, b->queue, pipe);
1950
1951 return 0;
1952
1953 queueing_failed:
1954 b->state = (r == -EBUSY || r == -EAGAIN) ?
1955 IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1956 list_del(&b->list);
1957
1958 return r;
1959 }
1960
1961 /*
1962 * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1963 * should be called again, or -EBUSY which means that there are no more
1964 * buffers available. May be called from interrupt context.
1965 */
imgu_css_buf_dequeue(struct imgu_css * css)1966 struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1967 {
1968 static const unsigned char evtype_to_queue[] = {
1969 [IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1970 [IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1971 [IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1972 [IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1973 };
1974 struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1975 u32 event, daddr;
1976 int evtype, pipe, pipeid, queue, qid, r;
1977 struct imgu_css_pipe *css_pipe;
1978
1979 if (!css->streaming)
1980 return ERR_PTR(-EPROTO);
1981
1982 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1983 if (r < 0)
1984 return ERR_PTR(r);
1985
1986 evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1987 IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1988
1989 switch (evtype) {
1990 case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1991 case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1992 case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1993 case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1994 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1995 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1996 pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1997 IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1998 queue = evtype_to_queue[evtype];
1999 qid = imgu_css_queues[queue].qid;
2000
2001 if (pipe >= IMGU_MAX_PIPE_NUM) {
2002 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2003 return ERR_PTR(-EIO);
2004 }
2005
2006 if (qid >= IMGU_ABI_QUEUE_NUM) {
2007 dev_err(css->dev, "Invalid qid: %i\n", qid);
2008 return ERR_PTR(-EIO);
2009 }
2010 css_pipe = &css->pipes[pipe];
2011 dev_dbg(css->dev,
2012 "event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2013 event, queue, pipe, pipeid);
2014
2015 r = imgu_css_dequeue_data(css, qid, &daddr);
2016 if (r < 0) {
2017 dev_err(css->dev, "failed to dequeue buffer\n");
2018 /* Force real error, not -EBUSY */
2019 return ERR_PTR(-EIO);
2020 }
2021
2022 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2023 IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2024 if (r < 0) {
2025 dev_err(css->dev, "failed to queue event\n");
2026 return ERR_PTR(-EIO);
2027 }
2028
2029 spin_lock(&css_pipe->qlock);
2030 if (list_empty(&css_pipe->queue[queue].bufs)) {
2031 spin_unlock(&css_pipe->qlock);
2032 dev_err(css->dev, "event on empty queue\n");
2033 return ERR_PTR(-EIO);
2034 }
2035 b = list_first_entry(&css_pipe->queue[queue].bufs,
2036 struct imgu_css_buffer, list);
2037 if (queue != b->queue ||
2038 daddr != css_pipe->abi_buffers
2039 [b->queue][b->queue_pos].daddr) {
2040 spin_unlock(&css_pipe->qlock);
2041 dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2042 return ERR_PTR(-EIO);
2043 }
2044
2045 dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2046 b->pipe = pipe;
2047 b->state = IPU3_CSS_BUFFER_DONE;
2048 list_del(&b->list);
2049 spin_unlock(&css_pipe->qlock);
2050 break;
2051 case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2052 pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2053 IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2054 if (pipe >= IMGU_MAX_PIPE_NUM) {
2055 dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2056 return ERR_PTR(-EIO);
2057 }
2058
2059 dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2060 event, pipe);
2061 break;
2062 case IMGU_ABI_EVTTYPE_TIMER:
2063 r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2064 if (r < 0)
2065 return ERR_PTR(r);
2066
2067 if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2068 IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2069 dev_dbg(css->dev, "event: timer\n");
2070 else
2071 dev_warn(css->dev, "half of timer event missing\n");
2072 break;
2073 case IMGU_ABI_EVTTYPE_FW_WARNING:
2074 dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2075 break;
2076 case IMGU_ABI_EVTTYPE_FW_ASSERT:
2077 dev_err(css->dev,
2078 "event: firmware assert 0x%x module_id %i line_no %i\n",
2079 event,
2080 (event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2081 IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2082 swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2083 IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2084 break;
2085 default:
2086 dev_warn(css->dev, "received unknown event 0x%x\n", event);
2087 }
2088
2089 return b;
2090 }
2091
2092 /*
2093 * Get a new set of parameters from pool and initialize them based on
2094 * the parameters params, gdc, and obgrid. Any of these may be NULL,
2095 * in which case the previously set parameters are used.
2096 * If parameters haven't been set previously, initialize from scratch.
2097 *
2098 * Return index to css->parameter_set_info which has the newly created
2099 * parameters or negative value on error.
2100 */
imgu_css_set_parameters(struct imgu_css * css,unsigned int pipe,struct ipu3_uapi_params * set_params)2101 int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2102 struct ipu3_uapi_params *set_params)
2103 {
2104 static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2105 struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2106 const int stage = 0;
2107 const struct imgu_fw_info *bi;
2108 int obgrid_size;
2109 unsigned int stripes, i;
2110 struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2111
2112 /* Destination buffers which are filled here */
2113 struct imgu_abi_parameter_set_info *param_set;
2114 struct imgu_abi_acc_param *acc = NULL;
2115 struct imgu_abi_gdc_warp_param *gdc = NULL;
2116 struct ipu3_uapi_obgrid_param *obgrid = NULL;
2117 const struct imgu_css_map *map;
2118 void *vmem0 = NULL;
2119 void *dmem0 = NULL;
2120
2121 enum imgu_abi_memories m;
2122 int r = -EBUSY;
2123
2124 if (!css->streaming)
2125 return -EPROTO;
2126
2127 dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2128
2129 bi = &css->fwp->binary_header[css_pipe->bindex];
2130 obgrid_size = imgu_css_fw_obgrid_size(bi);
2131 stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2132
2133 imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2134 param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2135 0)->vaddr;
2136
2137 /* Get a new acc only if new parameters given, or none yet */
2138 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2139 if (set_params || !map->vaddr) {
2140 imgu_css_pool_get(&css_pipe->pool.acc);
2141 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2142 acc = map->vaddr;
2143 }
2144
2145 /* Get new VMEM0 only if needed, or none yet */
2146 m = IMGU_ABI_MEM_ISP_VMEM0;
2147 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2148 if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2149 set_params->use.tnr3_vmem_params ||
2150 set_params->use.xnr3_vmem_params))) {
2151 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2152 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2153 vmem0 = map->vaddr;
2154 }
2155
2156 /* Get new DMEM0 only if needed, or none yet */
2157 m = IMGU_ABI_MEM_ISP_DMEM0;
2158 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2159 if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2160 set_params->use.xnr3_dmem_params))) {
2161 imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2162 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2163 dmem0 = map->vaddr;
2164 }
2165
2166 /* Configure acc parameter cluster */
2167 if (acc) {
2168 /* get acc_old */
2169 map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2170 /* user acc */
2171 r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2172 set_params ? &set_params->acc_param : NULL);
2173 if (r < 0)
2174 goto fail;
2175 }
2176
2177 /* Configure late binding parameters */
2178 if (vmem0) {
2179 m = IMGU_ABI_MEM_ISP_VMEM0;
2180 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2181 r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2182 map->vaddr, set_params);
2183 if (r < 0)
2184 goto fail;
2185 }
2186
2187 if (dmem0) {
2188 m = IMGU_ABI_MEM_ISP_DMEM0;
2189 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2190 r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2191 map->vaddr, set_params);
2192 if (r < 0)
2193 goto fail;
2194 }
2195
2196 /* Get a new gdc only if a new gdc is given, or none yet */
2197 if (bi->info.isp.sp.enable.dvs_6axis) {
2198 unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2199 unsigned int g = IPU3_CSS_RECT_GDC;
2200 unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2201
2202 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2203 if (!map->vaddr) {
2204 imgu_css_pool_get(&css_pipe->pool.gdc);
2205 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2206 gdc = map->vaddr;
2207 imgu_css_cfg_gdc_table(map->vaddr,
2208 css_pipe->aux_frames[a].bytesperline /
2209 css_pipe->aux_frames[a].bytesperpixel,
2210 css_pipe->aux_frames[a].height,
2211 css_pipe->rect[g].width,
2212 css_pipe->rect[g].height,
2213 css_pipe->rect[e].width,
2214 css_pipe->rect[e].height);
2215 }
2216 }
2217
2218 /* Get a new obgrid only if a new obgrid is given, or none yet */
2219 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2220 if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2221 imgu_css_pool_get(&css_pipe->pool.obgrid);
2222 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2223 obgrid = map->vaddr;
2224
2225 /* Configure optical black level grid (obgrid) */
2226 if (set_params && set_params->use.obgrid_param)
2227 for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2228 obgrid[i] = set_params->obgrid_param;
2229 else
2230 memset(obgrid, 0, obgrid_size);
2231 }
2232
2233 /* Configure parameter set info, queued to `queue_id' */
2234
2235 memset(param_set, 0, sizeof(*param_set));
2236 map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2237 param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2238
2239 map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2240 param_set->mem_map.dvs_6axis_params_y = map->daddr;
2241
2242 for (i = 0; i < stripes; i++) {
2243 map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2244 param_set->mem_map.obgrid_tbl[i] =
2245 map->daddr + (obgrid_size / stripes) * i;
2246 }
2247
2248 for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2249 map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2250 param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2251 }
2252
2253 /* Then queue the new parameter buffer */
2254 map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2255 r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2256 if (r < 0)
2257 goto fail;
2258
2259 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2260 IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2261 queue_id));
2262 if (r < 0)
2263 goto fail_no_put;
2264
2265 /* Finally dequeue all old parameter buffers */
2266
2267 do {
2268 u32 daddr;
2269
2270 r = imgu_css_dequeue_data(css, queue_id, &daddr);
2271 if (r == -EBUSY)
2272 break;
2273 if (r)
2274 goto fail_no_put;
2275 r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2276 IMGU_ABI_EVENT_BUFFER_DEQUEUED
2277 (queue_id));
2278 if (r < 0) {
2279 dev_err(css->dev, "failed to queue parameter event\n");
2280 goto fail_no_put;
2281 }
2282 } while (1);
2283
2284 return 0;
2285
2286 fail:
2287 /*
2288 * A failure, most likely the parameter queue was full.
2289 * Return error but continue streaming. User can try submitting new
2290 * parameters again later.
2291 */
2292
2293 imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2294 if (acc)
2295 imgu_css_pool_put(&css_pipe->pool.acc);
2296 if (gdc)
2297 imgu_css_pool_put(&css_pipe->pool.gdc);
2298 if (obgrid)
2299 imgu_css_pool_put(&css_pipe->pool.obgrid);
2300 if (vmem0)
2301 imgu_css_pool_put(
2302 &css_pipe->pool.binary_params_p
2303 [IMGU_ABI_MEM_ISP_VMEM0]);
2304 if (dmem0)
2305 imgu_css_pool_put(
2306 &css_pipe->pool.binary_params_p
2307 [IMGU_ABI_MEM_ISP_DMEM0]);
2308
2309 fail_no_put:
2310 return r;
2311 }
2312
imgu_css_irq_ack(struct imgu_css * css)2313 int imgu_css_irq_ack(struct imgu_css *css)
2314 {
2315 static const int NUM_SWIRQS = 3;
2316 struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2317 void __iomem *const base = css->base;
2318 u32 irq_status[IMGU_IRQCTRL_NUM];
2319 int i;
2320
2321 u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2322
2323 writel(imgu_status, base + IMGU_REG_INT_STATUS);
2324 for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2325 irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2326
2327 for (i = 0; i < NUM_SWIRQS; i++) {
2328 if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2329 /* SP SW interrupt */
2330 u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2331 bi->info.sp.output);
2332 u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333 bi->info.sp.output + 4 + 4 * i);
2334
2335 dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2336 __func__, i, cnt, val);
2337 }
2338 }
2339
2340 for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2341 if (irq_status[i]) {
2342 writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2343 /* Wait for write to complete */
2344 readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2345 }
2346
2347 dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2348 __func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2349 irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2350
2351 if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2352 return -ENOMSG;
2353
2354 return 0;
2355 }
2356