xref: /linux/drivers/media/platform/raspberrypi/pisp_be/pisp_be.c (revision fcb3ad4366b9c810cbb9da34c076a9a52d8aa1e0)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PiSP Back End driver.
4  * Copyright (c) 2021-2024 Raspberry Pi Limited.
5  *
6  */
7 #include <linux/clk.h>
8 #include <linux/interrupt.h>
9 #include <linux/io.h>
10 #include <linux/kernel.h>
11 #include <linux/lockdep.h>
12 #include <linux/module.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15 #include <media/v4l2-device.h>
16 #include <media/v4l2-ioctl.h>
17 #include <media/videobuf2-dma-contig.h>
18 #include <media/videobuf2-vmalloc.h>
19 
20 #include <uapi/linux/media/raspberrypi/pisp_be_config.h>
21 
22 #include "pisp_be_formats.h"
23 
24 /* Maximum number of config buffers possible */
25 #define PISP_BE_NUM_CONFIG_BUFFERS VB2_MAX_FRAME
26 
27 #define PISPBE_NAME "pispbe"
28 
29 /* Some ISP-BE registers */
30 #define PISP_BE_VERSION_REG		0x0
31 #define PISP_BE_CONTROL_REG		0x4
32 #define PISP_BE_CONTROL_COPY_CONFIG	BIT(1)
33 #define PISP_BE_CONTROL_QUEUE_JOB	BIT(0)
34 #define PISP_BE_CONTROL_NUM_TILES(n)	((n) << 16)
35 #define PISP_BE_TILE_ADDR_LO_REG	0x8
36 #define PISP_BE_TILE_ADDR_HI_REG	0xc
37 #define PISP_BE_STATUS_REG		0x10
38 #define PISP_BE_STATUS_QUEUED		BIT(0)
39 #define PISP_BE_BATCH_STATUS_REG	0x14
40 #define PISP_BE_INTERRUPT_EN_REG	0x18
41 #define PISP_BE_INTERRUPT_STATUS_REG	0x1c
42 #define PISP_BE_AXI_REG			0x20
43 #define PISP_BE_CONFIG_BASE_REG		0x40
44 #define PISP_BE_IO_ADDR_LOW(n)		(PISP_BE_CONFIG_BASE_REG + 8 * (n))
45 #define PISP_BE_IO_ADDR_HIGH(n)		(PISP_BE_IO_ADDR_LOW((n)) + 4)
46 #define PISP_BE_GLOBAL_BAYER_ENABLE	0xb0
47 #define PISP_BE_GLOBAL_RGB_ENABLE	0xb4
48 #define N_HW_ADDRESSES			13
49 #define N_HW_ENABLES			2
50 
51 #define PISP_BE_VERSION_2712		0x02252700
52 #define PISP_BE_VERSION_MINOR_BITS	0xf
53 
54 /*
55  * This maps our nodes onto the inputs/outputs of the actual PiSP Back End.
56  * Be wary of the word "OUTPUT" which is used ambiguously here. In a V4L2
57  * context it means an input to the hardware (source image or metadata).
58  * Elsewhere it means an output from the hardware.
59  */
60 enum pispbe_node_ids {
61 	MAIN_INPUT_NODE,
62 	TDN_INPUT_NODE,
63 	STITCH_INPUT_NODE,
64 	OUTPUT0_NODE,
65 	OUTPUT1_NODE,
66 	TDN_OUTPUT_NODE,
67 	STITCH_OUTPUT_NODE,
68 	CONFIG_NODE,
69 	PISPBE_NUM_NODES
70 };
71 
72 struct pispbe_node_description {
73 	const char *ent_name;
74 	enum v4l2_buf_type buf_type;
75 	unsigned int caps;
76 };
77 
78 static const struct pispbe_node_description node_desc[PISPBE_NUM_NODES] = {
79 	/* MAIN_INPUT_NODE */
80 	{
81 		.ent_name = PISPBE_NAME "-input",
82 		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
83 		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
84 	},
85 	/* TDN_INPUT_NODE */
86 	{
87 		.ent_name = PISPBE_NAME "-tdn_input",
88 		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
89 		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
90 	},
91 	/* STITCH_INPUT_NODE */
92 	{
93 		.ent_name = PISPBE_NAME "-stitch_input",
94 		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
95 		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
96 	},
97 	/* OUTPUT0_NODE */
98 	{
99 		.ent_name = PISPBE_NAME "-output0",
100 		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
101 		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
102 	},
103 	/* OUTPUT1_NODE */
104 	{
105 		.ent_name = PISPBE_NAME "-output1",
106 		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
107 		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
108 	},
109 	/* TDN_OUTPUT_NODE */
110 	{
111 		.ent_name = PISPBE_NAME "-tdn_output",
112 		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
113 		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
114 	},
115 	/* STITCH_OUTPUT_NODE */
116 	{
117 		.ent_name = PISPBE_NAME "-stitch_output",
118 		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
119 		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
120 	},
121 	/* CONFIG_NODE */
122 	{
123 		.ent_name = PISPBE_NAME "-config",
124 		.buf_type = V4L2_BUF_TYPE_META_OUTPUT,
125 		.caps = V4L2_CAP_META_OUTPUT,
126 	}
127 };
128 
129 #define NODE_DESC_IS_OUTPUT(desc) ( \
130 	((desc)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \
131 	((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \
132 	((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))
133 
134 #define NODE_IS_META(node) ( \
135 	((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT))
136 #define NODE_IS_OUTPUT(node) ( \
137 	((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \
138 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \
139 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))
140 #define NODE_IS_CAPTURE(node) ( \
141 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) || \
142 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))
143 #define NODE_IS_MPLANE(node) ( \
144 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) || \
145 	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))
146 
147 /*
148  * Structure to describe a single node /dev/video<N> which represents a single
149  * input or output queue to the PiSP Back End device.
150  */
151 struct pispbe_node {
152 	unsigned int id;
153 	int vfl_dir;
154 	enum v4l2_buf_type buf_type;
155 	struct video_device vfd;
156 	struct media_pad pad;
157 	struct media_intf_devnode *intf_devnode;
158 	struct media_link *intf_link;
159 	struct pispbe_dev *pispbe;
160 	/* Video device lock */
161 	struct mutex node_lock;
162 	/* vb2_queue lock */
163 	struct mutex queue_lock;
164 	/* Protect pispbe_node->ready_queue and pispbe_buffer->ready_list */
165 	spinlock_t ready_lock;
166 	struct list_head ready_queue;
167 	struct vb2_queue queue;
168 	struct v4l2_format format;
169 	const struct pisp_be_format *pisp_format;
170 };
171 
172 /* For logging only, use the entity name with "pispbe" and separator removed */
173 #define NODE_NAME(node) \
174 		(node_desc[(node)->id].ent_name + sizeof(PISPBE_NAME))
175 
176 /* Records details of the jobs currently running or queued on the h/w. */
177 struct pispbe_job {
178 	bool valid;
179 	/*
180 	 * An array of buffer pointers - remember it's source buffers first,
181 	 * then captures, then metadata last.
182 	 */
183 	struct pispbe_buffer *buf[PISPBE_NUM_NODES];
184 };
185 
186 struct pispbe_hw_enables {
187 	u32 bayer_enables;
188 	u32 rgb_enables;
189 };
190 
191 /* Records a job configuration and memory addresses. */
192 struct pispbe_job_descriptor {
193 	dma_addr_t hw_dma_addrs[N_HW_ADDRESSES];
194 	struct pisp_be_tiles_config *config;
195 	struct pispbe_hw_enables hw_enables;
196 	dma_addr_t tiles;
197 };
198 
199 /*
200  * Structure representing the entire PiSP Back End device, comprising several
201  * nodes which share platform resources and a mutex for the actual HW.
202  */
203 struct pispbe_dev {
204 	struct device *dev;
205 	struct pispbe_dev *pispbe;
206 	struct pisp_be_tiles_config *config;
207 	void __iomem *be_reg_base;
208 	struct clk *clk;
209 	struct v4l2_device v4l2_dev;
210 	struct v4l2_subdev sd;
211 	struct media_device mdev;
212 	struct media_pad pad[PISPBE_NUM_NODES]; /* output pads first */
213 	struct pispbe_node node[PISPBE_NUM_NODES];
214 	dma_addr_t config_dma_addr;
215 	unsigned int sequence;
216 	u32 streaming_map;
217 	struct pispbe_job queued_job, running_job;
218 	spinlock_t hw_lock; /* protects "hw_busy" flag and streaming_map */
219 	bool hw_busy; /* non-zero if a job is queued or is being started */
220 	int irq;
221 	u32 hw_version;
222 	u8 done, started;
223 };
224 
225 static u32 pispbe_rd(struct pispbe_dev *pispbe, unsigned int offset)
226 {
227 	return readl(pispbe->be_reg_base + offset);
228 }
229 
230 static void pispbe_wr(struct pispbe_dev *pispbe, unsigned int offset, u32 val)
231 {
232 	writel(val, pispbe->be_reg_base + offset);
233 }
234 
235 /*
236  * Queue a job to the h/w. If the h/w is idle it will begin immediately.
237  * Caller must ensure it is "safe to queue", i.e. we don't already have a
238  * queued, unstarted job.
239  */
240 static void pispbe_queue_job(struct pispbe_dev *pispbe,
241 			     struct pispbe_job_descriptor *job)
242 {
243 	unsigned int begin, end;
244 
245 	if (pispbe_rd(pispbe, PISP_BE_STATUS_REG) & PISP_BE_STATUS_QUEUED)
246 		dev_err(pispbe->dev, "ERROR: not safe to queue new job!\n");
247 
248 	/*
249 	 * Write configuration to hardware. DMA addresses and enable flags
250 	 * are passed separately, because the driver needs to sanitize them,
251 	 * and we don't want to modify (or be vulnerable to modifications of)
252 	 * the mmap'd buffer.
253 	 */
254 	for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {
255 		pispbe_wr(pispbe, PISP_BE_IO_ADDR_LOW(u),
256 			  lower_32_bits(job->hw_dma_addrs[u]));
257 		pispbe_wr(pispbe, PISP_BE_IO_ADDR_HIGH(u),
258 			  upper_32_bits(job->hw_dma_addrs[u]));
259 	}
260 	pispbe_wr(pispbe, PISP_BE_GLOBAL_BAYER_ENABLE,
261 		  job->hw_enables.bayer_enables);
262 	pispbe_wr(pispbe, PISP_BE_GLOBAL_RGB_ENABLE,
263 		  job->hw_enables.rgb_enables);
264 
265 	/* Everything else is as supplied by the user. */
266 	begin =	offsetof(struct pisp_be_config, global.bayer_order) /
267 		sizeof(u32);
268 	end = sizeof(struct pisp_be_config) / sizeof(u32);
269 	for (unsigned int u = begin; u < end; u++)
270 		pispbe_wr(pispbe, PISP_BE_CONFIG_BASE_REG + sizeof(u32) * u,
271 			  ((u32 *)job->config)[u]);
272 
273 	/* Read back the addresses -- an error here could be fatal */
274 	for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {
275 		unsigned int offset = PISP_BE_IO_ADDR_LOW(u);
276 		u64 along = pispbe_rd(pispbe, offset);
277 
278 		along += ((u64)pispbe_rd(pispbe, offset + 4)) << 32;
279 		if (along != (u64)(job->hw_dma_addrs[u])) {
280 			dev_dbg(pispbe->dev,
281 				"ISP BE config error: check if ISP RAMs enabled?\n");
282 			return;
283 		}
284 	}
285 
286 	/*
287 	 * Write tile pointer to hardware. The IOMMU should prevent
288 	 * out-of-bounds offsets reaching non-ISP buffers.
289 	 */
290 	pispbe_wr(pispbe, PISP_BE_TILE_ADDR_LO_REG, lower_32_bits(job->tiles));
291 	pispbe_wr(pispbe, PISP_BE_TILE_ADDR_HI_REG, upper_32_bits(job->tiles));
292 
293 	/* Enqueue the job */
294 	pispbe_wr(pispbe, PISP_BE_CONTROL_REG,
295 		  PISP_BE_CONTROL_COPY_CONFIG | PISP_BE_CONTROL_QUEUE_JOB |
296 		  PISP_BE_CONTROL_NUM_TILES(job->config->num_tiles));
297 }
298 
299 struct pispbe_buffer {
300 	struct vb2_v4l2_buffer vb;
301 	struct list_head ready_list;
302 	unsigned int config_index;
303 };
304 
305 static int pispbe_get_planes_addr(dma_addr_t addr[3], struct pispbe_buffer *buf,
306 				  struct pispbe_node *node)
307 {
308 	unsigned int num_planes = node->format.fmt.pix_mp.num_planes;
309 	unsigned int plane_factor = 0;
310 	unsigned int size;
311 	unsigned int p;
312 
313 	if (!buf || !node->pisp_format)
314 		return 0;
315 
316 	/*
317 	 * Determine the base plane size. This will not be the same
318 	 * as node->format.fmt.pix_mp.plane_fmt[0].sizeimage for a single
319 	 * plane buffer in an mplane format.
320 	 */
321 	size = node->format.fmt.pix_mp.plane_fmt[0].bytesperline *
322 	       node->format.fmt.pix_mp.height;
323 
324 	for (p = 0; p < num_planes && p < PISPBE_MAX_PLANES; p++) {
325 		addr[p] = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, p);
326 		plane_factor += node->pisp_format->plane_factor[p];
327 	}
328 
329 	for (; p < PISPBE_MAX_PLANES && node->pisp_format->plane_factor[p]; p++) {
330 		/*
331 		 * Calculate the address offset of this plane as needed
332 		 * by the hardware. This is specifically for non-mplane
333 		 * buffer formats, where there are 3 image planes, e.g.
334 		 * for the V4L2_PIX_FMT_YUV420 format.
335 		 */
336 		addr[p] = addr[0] + ((size * plane_factor) >> 3);
337 		plane_factor += node->pisp_format->plane_factor[p];
338 	}
339 
340 	return num_planes;
341 }
342 
343 static dma_addr_t pispbe_get_addr(struct pispbe_buffer *buf)
344 {
345 	if (buf)
346 		return vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
347 
348 	return 0;
349 }
350 
351 static void pispbe_xlate_addrs(struct pispbe_dev *pispbe,
352 			       struct pispbe_job_descriptor *job,
353 			       struct pispbe_buffer *buf[PISPBE_NUM_NODES])
354 {
355 	struct pispbe_hw_enables *hw_en = &job->hw_enables;
356 	struct pisp_be_tiles_config *config = job->config;
357 	dma_addr_t *addrs = job->hw_dma_addrs;
358 	int ret;
359 
360 	/* Take a copy of the "enable" bitmaps so we can modify them. */
361 	hw_en->bayer_enables = config->config.global.bayer_enables;
362 	hw_en->rgb_enables = config->config.global.rgb_enables;
363 
364 	/*
365 	 * Main input first. There are 3 address pointers, corresponding to up
366 	 * to 3 planes.
367 	 */
368 	ret = pispbe_get_planes_addr(addrs, buf[MAIN_INPUT_NODE],
369 				     &pispbe->node[MAIN_INPUT_NODE]);
370 	if (ret <= 0) {
371 		/*
372 		 * This shouldn't happen; pispbe_schedule_internal should insist
373 		 * on an input.
374 		 */
375 		dev_warn(pispbe->dev, "ISP-BE missing input\n");
376 		hw_en->bayer_enables = 0;
377 		hw_en->rgb_enables = 0;
378 		return;
379 	}
380 
381 	/*
382 	 * Now TDN/Stitch inputs and outputs. These are single-plane and only
383 	 * used with Bayer input. Input enables must match the requirements
384 	 * of the processing stages, otherwise the hardware can lock up!
385 	 */
386 	if (hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) {
387 		addrs[3] = pispbe_get_addr(buf[TDN_INPUT_NODE]);
388 		if (addrs[3] == 0 ||
389 		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN_INPUT) ||
390 		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN) ||
391 		    (config->config.tdn.reset & 1)) {
392 			hw_en->bayer_enables &=
393 				~(PISP_BE_BAYER_ENABLE_TDN_INPUT |
394 				  PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS);
395 			if (!(config->config.tdn.reset & 1))
396 				hw_en->bayer_enables &=
397 					~PISP_BE_BAYER_ENABLE_TDN;
398 		}
399 
400 		addrs[4] = pispbe_get_addr(buf[STITCH_INPUT_NODE]);
401 		if (addrs[4] == 0 ||
402 		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_INPUT) ||
403 		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH)) {
404 			hw_en->bayer_enables &=
405 				~(PISP_BE_BAYER_ENABLE_STITCH_INPUT |
406 				  PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS |
407 				  PISP_BE_BAYER_ENABLE_STITCH);
408 		}
409 
410 		addrs[5] = pispbe_get_addr(buf[TDN_OUTPUT_NODE]);
411 		if (addrs[5] == 0)
412 			hw_en->bayer_enables &=
413 				~(PISP_BE_BAYER_ENABLE_TDN_COMPRESS |
414 				  PISP_BE_BAYER_ENABLE_TDN_OUTPUT);
415 
416 		addrs[6] = pispbe_get_addr(buf[STITCH_OUTPUT_NODE]);
417 		if (addrs[6] == 0)
418 			hw_en->bayer_enables &=
419 				~(PISP_BE_BAYER_ENABLE_STITCH_COMPRESS |
420 				  PISP_BE_BAYER_ENABLE_STITCH_OUTPUT);
421 	} else {
422 		/* No Bayer input? Disable entire Bayer pipe (else lockup) */
423 		hw_en->bayer_enables = 0;
424 	}
425 
426 	/* Main image output channels. */
427 	for (unsigned int i = 0; i < PISP_BACK_END_NUM_OUTPUTS; i++) {
428 		ret = pispbe_get_planes_addr(addrs + 7 + 3 * i,
429 					     buf[OUTPUT0_NODE + i],
430 					     &pispbe->node[OUTPUT0_NODE + i]);
431 		if (ret <= 0)
432 			hw_en->rgb_enables &= ~(PISP_BE_RGB_ENABLE_OUTPUT0 << i);
433 	}
434 }
435 
436 /*
437  * Prepare a job description to be submitted to the HW.
438  *
439  * To schedule a job, we need all streaming nodes (apart from Output0,
440  * Output1, Tdn and Stitch) to have a buffer ready, which must
441  * include at least a config buffer and a main input image.
442  *
443  * For Output0, Output1, Tdn and Stitch, a buffer only needs to be
444  * available if the blocks are enabled in the config.
445  *
446  * Needs to be called with hw_lock held.
447  *
448  * Returns 0 if a job has been successfully prepared, < 0 otherwise.
449  */
450 static int pispbe_prepare_job(struct pispbe_dev *pispbe,
451 			      struct pispbe_job_descriptor *job)
452 {
453 	struct pispbe_buffer *buf[PISPBE_NUM_NODES] = {};
454 	unsigned int config_index;
455 	struct pispbe_node *node;
456 	unsigned long flags;
457 
458 	lockdep_assert_held(&pispbe->hw_lock);
459 
460 	memset(job, 0, sizeof(struct pispbe_job_descriptor));
461 
462 	if (((BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE)) &
463 		pispbe->streaming_map) !=
464 			(BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE)))
465 		return -ENODEV;
466 
467 	node = &pispbe->node[CONFIG_NODE];
468 	spin_lock_irqsave(&node->ready_lock, flags);
469 	buf[CONFIG_NODE] = list_first_entry_or_null(&node->ready_queue,
470 						    struct pispbe_buffer,
471 						    ready_list);
472 	if (buf[CONFIG_NODE]) {
473 		list_del(&buf[CONFIG_NODE]->ready_list);
474 		pispbe->queued_job.buf[CONFIG_NODE] = buf[CONFIG_NODE];
475 	}
476 	spin_unlock_irqrestore(&node->ready_lock, flags);
477 
478 	/* Exit early if no config buffer has been queued. */
479 	if (!buf[CONFIG_NODE])
480 		return -ENODEV;
481 
482 	config_index = buf[CONFIG_NODE]->vb.vb2_buf.index;
483 	job->config = &pispbe->config[config_index];
484 	job->tiles = pispbe->config_dma_addr +
485 		     config_index * sizeof(struct pisp_be_tiles_config) +
486 		     offsetof(struct pisp_be_tiles_config, tiles);
487 
488 	/* remember: srcimages, captures then metadata */
489 	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
490 		unsigned int bayer_en =
491 			job->config->config.global.bayer_enables;
492 		unsigned int rgb_en =
493 			job->config->config.global.rgb_enables;
494 		bool ignore_buffers = false;
495 
496 		/* Config node is handled outside the loop above. */
497 		if (i == CONFIG_NODE)
498 			continue;
499 
500 		buf[i] = NULL;
501 		if (!(pispbe->streaming_map & BIT(i)))
502 			continue;
503 
504 		if ((!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT0) &&
505 		     i == OUTPUT0_NODE) ||
506 		    (!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT1) &&
507 		     i == OUTPUT1_NODE) ||
508 		    (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_INPUT) &&
509 		     i == TDN_INPUT_NODE) ||
510 		    (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) &&
511 		     i == TDN_OUTPUT_NODE) ||
512 		    (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_INPUT) &&
513 		     i == STITCH_INPUT_NODE) ||
514 		    (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) &&
515 		     i == STITCH_OUTPUT_NODE)) {
516 			/*
517 			 * Ignore Output0/Output1/Tdn/Stitch buffer check if the
518 			 * global enables aren't set for these blocks. If a
519 			 * buffer has been provided, we dequeue it back to the
520 			 * user with the other in-use buffers.
521 			 */
522 			ignore_buffers = true;
523 		}
524 
525 		node = &pispbe->node[i];
526 
527 		/* Pull a buffer from each V4L2 queue to form the queued job */
528 		spin_lock_irqsave(&node->ready_lock, flags);
529 		buf[i] = list_first_entry_or_null(&node->ready_queue,
530 						  struct pispbe_buffer,
531 						  ready_list);
532 		if (buf[i]) {
533 			list_del(&buf[i]->ready_list);
534 			pispbe->queued_job.buf[i] = buf[i];
535 		}
536 		spin_unlock_irqrestore(&node->ready_lock, flags);
537 
538 		if (!buf[i] && !ignore_buffers)
539 			goto err_return_buffers;
540 	}
541 
542 	pispbe->queued_job.valid = true;
543 
544 	/* Convert buffers to DMA addresses for the hardware */
545 	pispbe_xlate_addrs(pispbe, job, buf);
546 
547 	return 0;
548 
549 err_return_buffers:
550 	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
551 		struct pispbe_node *n =  &pispbe->node[i];
552 
553 		if (!buf[i])
554 			continue;
555 
556 		/* Return the buffer to the ready_list queue */
557 		spin_lock_irqsave(&n->ready_lock, flags);
558 		list_add(&buf[i]->ready_list, &n->ready_queue);
559 		spin_unlock_irqrestore(&n->ready_lock, flags);
560 	}
561 
562 	memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job));
563 
564 	return -ENODEV;
565 }
566 
567 static void pispbe_schedule(struct pispbe_dev *pispbe, bool clear_hw_busy)
568 {
569 	struct pispbe_job_descriptor job;
570 	unsigned long flags;
571 	int ret;
572 
573 	spin_lock_irqsave(&pispbe->hw_lock, flags);
574 
575 	if (clear_hw_busy)
576 		pispbe->hw_busy = false;
577 
578 	if (pispbe->hw_busy)
579 		goto unlock_and_return;
580 
581 	ret = pispbe_prepare_job(pispbe, &job);
582 	if (ret)
583 		goto unlock_and_return;
584 
585 	/*
586 	 * We can kick the job off without the hw_lock, as this can
587 	 * never run again until hw_busy is cleared, which will happen
588 	 * only when the following job has been queued and an interrupt
589 	 * is rised.
590 	 */
591 	pispbe->hw_busy = true;
592 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
593 
594 	if (job.config->num_tiles <= 0 ||
595 	    job.config->num_tiles > PISP_BACK_END_NUM_TILES ||
596 	    !((job.hw_enables.bayer_enables | job.hw_enables.rgb_enables) &
597 	      PISP_BE_BAYER_ENABLE_INPUT)) {
598 		/*
599 		 * Bad job. We can't let it proceed as it could lock up
600 		 * the hardware, or worse!
601 		 *
602 		 * For now, just force num_tiles to 0, which causes the
603 		 * H/W to do something bizarre but survivable. It
604 		 * increments (started,done) counters by more than 1,
605 		 * but we seem to survive...
606 		 */
607 		dev_dbg(pispbe->dev, "Bad job: invalid number of tiles: %u\n",
608 			job.config->num_tiles);
609 		job.config->num_tiles = 0;
610 	}
611 
612 	pispbe_queue_job(pispbe, &job);
613 
614 	return;
615 
616 unlock_and_return:
617 	/* No job has been queued, just release the lock and return. */
618 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
619 }
620 
621 static void pispbe_isr_jobdone(struct pispbe_dev *pispbe,
622 			       struct pispbe_job *job)
623 {
624 	struct pispbe_buffer **buf = job->buf;
625 	u64 ts = ktime_get_ns();
626 
627 	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
628 		if (buf[i]) {
629 			buf[i]->vb.vb2_buf.timestamp = ts;
630 			buf[i]->vb.sequence = pispbe->sequence;
631 			vb2_buffer_done(&buf[i]->vb.vb2_buf,
632 					VB2_BUF_STATE_DONE);
633 		}
634 	}
635 
636 	pispbe->sequence++;
637 }
638 
639 static irqreturn_t pispbe_isr(int irq, void *dev)
640 {
641 	struct pispbe_dev *pispbe = (struct pispbe_dev *)dev;
642 	bool can_queue_another = false;
643 	u8 started, done;
644 	u32 u;
645 
646 	u = pispbe_rd(pispbe, PISP_BE_INTERRUPT_STATUS_REG);
647 	if (u == 0)
648 		return IRQ_NONE;
649 
650 	pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, u);
651 	u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);
652 	done = (uint8_t)u;
653 	started = (uint8_t)(u >> 8);
654 
655 	/*
656 	 * Be aware that done can go up by 2 and started by 1 when: a job that
657 	 * we previously saw "start" now finishes, and we then queued a new job
658 	 * which we see both start and finish "simultaneously".
659 	 */
660 	if (pispbe->running_job.valid && pispbe->done != done) {
661 		pispbe_isr_jobdone(pispbe, &pispbe->running_job);
662 		memset(&pispbe->running_job, 0, sizeof(pispbe->running_job));
663 		pispbe->done++;
664 	}
665 
666 	if (pispbe->started != started) {
667 		pispbe->started++;
668 		can_queue_another = 1;
669 
670 		if (pispbe->done != done && pispbe->queued_job.valid) {
671 			pispbe_isr_jobdone(pispbe, &pispbe->queued_job);
672 			pispbe->done++;
673 		} else {
674 			pispbe->running_job = pispbe->queued_job;
675 		}
676 
677 		memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job));
678 	}
679 
680 	if (pispbe->done != done || pispbe->started != started) {
681 		dev_dbg(pispbe->dev,
682 			"Job counters not matching: done = %u, expected %u - started = %u, expected %u\n",
683 			pispbe->done, done, pispbe->started, started);
684 		pispbe->started = started;
685 		pispbe->done = done;
686 	}
687 
688 	/* check if there's more to do before going to sleep */
689 	pispbe_schedule(pispbe, can_queue_another);
690 
691 	return IRQ_HANDLED;
692 }
693 
694 static int pisp_be_validate_config(struct pispbe_dev *pispbe,
695 				   struct pisp_be_tiles_config *config)
696 {
697 	u32 bayer_enables = config->config.global.bayer_enables;
698 	u32 rgb_enables = config->config.global.rgb_enables;
699 	struct device *dev = pispbe->dev;
700 	struct v4l2_format *fmt;
701 	unsigned int bpl, size;
702 
703 	if (!(bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) ==
704 	    !(rgb_enables & PISP_BE_RGB_ENABLE_INPUT)) {
705 		dev_dbg(dev, "%s: Not one input enabled\n", __func__);
706 		return -EIO;
707 	}
708 
709 	/* Ensure output config strides and buffer sizes match the V4L2 formats. */
710 	fmt = &pispbe->node[TDN_OUTPUT_NODE].format;
711 	if (bayer_enables & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) {
712 		bpl = config->config.tdn_output_format.stride;
713 		size = bpl * config->config.tdn_output_format.height;
714 
715 		if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {
716 			dev_dbg(dev, "%s: bpl mismatch on tdn_output\n",
717 				__func__);
718 			return -EINVAL;
719 		}
720 
721 		if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {
722 			dev_dbg(dev, "%s: size mismatch on tdn_output\n",
723 				__func__);
724 			return -EINVAL;
725 		}
726 	}
727 
728 	fmt = &pispbe->node[STITCH_OUTPUT_NODE].format;
729 	if (bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) {
730 		bpl = config->config.stitch_output_format.stride;
731 		size = bpl * config->config.stitch_output_format.height;
732 
733 		if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {
734 			dev_dbg(dev, "%s: bpl mismatch on stitch_output\n",
735 				__func__);
736 			return -EINVAL;
737 		}
738 
739 		if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {
740 			dev_dbg(dev, "%s: size mismatch on stitch_output\n",
741 				__func__);
742 			return -EINVAL;
743 		}
744 	}
745 
746 	for (unsigned int j = 0; j < PISP_BACK_END_NUM_OUTPUTS; j++) {
747 		if (!(rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT(j)))
748 			continue;
749 
750 		if (config->config.output_format[j].image.format &
751 		    PISP_IMAGE_FORMAT_WALLPAPER_ROLL)
752 			continue; /* TODO: Size checks for wallpaper formats */
753 
754 		fmt = &pispbe->node[OUTPUT0_NODE + j].format;
755 		for (unsigned int i = 0; i < fmt->fmt.pix_mp.num_planes; i++) {
756 			bpl = !i ? config->config.output_format[j].image.stride
757 			    : config->config.output_format[j].image.stride2;
758 			size = bpl * config->config.output_format[j].image.height;
759 
760 			if (config->config.output_format[j].image.format &
761 						PISP_IMAGE_FORMAT_SAMPLING_420)
762 				size >>= 1;
763 
764 			if (fmt->fmt.pix_mp.plane_fmt[i].bytesperline < bpl) {
765 				dev_dbg(dev, "%s: bpl mismatch on output %d\n",
766 					__func__, j);
767 				return -EINVAL;
768 			}
769 
770 			if (fmt->fmt.pix_mp.plane_fmt[i].sizeimage < size) {
771 				dev_dbg(dev, "%s: size mismatch on output\n",
772 					__func__);
773 				return -EINVAL;
774 			}
775 		}
776 	}
777 
778 	return 0;
779 }
780 
781 static int pispbe_node_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,
782 				   unsigned int *nplanes, unsigned int sizes[],
783 				   struct device *alloc_devs[])
784 {
785 	struct pispbe_node *node = vb2_get_drv_priv(q);
786 	struct pispbe_dev *pispbe = node->pispbe;
787 	unsigned int num_planes = NODE_IS_MPLANE(node) ?
788 				  node->format.fmt.pix_mp.num_planes : 1;
789 
790 	if (*nplanes) {
791 		if (*nplanes != num_planes)
792 			return -EINVAL;
793 
794 		for (unsigned int i = 0; i < *nplanes; i++) {
795 			unsigned int size = NODE_IS_MPLANE(node) ?
796 				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
797 				node->format.fmt.meta.buffersize;
798 
799 			if (sizes[i] < size)
800 				return -EINVAL;
801 		}
802 
803 		return 0;
804 	}
805 
806 	*nplanes = num_planes;
807 	for (unsigned int i = 0; i < *nplanes; i++) {
808 		unsigned int size = NODE_IS_MPLANE(node) ?
809 				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
810 				node->format.fmt.meta.buffersize;
811 		sizes[i] = size;
812 	}
813 
814 	dev_dbg(pispbe->dev,
815 		"Image (or metadata) size %u, nbuffers %u for node %s\n",
816 		sizes[0], *nbuffers, NODE_NAME(node));
817 
818 	return 0;
819 }
820 
821 static int pispbe_node_buffer_prepare(struct vb2_buffer *vb)
822 {
823 	struct pispbe_node *node = vb2_get_drv_priv(vb->vb2_queue);
824 	struct pispbe_dev *pispbe = node->pispbe;
825 	unsigned int num_planes = NODE_IS_MPLANE(node) ?
826 				  node->format.fmt.pix_mp.num_planes : 1;
827 
828 	for (unsigned int i = 0; i < num_planes; i++) {
829 		unsigned long size = NODE_IS_MPLANE(node) ?
830 				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
831 				node->format.fmt.meta.buffersize;
832 
833 		if (vb2_plane_size(vb, i) < size) {
834 			dev_dbg(pispbe->dev,
835 				"data will not fit into plane %d (%lu < %lu)\n",
836 				i, vb2_plane_size(vb, i), size);
837 			return -EINVAL;
838 		}
839 
840 		vb2_set_plane_payload(vb, i, size);
841 	}
842 
843 	if (node->id == CONFIG_NODE) {
844 		void *dst = &node->pispbe->config[vb->index];
845 		void *src = vb2_plane_vaddr(vb, 0);
846 
847 		memcpy(dst, src, sizeof(struct pisp_be_tiles_config));
848 
849 		return pisp_be_validate_config(pispbe, dst);
850 	}
851 
852 	return 0;
853 }
854 
855 static void pispbe_node_buffer_queue(struct vb2_buffer *buf)
856 {
857 	struct vb2_v4l2_buffer *vbuf =
858 		container_of(buf, struct vb2_v4l2_buffer, vb2_buf);
859 	struct pispbe_buffer *buffer =
860 		container_of(vbuf, struct pispbe_buffer, vb);
861 	struct pispbe_node *node = vb2_get_drv_priv(buf->vb2_queue);
862 	struct pispbe_dev *pispbe = node->pispbe;
863 	unsigned long flags;
864 
865 	dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));
866 	spin_lock_irqsave(&node->ready_lock, flags);
867 	list_add_tail(&buffer->ready_list, &node->ready_queue);
868 	spin_unlock_irqrestore(&node->ready_lock, flags);
869 
870 	/*
871 	 * Every time we add a buffer, check if there's now some work for the hw
872 	 * to do.
873 	 */
874 	pispbe_schedule(pispbe, false);
875 }
876 
877 static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count)
878 {
879 	struct pispbe_node *node = vb2_get_drv_priv(q);
880 	struct pispbe_dev *pispbe = node->pispbe;
881 	struct pispbe_buffer *buf, *tmp;
882 	unsigned long flags;
883 	int ret;
884 
885 	ret = pm_runtime_resume_and_get(pispbe->dev);
886 	if (ret < 0)
887 		goto err_return_buffers;
888 
889 	spin_lock_irqsave(&pispbe->hw_lock, flags);
890 	node->pispbe->streaming_map |=  BIT(node->id);
891 	node->pispbe->sequence = 0;
892 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
893 
894 	dev_dbg(pispbe->dev, "%s: for node %s (count %u)\n",
895 		__func__, NODE_NAME(node), count);
896 	dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",
897 		node->pispbe->streaming_map);
898 
899 	/* Maybe we're ready to run. */
900 	pispbe_schedule(pispbe, false);
901 
902 	return 0;
903 
904 err_return_buffers:
905 	spin_lock_irqsave(&pispbe->hw_lock, flags);
906 	list_for_each_entry_safe(buf, tmp, &node->ready_queue, ready_list) {
907 		list_del(&buf->ready_list);
908 		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
909 	}
910 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
911 
912 	return ret;
913 }
914 
915 static void pispbe_node_stop_streaming(struct vb2_queue *q)
916 {
917 	struct pispbe_node *node = vb2_get_drv_priv(q);
918 	struct pispbe_dev *pispbe = node->pispbe;
919 	struct pispbe_buffer *buf;
920 	unsigned long flags;
921 
922 	/*
923 	 * Now this is a bit awkward. In a simple M2M device we could just wait
924 	 * for all queued jobs to complete, but here there's a risk that a
925 	 * partial set of buffers was queued and cannot be run. For now, just
926 	 * cancel all buffers stuck in the "ready queue", then wait for any
927 	 * running job.
928 	 *
929 	 * This may return buffers out of order.
930 	 */
931 	dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));
932 	spin_lock_irqsave(&pispbe->hw_lock, flags);
933 	do {
934 		unsigned long flags1;
935 
936 		spin_lock_irqsave(&node->ready_lock, flags1);
937 		buf = list_first_entry_or_null(&node->ready_queue,
938 					       struct pispbe_buffer,
939 					       ready_list);
940 		if (buf) {
941 			list_del(&buf->ready_list);
942 			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
943 		}
944 		spin_unlock_irqrestore(&node->ready_lock, flags1);
945 	} while (buf);
946 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
947 
948 	vb2_wait_for_all_buffers(&node->queue);
949 
950 	spin_lock_irqsave(&pispbe->hw_lock, flags);
951 	pispbe->streaming_map &= ~BIT(node->id);
952 	spin_unlock_irqrestore(&pispbe->hw_lock, flags);
953 
954 	pm_runtime_mark_last_busy(pispbe->dev);
955 	pm_runtime_put_autosuspend(pispbe->dev);
956 
957 	dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",
958 		pispbe->streaming_map);
959 }
960 
961 static const struct vb2_ops pispbe_node_queue_ops = {
962 	.queue_setup = pispbe_node_queue_setup,
963 	.buf_prepare = pispbe_node_buffer_prepare,
964 	.buf_queue = pispbe_node_buffer_queue,
965 	.start_streaming = pispbe_node_start_streaming,
966 	.stop_streaming = pispbe_node_stop_streaming,
967 };
968 
969 static const struct v4l2_file_operations pispbe_fops = {
970 	.owner          = THIS_MODULE,
971 	.open           = v4l2_fh_open,
972 	.release        = vb2_fop_release,
973 	.poll           = vb2_fop_poll,
974 	.unlocked_ioctl = video_ioctl2,
975 	.mmap           = vb2_fop_mmap
976 };
977 
978 static int pispbe_node_querycap(struct file *file, void *priv,
979 				struct v4l2_capability *cap)
980 {
981 	struct pispbe_node *node = video_drvdata(file);
982 	struct pispbe_dev *pispbe = node->pispbe;
983 
984 	strscpy(cap->driver, PISPBE_NAME, sizeof(cap->driver));
985 	strscpy(cap->card, PISPBE_NAME, sizeof(cap->card));
986 
987 	dev_dbg(pispbe->dev, "Caps for node %s: %x and %x (dev %x)\n",
988 		NODE_NAME(node), cap->capabilities, cap->device_caps,
989 		node->vfd.device_caps);
990 
991 	return 0;
992 }
993 
994 static int pispbe_node_g_fmt_vid_cap(struct file *file, void *priv,
995 				     struct v4l2_format *f)
996 {
997 	struct pispbe_node *node = video_drvdata(file);
998 	struct pispbe_dev *pispbe = node->pispbe;
999 
1000 	if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
1001 		dev_dbg(pispbe->dev,
1002 			"Cannot get capture fmt for output node %s\n",
1003 			NODE_NAME(node));
1004 		return -EINVAL;
1005 	}
1006 
1007 	*f = node->format;
1008 	dev_dbg(pispbe->dev, "Get capture format for node %s\n",
1009 		NODE_NAME(node));
1010 
1011 	return 0;
1012 }
1013 
1014 static int pispbe_node_g_fmt_vid_out(struct file *file, void *priv,
1015 				     struct v4l2_format *f)
1016 {
1017 	struct pispbe_node *node = video_drvdata(file);
1018 	struct pispbe_dev *pispbe = node->pispbe;
1019 
1020 	if (NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
1021 		dev_dbg(pispbe->dev,
1022 			"Cannot get capture fmt for output node %s\n",
1023 			 NODE_NAME(node));
1024 		return -EINVAL;
1025 	}
1026 
1027 	*f = node->format;
1028 	dev_dbg(pispbe->dev, "Get output format for node %s\n",
1029 		NODE_NAME(node));
1030 
1031 	return 0;
1032 }
1033 
1034 static int pispbe_node_g_fmt_meta_out(struct file *file, void *priv,
1035 				      struct v4l2_format *f)
1036 {
1037 	struct pispbe_node *node = video_drvdata(file);
1038 	struct pispbe_dev *pispbe = node->pispbe;
1039 
1040 	if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {
1041 		dev_dbg(pispbe->dev,
1042 			"Cannot get capture fmt for meta output node %s\n",
1043 			NODE_NAME(node));
1044 		return -EINVAL;
1045 	}
1046 
1047 	*f = node->format;
1048 	dev_dbg(pispbe->dev, "Get output format for meta node %s\n",
1049 		NODE_NAME(node));
1050 
1051 	return 0;
1052 }
1053 
1054 static const struct pisp_be_format *pispbe_find_fmt(unsigned int fourcc)
1055 {
1056 	for (unsigned int i = 0; i < ARRAY_SIZE(supported_formats); i++) {
1057 		if (supported_formats[i].fourcc == fourcc)
1058 			return &supported_formats[i];
1059 	}
1060 
1061 	return NULL;
1062 }
1063 
1064 static void pispbe_set_plane_params(struct v4l2_format *f,
1065 				    const struct pisp_be_format *fmt)
1066 {
1067 	unsigned int nplanes = f->fmt.pix_mp.num_planes;
1068 	unsigned int total_plane_factor = 0;
1069 
1070 	for (unsigned int i = 0; i < PISPBE_MAX_PLANES; i++)
1071 		total_plane_factor += fmt->plane_factor[i];
1072 
1073 	for (unsigned int i = 0; i < nplanes; i++) {
1074 		struct v4l2_plane_pix_format *p = &f->fmt.pix_mp.plane_fmt[i];
1075 		unsigned int bpl, plane_size;
1076 
1077 		bpl = (f->fmt.pix_mp.width * fmt->bit_depth) >> 3;
1078 		bpl = ALIGN(max(p->bytesperline, bpl), fmt->align);
1079 
1080 		plane_size = bpl * f->fmt.pix_mp.height *
1081 		      (nplanes > 1 ? fmt->plane_factor[i] : total_plane_factor);
1082 		/*
1083 		 * The shift is to divide out the plane_factor fixed point
1084 		 * scaling of 8.
1085 		 */
1086 		plane_size = max(p->sizeimage, plane_size >> 3);
1087 
1088 		p->bytesperline = bpl;
1089 		p->sizeimage = plane_size;
1090 	}
1091 }
1092 
1093 static void pispbe_try_format(struct v4l2_format *f, struct pispbe_node *node)
1094 {
1095 	struct pispbe_dev *pispbe = node->pispbe;
1096 	u32 pixfmt = f->fmt.pix_mp.pixelformat;
1097 	const struct pisp_be_format *fmt;
1098 	bool is_rgb;
1099 
1100 	dev_dbg(pispbe->dev,
1101 		"%s: [%s] req %ux%u %p4cc, planes %d\n",
1102 		__func__, NODE_NAME(node), f->fmt.pix_mp.width,
1103 		f->fmt.pix_mp.height, &pixfmt,
1104 		f->fmt.pix_mp.num_planes);
1105 
1106 	fmt = pispbe_find_fmt(pixfmt);
1107 	if (!fmt) {
1108 		dev_dbg(pispbe->dev,
1109 			"%s: [%s] Format not found, defaulting to YUV420\n",
1110 			__func__, NODE_NAME(node));
1111 		fmt = pispbe_find_fmt(V4L2_PIX_FMT_YUV420);
1112 	}
1113 
1114 	f->fmt.pix_mp.pixelformat = fmt->fourcc;
1115 	f->fmt.pix_mp.num_planes = fmt->num_planes;
1116 	f->fmt.pix_mp.field = V4L2_FIELD_NONE;
1117 	f->fmt.pix_mp.width = max(min(f->fmt.pix_mp.width, 65536u),
1118 				  PISP_BACK_END_MIN_TILE_WIDTH);
1119 	f->fmt.pix_mp.height = max(min(f->fmt.pix_mp.height, 65536u),
1120 				   PISP_BACK_END_MIN_TILE_HEIGHT);
1121 
1122 	/*
1123 	 * Fill in the actual colour space when the requested one was
1124 	 * not supported. This also catches the case when the "default"
1125 	 * colour space was requested (as that's never in the mask).
1126 	 */
1127 	if (!(V4L2_COLORSPACE_MASK(f->fmt.pix_mp.colorspace) &
1128 	    fmt->colorspace_mask))
1129 		f->fmt.pix_mp.colorspace = fmt->colorspace_default;
1130 
1131 	/* In all cases, we only support the defaults for these: */
1132 	f->fmt.pix_mp.ycbcr_enc =
1133 		V4L2_MAP_YCBCR_ENC_DEFAULT(f->fmt.pix_mp.colorspace);
1134 	f->fmt.pix_mp.xfer_func =
1135 		V4L2_MAP_XFER_FUNC_DEFAULT(f->fmt.pix_mp.colorspace);
1136 
1137 	is_rgb = f->fmt.pix_mp.colorspace == V4L2_COLORSPACE_SRGB;
1138 	f->fmt.pix_mp.quantization =
1139 		V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, f->fmt.pix_mp.colorspace,
1140 					      f->fmt.pix_mp.ycbcr_enc);
1141 
1142 	/* Set plane size and bytes/line for each plane. */
1143 	pispbe_set_plane_params(f, fmt);
1144 
1145 	for (unsigned int i = 0; i < f->fmt.pix_mp.num_planes; i++) {
1146 		dev_dbg(pispbe->dev,
1147 			"%s: [%s] calc plane %d, %ux%u, depth %u, bpl %u size %u\n",
1148 			__func__, NODE_NAME(node), i, f->fmt.pix_mp.width,
1149 			f->fmt.pix_mp.height, fmt->bit_depth,
1150 			f->fmt.pix_mp.plane_fmt[i].bytesperline,
1151 			f->fmt.pix_mp.plane_fmt[i].sizeimage);
1152 	}
1153 }
1154 
1155 static int pispbe_node_try_fmt_vid_cap(struct file *file, void *priv,
1156 				       struct v4l2_format *f)
1157 {
1158 	struct pispbe_node *node = video_drvdata(file);
1159 	struct pispbe_dev *pispbe = node->pispbe;
1160 
1161 	if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
1162 		dev_dbg(pispbe->dev,
1163 			"Cannot set capture fmt for output node %s\n",
1164 			NODE_NAME(node));
1165 		return -EINVAL;
1166 	}
1167 
1168 	pispbe_try_format(f, node);
1169 
1170 	return 0;
1171 }
1172 
1173 static int pispbe_node_try_fmt_vid_out(struct file *file, void *priv,
1174 				       struct v4l2_format *f)
1175 {
1176 	struct pispbe_node *node = video_drvdata(file);
1177 	struct pispbe_dev *pispbe = node->pispbe;
1178 
1179 	if (!NODE_IS_OUTPUT(node) || NODE_IS_META(node)) {
1180 		dev_dbg(pispbe->dev,
1181 			"Cannot set capture fmt for output node %s\n",
1182 			NODE_NAME(node));
1183 		return -EINVAL;
1184 	}
1185 
1186 	pispbe_try_format(f, node);
1187 
1188 	return 0;
1189 }
1190 
1191 static int pispbe_node_try_fmt_meta_out(struct file *file, void *priv,
1192 					struct v4l2_format *f)
1193 {
1194 	struct pispbe_node *node = video_drvdata(file);
1195 	struct pispbe_dev *pispbe = node->pispbe;
1196 
1197 	if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {
1198 		dev_dbg(pispbe->dev,
1199 			"Cannot set capture fmt for meta output node %s\n",
1200 			NODE_NAME(node));
1201 		return -EINVAL;
1202 	}
1203 
1204 	f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;
1205 	f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);
1206 
1207 	return 0;
1208 }
1209 
1210 static int pispbe_node_s_fmt_vid_cap(struct file *file, void *priv,
1211 				     struct v4l2_format *f)
1212 {
1213 	struct pispbe_node *node = video_drvdata(file);
1214 	struct pispbe_dev *pispbe = node->pispbe;
1215 	int ret;
1216 
1217 	ret = pispbe_node_try_fmt_vid_cap(file, priv, f);
1218 	if (ret < 0)
1219 		return ret;
1220 
1221 	if (vb2_is_busy(&node->queue))
1222 		return -EBUSY;
1223 
1224 	node->format = *f;
1225 	node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);
1226 
1227 	dev_dbg(pispbe->dev, "Set capture format for node %s to %p4cc\n",
1228 		NODE_NAME(node), &f->fmt.pix_mp.pixelformat);
1229 
1230 	return 0;
1231 }
1232 
1233 static int pispbe_node_s_fmt_vid_out(struct file *file, void *priv,
1234 				     struct v4l2_format *f)
1235 {
1236 	struct pispbe_node *node = video_drvdata(file);
1237 	struct pispbe_dev *pispbe = node->pispbe;
1238 	int ret;
1239 
1240 	ret = pispbe_node_try_fmt_vid_out(file, priv, f);
1241 	if (ret < 0)
1242 		return ret;
1243 
1244 	if (vb2_is_busy(&node->queue))
1245 		return -EBUSY;
1246 
1247 	node->format = *f;
1248 	node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);
1249 
1250 	dev_dbg(pispbe->dev, "Set output format for node %s to %p4cc\n",
1251 		NODE_NAME(node), &f->fmt.pix_mp.pixelformat);
1252 
1253 	return 0;
1254 }
1255 
1256 static int pispbe_node_s_fmt_meta_out(struct file *file, void *priv,
1257 				      struct v4l2_format *f)
1258 {
1259 	struct pispbe_node *node = video_drvdata(file);
1260 	struct pispbe_dev *pispbe = node->pispbe;
1261 	int ret;
1262 
1263 	ret = pispbe_node_try_fmt_meta_out(file, priv, f);
1264 	if (ret < 0)
1265 		return ret;
1266 
1267 	if (vb2_is_busy(&node->queue))
1268 		return -EBUSY;
1269 
1270 	node->format = *f;
1271 	node->pisp_format = &meta_out_supported_formats[0];
1272 
1273 	dev_dbg(pispbe->dev, "Set output format for meta node %s to %p4cc\n",
1274 		NODE_NAME(node), &f->fmt.meta.dataformat);
1275 
1276 	return 0;
1277 }
1278 
1279 static int pispbe_node_enum_fmt(struct file *file, void  *priv,
1280 				struct v4l2_fmtdesc *f)
1281 {
1282 	struct pispbe_node *node = video_drvdata(file);
1283 
1284 	if (f->type != node->queue.type)
1285 		return -EINVAL;
1286 
1287 	if (NODE_IS_META(node)) {
1288 		if (f->index)
1289 			return -EINVAL;
1290 
1291 		f->pixelformat = V4L2_META_FMT_RPI_BE_CFG;
1292 		f->flags = 0;
1293 		return 0;
1294 	}
1295 
1296 	if (f->index >= ARRAY_SIZE(supported_formats))
1297 		return -EINVAL;
1298 
1299 	f->pixelformat = supported_formats[f->index].fourcc;
1300 	f->flags = 0;
1301 
1302 	return 0;
1303 }
1304 
1305 static int pispbe_enum_framesizes(struct file *file, void *priv,
1306 				  struct v4l2_frmsizeenum *fsize)
1307 {
1308 	struct pispbe_node *node = video_drvdata(file);
1309 	struct pispbe_dev *pispbe = node->pispbe;
1310 
1311 	if (NODE_IS_META(node) || fsize->index)
1312 		return -EINVAL;
1313 
1314 	if (!pispbe_find_fmt(fsize->pixel_format)) {
1315 		dev_dbg(pispbe->dev, "Invalid pixel code: %x\n",
1316 			fsize->pixel_format);
1317 		return -EINVAL;
1318 	}
1319 
1320 	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
1321 	fsize->stepwise.min_width = 32;
1322 	fsize->stepwise.max_width = 65535;
1323 	fsize->stepwise.step_width = 2;
1324 
1325 	fsize->stepwise.min_height = 32;
1326 	fsize->stepwise.max_height = 65535;
1327 	fsize->stepwise.step_height = 2;
1328 
1329 	return 0;
1330 }
1331 
1332 static const struct v4l2_ioctl_ops pispbe_node_ioctl_ops = {
1333 	.vidioc_querycap = pispbe_node_querycap,
1334 	.vidioc_g_fmt_vid_cap_mplane = pispbe_node_g_fmt_vid_cap,
1335 	.vidioc_g_fmt_vid_out_mplane = pispbe_node_g_fmt_vid_out,
1336 	.vidioc_g_fmt_meta_out = pispbe_node_g_fmt_meta_out,
1337 	.vidioc_try_fmt_vid_cap_mplane = pispbe_node_try_fmt_vid_cap,
1338 	.vidioc_try_fmt_vid_out_mplane = pispbe_node_try_fmt_vid_out,
1339 	.vidioc_try_fmt_meta_out = pispbe_node_try_fmt_meta_out,
1340 	.vidioc_s_fmt_vid_cap_mplane = pispbe_node_s_fmt_vid_cap,
1341 	.vidioc_s_fmt_vid_out_mplane = pispbe_node_s_fmt_vid_out,
1342 	.vidioc_s_fmt_meta_out = pispbe_node_s_fmt_meta_out,
1343 	.vidioc_enum_fmt_vid_cap = pispbe_node_enum_fmt,
1344 	.vidioc_enum_fmt_vid_out = pispbe_node_enum_fmt,
1345 	.vidioc_enum_fmt_meta_out = pispbe_node_enum_fmt,
1346 	.vidioc_enum_framesizes = pispbe_enum_framesizes,
1347 	.vidioc_create_bufs = vb2_ioctl_create_bufs,
1348 	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
1349 	.vidioc_querybuf = vb2_ioctl_querybuf,
1350 	.vidioc_qbuf = vb2_ioctl_qbuf,
1351 	.vidioc_dqbuf = vb2_ioctl_dqbuf,
1352 	.vidioc_expbuf = vb2_ioctl_expbuf,
1353 	.vidioc_reqbufs = vb2_ioctl_reqbufs,
1354 	.vidioc_streamon = vb2_ioctl_streamon,
1355 	.vidioc_streamoff = vb2_ioctl_streamoff,
1356 };
1357 
1358 static const struct video_device pispbe_videodev = {
1359 	.name = PISPBE_NAME,
1360 	.vfl_dir = VFL_DIR_M2M, /* gets overwritten */
1361 	.fops = &pispbe_fops,
1362 	.ioctl_ops = &pispbe_node_ioctl_ops,
1363 	.minor = -1,
1364 	.release = video_device_release_empty,
1365 };
1366 
1367 static void pispbe_node_def_fmt(struct pispbe_node *node)
1368 {
1369 	if (NODE_IS_META(node) && NODE_IS_OUTPUT(node)) {
1370 		/* Config node */
1371 		struct v4l2_format *f = &node->format;
1372 
1373 		f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;
1374 		f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);
1375 		f->type = node->buf_type;
1376 	} else {
1377 		struct v4l2_format f = {
1378 			.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420,
1379 			.fmt.pix_mp.width = 1920,
1380 			.fmt.pix_mp.height = 1080,
1381 			.type = node->buf_type,
1382 		};
1383 		pispbe_try_format(&f, node);
1384 		node->format = f;
1385 	}
1386 
1387 	node->pisp_format = pispbe_find_fmt(node->format.fmt.pix_mp.pixelformat);
1388 }
1389 
1390 /*
1391  * Initialise a struct pispbe_node and register it as /dev/video<N>
1392  * to represent one of the PiSP Back End's input or output streams.
1393  */
1394 static int pispbe_init_node(struct pispbe_dev *pispbe, unsigned int id)
1395 {
1396 	bool output = NODE_DESC_IS_OUTPUT(&node_desc[id]);
1397 	struct pispbe_node *node = &pispbe->node[id];
1398 	struct media_entity *entity = &node->vfd.entity;
1399 	struct video_device *vdev = &node->vfd;
1400 	struct vb2_queue *q = &node->queue;
1401 	int ret;
1402 
1403 	node->id = id;
1404 	node->pispbe = pispbe;
1405 	node->buf_type = node_desc[id].buf_type;
1406 
1407 	mutex_init(&node->node_lock);
1408 	mutex_init(&node->queue_lock);
1409 	INIT_LIST_HEAD(&node->ready_queue);
1410 	spin_lock_init(&node->ready_lock);
1411 
1412 	node->format.type = node->buf_type;
1413 	pispbe_node_def_fmt(node);
1414 
1415 	q->type = node->buf_type;
1416 	q->io_modes = VB2_MMAP | VB2_DMABUF;
1417 	q->mem_ops = &vb2_dma_contig_memops;
1418 	q->drv_priv = node;
1419 	q->ops = &pispbe_node_queue_ops;
1420 	q->buf_struct_size = sizeof(struct pispbe_buffer);
1421 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
1422 	q->dev = pispbe->dev;
1423 	/* get V4L2 to handle node->queue locking */
1424 	q->lock = &node->queue_lock;
1425 
1426 	ret = vb2_queue_init(q);
1427 	if (ret < 0) {
1428 		dev_err(pispbe->dev, "vb2_queue_init failed\n");
1429 		goto err_mutex_destroy;
1430 	}
1431 
1432 	*vdev = pispbe_videodev; /* default initialization */
1433 	strscpy(vdev->name, node_desc[id].ent_name, sizeof(vdev->name));
1434 	vdev->v4l2_dev = &pispbe->v4l2_dev;
1435 	vdev->vfl_dir = output ? VFL_DIR_TX : VFL_DIR_RX;
1436 	/* get V4L2 to serialise our ioctls */
1437 	vdev->lock = &node->node_lock;
1438 	vdev->queue = &node->queue;
1439 	vdev->device_caps = V4L2_CAP_STREAMING | node_desc[id].caps;
1440 
1441 	node->pad.flags = output ? MEDIA_PAD_FL_SOURCE : MEDIA_PAD_FL_SINK;
1442 	ret = media_entity_pads_init(entity, 1, &node->pad);
1443 	if (ret) {
1444 		dev_err(pispbe->dev,
1445 			"Failed to register media pads for %s device node\n",
1446 			NODE_NAME(node));
1447 		goto err_unregister_queue;
1448 	}
1449 
1450 	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
1451 	if (ret) {
1452 		dev_err(pispbe->dev,
1453 			"Failed to register video %s device node\n",
1454 			NODE_NAME(node));
1455 		goto err_unregister_queue;
1456 	}
1457 	video_set_drvdata(vdev, node);
1458 
1459 	if (output)
1460 		ret = media_create_pad_link(entity, 0, &pispbe->sd.entity,
1461 					    id, MEDIA_LNK_FL_IMMUTABLE |
1462 					    MEDIA_LNK_FL_ENABLED);
1463 	else
1464 		ret = media_create_pad_link(&pispbe->sd.entity, id, entity,
1465 					    0, MEDIA_LNK_FL_IMMUTABLE |
1466 					    MEDIA_LNK_FL_ENABLED);
1467 	if (ret)
1468 		goto err_unregister_video_dev;
1469 
1470 	dev_dbg(pispbe->dev, "%s device node registered as /dev/video%d\n",
1471 		NODE_NAME(node), node->vfd.num);
1472 
1473 	return 0;
1474 
1475 err_unregister_video_dev:
1476 	video_unregister_device(&node->vfd);
1477 err_unregister_queue:
1478 	vb2_queue_release(&node->queue);
1479 err_mutex_destroy:
1480 	mutex_destroy(&node->node_lock);
1481 	mutex_destroy(&node->queue_lock);
1482 	return ret;
1483 }
1484 
1485 static const struct v4l2_subdev_pad_ops pispbe_pad_ops = {
1486 	.link_validate = v4l2_subdev_link_validate_default,
1487 };
1488 
1489 static const struct v4l2_subdev_ops pispbe_sd_ops = {
1490 	.pad = &pispbe_pad_ops,
1491 };
1492 
1493 static int pispbe_init_subdev(struct pispbe_dev *pispbe)
1494 {
1495 	struct v4l2_subdev *sd = &pispbe->sd;
1496 	int ret;
1497 
1498 	v4l2_subdev_init(sd, &pispbe_sd_ops);
1499 	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
1500 	sd->owner = THIS_MODULE;
1501 	sd->dev = pispbe->dev;
1502 	strscpy(sd->name, PISPBE_NAME, sizeof(sd->name));
1503 
1504 	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++)
1505 		pispbe->pad[i].flags =
1506 			NODE_DESC_IS_OUTPUT(&node_desc[i]) ?
1507 			MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE;
1508 
1509 	ret = media_entity_pads_init(&sd->entity, PISPBE_NUM_NODES,
1510 				     pispbe->pad);
1511 	if (ret)
1512 		goto error;
1513 
1514 	ret = v4l2_device_register_subdev(&pispbe->v4l2_dev, sd);
1515 	if (ret)
1516 		goto error;
1517 
1518 	return 0;
1519 
1520 error:
1521 	media_entity_cleanup(&sd->entity);
1522 	return ret;
1523 }
1524 
1525 static int pispbe_init_devices(struct pispbe_dev *pispbe)
1526 {
1527 	struct v4l2_device *v4l2_dev;
1528 	struct media_device *mdev;
1529 	unsigned int num_regist;
1530 	int ret;
1531 
1532 	/* Register v4l2_device and media_device */
1533 	mdev = &pispbe->mdev;
1534 	mdev->hw_revision = pispbe->hw_version;
1535 	mdev->dev = pispbe->dev;
1536 	strscpy(mdev->model, PISPBE_NAME, sizeof(mdev->model));
1537 	media_device_init(mdev);
1538 
1539 	v4l2_dev = &pispbe->v4l2_dev;
1540 	v4l2_dev->mdev = &pispbe->mdev;
1541 	strscpy(v4l2_dev->name, PISPBE_NAME, sizeof(v4l2_dev->name));
1542 
1543 	ret = v4l2_device_register(pispbe->dev, v4l2_dev);
1544 	if (ret)
1545 		goto err_media_dev_cleanup;
1546 
1547 	/* Register the PISPBE subdevice. */
1548 	ret = pispbe_init_subdev(pispbe);
1549 	if (ret)
1550 		goto err_unregister_v4l2;
1551 
1552 	/* Create device video nodes */
1553 	for (num_regist = 0; num_regist < PISPBE_NUM_NODES; num_regist++) {
1554 		ret = pispbe_init_node(pispbe, num_regist);
1555 		if (ret)
1556 			goto err_unregister_nodes;
1557 	}
1558 
1559 	ret = media_device_register(mdev);
1560 	if (ret)
1561 		goto err_unregister_nodes;
1562 
1563 	pispbe->config =
1564 		dma_alloc_coherent(pispbe->dev,
1565 				   sizeof(struct pisp_be_tiles_config) *
1566 					PISP_BE_NUM_CONFIG_BUFFERS,
1567 				   &pispbe->config_dma_addr, GFP_KERNEL);
1568 	if (!pispbe->config) {
1569 		dev_err(pispbe->dev, "Unable to allocate cached config buffers.\n");
1570 		ret = -ENOMEM;
1571 		goto err_unregister_mdev;
1572 	}
1573 
1574 	return 0;
1575 
1576 err_unregister_mdev:
1577 	media_device_unregister(mdev);
1578 err_unregister_nodes:
1579 	while (num_regist-- > 0) {
1580 		video_unregister_device(&pispbe->node[num_regist].vfd);
1581 		vb2_queue_release(&pispbe->node[num_regist].queue);
1582 	}
1583 	v4l2_device_unregister_subdev(&pispbe->sd);
1584 	media_entity_cleanup(&pispbe->sd.entity);
1585 err_unregister_v4l2:
1586 	v4l2_device_unregister(v4l2_dev);
1587 err_media_dev_cleanup:
1588 	media_device_cleanup(mdev);
1589 	return ret;
1590 }
1591 
1592 static void pispbe_destroy_devices(struct pispbe_dev *pispbe)
1593 {
1594 	if (pispbe->config) {
1595 		dma_free_coherent(pispbe->dev,
1596 				  sizeof(struct pisp_be_tiles_config) *
1597 					PISP_BE_NUM_CONFIG_BUFFERS,
1598 				  pispbe->config,
1599 				  pispbe->config_dma_addr);
1600 	}
1601 
1602 	dev_dbg(pispbe->dev, "Unregister from media controller\n");
1603 
1604 	v4l2_device_unregister_subdev(&pispbe->sd);
1605 	media_entity_cleanup(&pispbe->sd.entity);
1606 	media_device_unregister(&pispbe->mdev);
1607 
1608 	for (int i = PISPBE_NUM_NODES - 1; i >= 0; i--) {
1609 		video_unregister_device(&pispbe->node[i].vfd);
1610 		vb2_queue_release(&pispbe->node[i].queue);
1611 		mutex_destroy(&pispbe->node[i].node_lock);
1612 		mutex_destroy(&pispbe->node[i].queue_lock);
1613 	}
1614 
1615 	media_device_cleanup(&pispbe->mdev);
1616 	v4l2_device_unregister(&pispbe->v4l2_dev);
1617 }
1618 
1619 static int pispbe_runtime_suspend(struct device *dev)
1620 {
1621 	struct pispbe_dev *pispbe = dev_get_drvdata(dev);
1622 
1623 	clk_disable_unprepare(pispbe->clk);
1624 
1625 	return 0;
1626 }
1627 
1628 static int pispbe_runtime_resume(struct device *dev)
1629 {
1630 	struct pispbe_dev *pispbe = dev_get_drvdata(dev);
1631 	int ret;
1632 
1633 	ret = clk_prepare_enable(pispbe->clk);
1634 	if (ret) {
1635 		dev_err(dev, "Unable to enable clock\n");
1636 		return ret;
1637 	}
1638 
1639 	dev_dbg(dev, "%s: Enabled clock, rate=%lu\n",
1640 		__func__, clk_get_rate(pispbe->clk));
1641 
1642 	return 0;
1643 }
1644 
1645 static int pispbe_hw_init(struct pispbe_dev *pispbe)
1646 {
1647 	u32 u;
1648 
1649 	/* Check the HW is present and has a known version */
1650 	u = pispbe_rd(pispbe, PISP_BE_VERSION_REG);
1651 	dev_dbg(pispbe->dev, "pispbe_probe: HW version:  0x%08x", u);
1652 	pispbe->hw_version = u;
1653 	if ((u & ~PISP_BE_VERSION_MINOR_BITS) != PISP_BE_VERSION_2712)
1654 		return -ENODEV;
1655 
1656 	/* Clear leftover interrupts */
1657 	pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, 0xFFFFFFFFu);
1658 	u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);
1659 	dev_dbg(pispbe->dev, "pispbe_probe: BatchStatus: 0x%08x", u);
1660 
1661 	pispbe->done = (uint8_t)u;
1662 	pispbe->started = (uint8_t)(u >> 8);
1663 	u = pispbe_rd(pispbe, PISP_BE_STATUS_REG);
1664 	dev_dbg(pispbe->dev, "pispbe_probe: Status:      0x%08x", u);
1665 
1666 	if (u != 0 || pispbe->done != pispbe->started) {
1667 		dev_err(pispbe->dev, "pispbe_probe: HW is stuck or busy\n");
1668 		return -EBUSY;
1669 	}
1670 
1671 	/*
1672 	 * AXI QOS=0, CACHE=4'b0010, PROT=3'b011
1673 	 * Also set "chicken bits" 22:20 which enable sub-64-byte bursts
1674 	 * and AXI AWID/BID variability (on versions which support this).
1675 	 */
1676 	pispbe_wr(pispbe, PISP_BE_AXI_REG, 0x32703200u);
1677 
1678 	/* Enable both interrupt flags */
1679 	pispbe_wr(pispbe, PISP_BE_INTERRUPT_EN_REG, 0x00000003u);
1680 
1681 	return 0;
1682 }
1683 
1684 /* Probe the ISP-BE hardware block, as a single platform device. */
1685 static int pispbe_probe(struct platform_device *pdev)
1686 {
1687 	struct pispbe_dev *pispbe;
1688 	int ret;
1689 
1690 	pispbe = devm_kzalloc(&pdev->dev, sizeof(*pispbe), GFP_KERNEL);
1691 	if (!pispbe)
1692 		return -ENOMEM;
1693 
1694 	dev_set_drvdata(&pdev->dev, pispbe);
1695 	pispbe->dev = &pdev->dev;
1696 	platform_set_drvdata(pdev, pispbe);
1697 
1698 	pispbe->be_reg_base = devm_platform_ioremap_resource(pdev, 0);
1699 	if (IS_ERR(pispbe->be_reg_base)) {
1700 		dev_err(&pdev->dev, "Failed to get ISP-BE registers address\n");
1701 		return PTR_ERR(pispbe->be_reg_base);
1702 	}
1703 
1704 	pispbe->irq = platform_get_irq(pdev, 0);
1705 	if (pispbe->irq <= 0)
1706 		return -EINVAL;
1707 
1708 	ret = devm_request_irq(&pdev->dev, pispbe->irq, pispbe_isr, 0,
1709 			       PISPBE_NAME, pispbe);
1710 	if (ret) {
1711 		dev_err(&pdev->dev, "Unable to request interrupt\n");
1712 		return ret;
1713 	}
1714 
1715 	ret = dma_set_mask_and_coherent(pispbe->dev, DMA_BIT_MASK(36));
1716 	if (ret)
1717 		return ret;
1718 
1719 	pispbe->clk = devm_clk_get(&pdev->dev, NULL);
1720 	if (IS_ERR(pispbe->clk))
1721 		return dev_err_probe(&pdev->dev, PTR_ERR(pispbe->clk),
1722 				     "Failed to get clock");
1723 
1724 	/* Hardware initialisation */
1725 	pm_runtime_set_autosuspend_delay(pispbe->dev, 200);
1726 	pm_runtime_use_autosuspend(pispbe->dev);
1727 	pm_runtime_enable(pispbe->dev);
1728 
1729 	ret = pispbe_runtime_resume(pispbe->dev);
1730 	if (ret)
1731 		goto pm_runtime_disable_err;
1732 
1733 	pispbe->hw_busy = false;
1734 	spin_lock_init(&pispbe->hw_lock);
1735 	ret = pispbe_hw_init(pispbe);
1736 	if (ret)
1737 		goto pm_runtime_suspend_err;
1738 
1739 	ret = pispbe_init_devices(pispbe);
1740 	if (ret)
1741 		goto disable_devs_err;
1742 
1743 	pm_runtime_mark_last_busy(pispbe->dev);
1744 	pm_runtime_put_autosuspend(pispbe->dev);
1745 
1746 	return 0;
1747 
1748 disable_devs_err:
1749 	pispbe_destroy_devices(pispbe);
1750 pm_runtime_suspend_err:
1751 	pispbe_runtime_suspend(pispbe->dev);
1752 pm_runtime_disable_err:
1753 	pm_runtime_dont_use_autosuspend(pispbe->dev);
1754 	pm_runtime_disable(pispbe->dev);
1755 
1756 	return ret;
1757 }
1758 
1759 static void pispbe_remove(struct platform_device *pdev)
1760 {
1761 	struct pispbe_dev *pispbe = platform_get_drvdata(pdev);
1762 
1763 	pispbe_destroy_devices(pispbe);
1764 
1765 	pispbe_runtime_suspend(pispbe->dev);
1766 	pm_runtime_dont_use_autosuspend(pispbe->dev);
1767 	pm_runtime_disable(pispbe->dev);
1768 }
1769 
1770 static const struct dev_pm_ops pispbe_pm_ops = {
1771 	SET_RUNTIME_PM_OPS(pispbe_runtime_suspend, pispbe_runtime_resume, NULL)
1772 };
1773 
1774 static const struct of_device_id pispbe_of_match[] = {
1775 	{
1776 		.compatible = "raspberrypi,pispbe",
1777 	},
1778 	{ /* sentinel */ },
1779 };
1780 MODULE_DEVICE_TABLE(of, pispbe_of_match);
1781 
1782 static struct platform_driver pispbe_pdrv = {
1783 	.probe		= pispbe_probe,
1784 	.remove		= pispbe_remove,
1785 	.driver		= {
1786 		.name	= PISPBE_NAME,
1787 		.of_match_table = pispbe_of_match,
1788 		.pm = &pispbe_pm_ops,
1789 	},
1790 };
1791 
1792 module_platform_driver(pispbe_pdrv);
1793 
1794 MODULE_DESCRIPTION("PiSP Back End driver");
1795 MODULE_AUTHOR("David Plowman <david.plowman@raspberrypi.com>");
1796 MODULE_AUTHOR("Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>");
1797 MODULE_LICENSE("GPL");
1798