1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PiSP Back End driver. 4 * Copyright (c) 2021-2024 Raspberry Pi Limited. 5 * 6 */ 7 #include <linux/clk.h> 8 #include <linux/interrupt.h> 9 #include <linux/io.h> 10 #include <linux/kernel.h> 11 #include <linux/lockdep.h> 12 #include <linux/module.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <media/v4l2-device.h> 16 #include <media/v4l2-ioctl.h> 17 #include <media/videobuf2-dma-contig.h> 18 #include <media/videobuf2-vmalloc.h> 19 20 #include <uapi/linux/media/raspberrypi/pisp_be_config.h> 21 22 #include "pisp_be_formats.h" 23 24 /* Maximum number of config buffers possible */ 25 #define PISP_BE_NUM_CONFIG_BUFFERS VB2_MAX_FRAME 26 27 #define PISPBE_NAME "pispbe" 28 29 /* Some ISP-BE registers */ 30 #define PISP_BE_VERSION_REG 0x0 31 #define PISP_BE_CONTROL_REG 0x4 32 #define PISP_BE_CONTROL_COPY_CONFIG BIT(1) 33 #define PISP_BE_CONTROL_QUEUE_JOB BIT(0) 34 #define PISP_BE_CONTROL_NUM_TILES(n) ((n) << 16) 35 #define PISP_BE_TILE_ADDR_LO_REG 0x8 36 #define PISP_BE_TILE_ADDR_HI_REG 0xc 37 #define PISP_BE_STATUS_REG 0x10 38 #define PISP_BE_STATUS_QUEUED BIT(0) 39 #define PISP_BE_BATCH_STATUS_REG 0x14 40 #define PISP_BE_INTERRUPT_EN_REG 0x18 41 #define PISP_BE_INTERRUPT_STATUS_REG 0x1c 42 #define PISP_BE_AXI_REG 0x20 43 #define PISP_BE_CONFIG_BASE_REG 0x40 44 #define PISP_BE_IO_ADDR_LOW(n) (PISP_BE_CONFIG_BASE_REG + 8 * (n)) 45 #define PISP_BE_IO_ADDR_HIGH(n) (PISP_BE_IO_ADDR_LOW((n)) + 4) 46 #define PISP_BE_GLOBAL_BAYER_ENABLE 0xb0 47 #define PISP_BE_GLOBAL_RGB_ENABLE 0xb4 48 #define N_HW_ADDRESSES 13 49 #define N_HW_ENABLES 2 50 51 #define PISP_BE_VERSION_2712 0x02252700 52 #define PISP_BE_VERSION_MINOR_BITS 0xf 53 54 /* 55 * This maps our nodes onto the inputs/outputs of the actual PiSP Back End. 56 * Be wary of the word "OUTPUT" which is used ambiguously here. In a V4L2 57 * context it means an input to the hardware (source image or metadata). 58 * Elsewhere it means an output from the hardware. 59 */ 60 enum pispbe_node_ids { 61 MAIN_INPUT_NODE, 62 TDN_INPUT_NODE, 63 STITCH_INPUT_NODE, 64 OUTPUT0_NODE, 65 OUTPUT1_NODE, 66 TDN_OUTPUT_NODE, 67 STITCH_OUTPUT_NODE, 68 CONFIG_NODE, 69 PISPBE_NUM_NODES 70 }; 71 72 struct pispbe_node_description { 73 const char *ent_name; 74 enum v4l2_buf_type buf_type; 75 unsigned int caps; 76 }; 77 78 static const struct pispbe_node_description node_desc[PISPBE_NUM_NODES] = { 79 /* MAIN_INPUT_NODE */ 80 { 81 .ent_name = PISPBE_NAME "-input", 82 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, 83 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE, 84 }, 85 /* TDN_INPUT_NODE */ 86 { 87 .ent_name = PISPBE_NAME "-tdn_input", 88 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, 89 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE, 90 }, 91 /* STITCH_INPUT_NODE */ 92 { 93 .ent_name = PISPBE_NAME "-stitch_input", 94 .buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, 95 .caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE, 96 }, 97 /* OUTPUT0_NODE */ 98 { 99 .ent_name = PISPBE_NAME "-output0", 100 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 101 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE, 102 }, 103 /* OUTPUT1_NODE */ 104 { 105 .ent_name = PISPBE_NAME "-output1", 106 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 107 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE, 108 }, 109 /* TDN_OUTPUT_NODE */ 110 { 111 .ent_name = PISPBE_NAME "-tdn_output", 112 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 113 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE, 114 }, 115 /* STITCH_OUTPUT_NODE */ 116 { 117 .ent_name = PISPBE_NAME "-stitch_output", 118 .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 119 .caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE, 120 }, 121 /* CONFIG_NODE */ 122 { 123 .ent_name = PISPBE_NAME "-config", 124 .buf_type = V4L2_BUF_TYPE_META_OUTPUT, 125 .caps = V4L2_CAP_META_OUTPUT, 126 } 127 }; 128 129 #define NODE_DESC_IS_OUTPUT(desc) ( \ 130 ((desc)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \ 131 ((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \ 132 ((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)) 133 134 #define NODE_IS_META(node) ( \ 135 ((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT)) 136 #define NODE_IS_OUTPUT(node) ( \ 137 ((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \ 138 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \ 139 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)) 140 #define NODE_IS_CAPTURE(node) ( \ 141 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) || \ 142 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) 143 #define NODE_IS_MPLANE(node) ( \ 144 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) || \ 145 ((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) 146 147 /* 148 * Structure to describe a single node /dev/video<N> which represents a single 149 * input or output queue to the PiSP Back End device. 150 */ 151 struct pispbe_node { 152 unsigned int id; 153 int vfl_dir; 154 enum v4l2_buf_type buf_type; 155 struct video_device vfd; 156 struct media_pad pad; 157 struct media_intf_devnode *intf_devnode; 158 struct media_link *intf_link; 159 struct pispbe_dev *pispbe; 160 /* Video device lock */ 161 struct mutex node_lock; 162 /* vb2_queue lock */ 163 struct mutex queue_lock; 164 /* Protect pispbe_node->ready_queue and pispbe_buffer->ready_list */ 165 spinlock_t ready_lock; 166 struct list_head ready_queue; 167 struct vb2_queue queue; 168 struct v4l2_format format; 169 const struct pisp_be_format *pisp_format; 170 }; 171 172 /* For logging only, use the entity name with "pispbe" and separator removed */ 173 #define NODE_NAME(node) \ 174 (node_desc[(node)->id].ent_name + sizeof(PISPBE_NAME)) 175 176 /* Records details of the jobs currently running or queued on the h/w. */ 177 struct pispbe_job { 178 bool valid; 179 /* 180 * An array of buffer pointers - remember it's source buffers first, 181 * then captures, then metadata last. 182 */ 183 struct pispbe_buffer *buf[PISPBE_NUM_NODES]; 184 }; 185 186 struct pispbe_hw_enables { 187 u32 bayer_enables; 188 u32 rgb_enables; 189 }; 190 191 /* Records a job configuration and memory addresses. */ 192 struct pispbe_job_descriptor { 193 dma_addr_t hw_dma_addrs[N_HW_ADDRESSES]; 194 struct pisp_be_tiles_config *config; 195 struct pispbe_hw_enables hw_enables; 196 dma_addr_t tiles; 197 }; 198 199 /* 200 * Structure representing the entire PiSP Back End device, comprising several 201 * nodes which share platform resources and a mutex for the actual HW. 202 */ 203 struct pispbe_dev { 204 struct device *dev; 205 struct pispbe_dev *pispbe; 206 struct pisp_be_tiles_config *config; 207 void __iomem *be_reg_base; 208 struct clk *clk; 209 struct v4l2_device v4l2_dev; 210 struct v4l2_subdev sd; 211 struct media_device mdev; 212 struct media_pad pad[PISPBE_NUM_NODES]; /* output pads first */ 213 struct pispbe_node node[PISPBE_NUM_NODES]; 214 dma_addr_t config_dma_addr; 215 unsigned int sequence; 216 u32 streaming_map; 217 struct pispbe_job queued_job, running_job; 218 spinlock_t hw_lock; /* protects "hw_busy" flag and streaming_map */ 219 bool hw_busy; /* non-zero if a job is queued or is being started */ 220 int irq; 221 u32 hw_version; 222 u8 done, started; 223 }; 224 225 static u32 pispbe_rd(struct pispbe_dev *pispbe, unsigned int offset) 226 { 227 return readl(pispbe->be_reg_base + offset); 228 } 229 230 static void pispbe_wr(struct pispbe_dev *pispbe, unsigned int offset, u32 val) 231 { 232 writel(val, pispbe->be_reg_base + offset); 233 } 234 235 /* 236 * Queue a job to the h/w. If the h/w is idle it will begin immediately. 237 * Caller must ensure it is "safe to queue", i.e. we don't already have a 238 * queued, unstarted job. 239 */ 240 static void pispbe_queue_job(struct pispbe_dev *pispbe, 241 struct pispbe_job_descriptor *job) 242 { 243 unsigned int begin, end; 244 245 if (pispbe_rd(pispbe, PISP_BE_STATUS_REG) & PISP_BE_STATUS_QUEUED) 246 dev_err(pispbe->dev, "ERROR: not safe to queue new job!\n"); 247 248 /* 249 * Write configuration to hardware. DMA addresses and enable flags 250 * are passed separately, because the driver needs to sanitize them, 251 * and we don't want to modify (or be vulnerable to modifications of) 252 * the mmap'd buffer. 253 */ 254 for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) { 255 pispbe_wr(pispbe, PISP_BE_IO_ADDR_LOW(u), 256 lower_32_bits(job->hw_dma_addrs[u])); 257 pispbe_wr(pispbe, PISP_BE_IO_ADDR_HIGH(u), 258 upper_32_bits(job->hw_dma_addrs[u])); 259 } 260 pispbe_wr(pispbe, PISP_BE_GLOBAL_BAYER_ENABLE, 261 job->hw_enables.bayer_enables); 262 pispbe_wr(pispbe, PISP_BE_GLOBAL_RGB_ENABLE, 263 job->hw_enables.rgb_enables); 264 265 /* Everything else is as supplied by the user. */ 266 begin = offsetof(struct pisp_be_config, global.bayer_order) / 267 sizeof(u32); 268 end = sizeof(struct pisp_be_config) / sizeof(u32); 269 for (unsigned int u = begin; u < end; u++) 270 pispbe_wr(pispbe, PISP_BE_CONFIG_BASE_REG + sizeof(u32) * u, 271 ((u32 *)job->config)[u]); 272 273 /* Read back the addresses -- an error here could be fatal */ 274 for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) { 275 unsigned int offset = PISP_BE_IO_ADDR_LOW(u); 276 u64 along = pispbe_rd(pispbe, offset); 277 278 along += ((u64)pispbe_rd(pispbe, offset + 4)) << 32; 279 if (along != (u64)(job->hw_dma_addrs[u])) { 280 dev_dbg(pispbe->dev, 281 "ISP BE config error: check if ISP RAMs enabled?\n"); 282 return; 283 } 284 } 285 286 /* 287 * Write tile pointer to hardware. The IOMMU should prevent 288 * out-of-bounds offsets reaching non-ISP buffers. 289 */ 290 pispbe_wr(pispbe, PISP_BE_TILE_ADDR_LO_REG, lower_32_bits(job->tiles)); 291 pispbe_wr(pispbe, PISP_BE_TILE_ADDR_HI_REG, upper_32_bits(job->tiles)); 292 293 /* Enqueue the job */ 294 pispbe_wr(pispbe, PISP_BE_CONTROL_REG, 295 PISP_BE_CONTROL_COPY_CONFIG | PISP_BE_CONTROL_QUEUE_JOB | 296 PISP_BE_CONTROL_NUM_TILES(job->config->num_tiles)); 297 } 298 299 struct pispbe_buffer { 300 struct vb2_v4l2_buffer vb; 301 struct list_head ready_list; 302 unsigned int config_index; 303 }; 304 305 static int pispbe_get_planes_addr(dma_addr_t addr[3], struct pispbe_buffer *buf, 306 struct pispbe_node *node) 307 { 308 unsigned int num_planes = node->format.fmt.pix_mp.num_planes; 309 unsigned int plane_factor = 0; 310 unsigned int size; 311 unsigned int p; 312 313 if (!buf || !node->pisp_format) 314 return 0; 315 316 /* 317 * Determine the base plane size. This will not be the same 318 * as node->format.fmt.pix_mp.plane_fmt[0].sizeimage for a single 319 * plane buffer in an mplane format. 320 */ 321 size = node->format.fmt.pix_mp.plane_fmt[0].bytesperline * 322 node->format.fmt.pix_mp.height; 323 324 for (p = 0; p < num_planes && p < PISPBE_MAX_PLANES; p++) { 325 addr[p] = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, p); 326 plane_factor += node->pisp_format->plane_factor[p]; 327 } 328 329 for (; p < PISPBE_MAX_PLANES && node->pisp_format->plane_factor[p]; p++) { 330 /* 331 * Calculate the address offset of this plane as needed 332 * by the hardware. This is specifically for non-mplane 333 * buffer formats, where there are 3 image planes, e.g. 334 * for the V4L2_PIX_FMT_YUV420 format. 335 */ 336 addr[p] = addr[0] + ((size * plane_factor) >> 3); 337 plane_factor += node->pisp_format->plane_factor[p]; 338 } 339 340 return num_planes; 341 } 342 343 static dma_addr_t pispbe_get_addr(struct pispbe_buffer *buf) 344 { 345 if (buf) 346 return vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0); 347 348 return 0; 349 } 350 351 static void pispbe_xlate_addrs(struct pispbe_dev *pispbe, 352 struct pispbe_job_descriptor *job, 353 struct pispbe_buffer *buf[PISPBE_NUM_NODES]) 354 { 355 struct pispbe_hw_enables *hw_en = &job->hw_enables; 356 struct pisp_be_tiles_config *config = job->config; 357 dma_addr_t *addrs = job->hw_dma_addrs; 358 int ret; 359 360 /* Take a copy of the "enable" bitmaps so we can modify them. */ 361 hw_en->bayer_enables = config->config.global.bayer_enables; 362 hw_en->rgb_enables = config->config.global.rgb_enables; 363 364 /* 365 * Main input first. There are 3 address pointers, corresponding to up 366 * to 3 planes. 367 */ 368 ret = pispbe_get_planes_addr(addrs, buf[MAIN_INPUT_NODE], 369 &pispbe->node[MAIN_INPUT_NODE]); 370 if (ret <= 0) { 371 /* 372 * This shouldn't happen; pispbe_schedule_internal should insist 373 * on an input. 374 */ 375 dev_warn(pispbe->dev, "ISP-BE missing input\n"); 376 hw_en->bayer_enables = 0; 377 hw_en->rgb_enables = 0; 378 return; 379 } 380 381 /* 382 * Now TDN/Stitch inputs and outputs. These are single-plane and only 383 * used with Bayer input. Input enables must match the requirements 384 * of the processing stages, otherwise the hardware can lock up! 385 */ 386 if (hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) { 387 addrs[3] = pispbe_get_addr(buf[TDN_INPUT_NODE]); 388 if (addrs[3] == 0 || 389 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN_INPUT) || 390 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN) || 391 (config->config.tdn.reset & 1)) { 392 hw_en->bayer_enables &= 393 ~(PISP_BE_BAYER_ENABLE_TDN_INPUT | 394 PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS); 395 if (!(config->config.tdn.reset & 1)) 396 hw_en->bayer_enables &= 397 ~PISP_BE_BAYER_ENABLE_TDN; 398 } 399 400 addrs[4] = pispbe_get_addr(buf[STITCH_INPUT_NODE]); 401 if (addrs[4] == 0 || 402 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_INPUT) || 403 !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH)) { 404 hw_en->bayer_enables &= 405 ~(PISP_BE_BAYER_ENABLE_STITCH_INPUT | 406 PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS | 407 PISP_BE_BAYER_ENABLE_STITCH); 408 } 409 410 addrs[5] = pispbe_get_addr(buf[TDN_OUTPUT_NODE]); 411 if (addrs[5] == 0) 412 hw_en->bayer_enables &= 413 ~(PISP_BE_BAYER_ENABLE_TDN_COMPRESS | 414 PISP_BE_BAYER_ENABLE_TDN_OUTPUT); 415 416 addrs[6] = pispbe_get_addr(buf[STITCH_OUTPUT_NODE]); 417 if (addrs[6] == 0) 418 hw_en->bayer_enables &= 419 ~(PISP_BE_BAYER_ENABLE_STITCH_COMPRESS | 420 PISP_BE_BAYER_ENABLE_STITCH_OUTPUT); 421 } else { 422 /* No Bayer input? Disable entire Bayer pipe (else lockup) */ 423 hw_en->bayer_enables = 0; 424 } 425 426 /* Main image output channels. */ 427 for (unsigned int i = 0; i < PISP_BACK_END_NUM_OUTPUTS; i++) { 428 ret = pispbe_get_planes_addr(addrs + 7 + 3 * i, 429 buf[OUTPUT0_NODE + i], 430 &pispbe->node[OUTPUT0_NODE + i]); 431 if (ret <= 0) 432 hw_en->rgb_enables &= ~(PISP_BE_RGB_ENABLE_OUTPUT0 << i); 433 } 434 } 435 436 /* 437 * Prepare a job description to be submitted to the HW. 438 * 439 * To schedule a job, we need all streaming nodes (apart from Output0, 440 * Output1, Tdn and Stitch) to have a buffer ready, which must 441 * include at least a config buffer and a main input image. 442 * 443 * For Output0, Output1, Tdn and Stitch, a buffer only needs to be 444 * available if the blocks are enabled in the config. 445 * 446 * Needs to be called with hw_lock held. 447 * 448 * Returns 0 if a job has been successfully prepared, < 0 otherwise. 449 */ 450 static int pispbe_prepare_job(struct pispbe_dev *pispbe, 451 struct pispbe_job_descriptor *job) 452 { 453 struct pispbe_buffer *buf[PISPBE_NUM_NODES] = {}; 454 unsigned int config_index; 455 struct pispbe_node *node; 456 unsigned long flags; 457 458 lockdep_assert_held(&pispbe->hw_lock); 459 460 memset(job, 0, sizeof(struct pispbe_job_descriptor)); 461 462 if (((BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE)) & 463 pispbe->streaming_map) != 464 (BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE))) 465 return -ENODEV; 466 467 node = &pispbe->node[CONFIG_NODE]; 468 spin_lock_irqsave(&node->ready_lock, flags); 469 buf[CONFIG_NODE] = list_first_entry_or_null(&node->ready_queue, 470 struct pispbe_buffer, 471 ready_list); 472 if (buf[CONFIG_NODE]) { 473 list_del(&buf[CONFIG_NODE]->ready_list); 474 pispbe->queued_job.buf[CONFIG_NODE] = buf[CONFIG_NODE]; 475 } 476 spin_unlock_irqrestore(&node->ready_lock, flags); 477 478 /* Exit early if no config buffer has been queued. */ 479 if (!buf[CONFIG_NODE]) 480 return -ENODEV; 481 482 config_index = buf[CONFIG_NODE]->vb.vb2_buf.index; 483 job->config = &pispbe->config[config_index]; 484 job->tiles = pispbe->config_dma_addr + 485 config_index * sizeof(struct pisp_be_tiles_config) + 486 offsetof(struct pisp_be_tiles_config, tiles); 487 488 /* remember: srcimages, captures then metadata */ 489 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) { 490 unsigned int bayer_en = 491 job->config->config.global.bayer_enables; 492 unsigned int rgb_en = 493 job->config->config.global.rgb_enables; 494 bool ignore_buffers = false; 495 496 /* Config node is handled outside the loop above. */ 497 if (i == CONFIG_NODE) 498 continue; 499 500 buf[i] = NULL; 501 if (!(pispbe->streaming_map & BIT(i))) 502 continue; 503 504 if ((!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT0) && 505 i == OUTPUT0_NODE) || 506 (!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT1) && 507 i == OUTPUT1_NODE) || 508 (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_INPUT) && 509 i == TDN_INPUT_NODE) || 510 (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) && 511 i == TDN_OUTPUT_NODE) || 512 (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_INPUT) && 513 i == STITCH_INPUT_NODE) || 514 (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) && 515 i == STITCH_OUTPUT_NODE)) { 516 /* 517 * Ignore Output0/Output1/Tdn/Stitch buffer check if the 518 * global enables aren't set for these blocks. If a 519 * buffer has been provided, we dequeue it back to the 520 * user with the other in-use buffers. 521 */ 522 ignore_buffers = true; 523 } 524 525 node = &pispbe->node[i]; 526 527 /* Pull a buffer from each V4L2 queue to form the queued job */ 528 spin_lock_irqsave(&node->ready_lock, flags); 529 buf[i] = list_first_entry_or_null(&node->ready_queue, 530 struct pispbe_buffer, 531 ready_list); 532 if (buf[i]) { 533 list_del(&buf[i]->ready_list); 534 pispbe->queued_job.buf[i] = buf[i]; 535 } 536 spin_unlock_irqrestore(&node->ready_lock, flags); 537 538 if (!buf[i] && !ignore_buffers) 539 goto err_return_buffers; 540 } 541 542 pispbe->queued_job.valid = true; 543 544 /* Convert buffers to DMA addresses for the hardware */ 545 pispbe_xlate_addrs(pispbe, job, buf); 546 547 return 0; 548 549 err_return_buffers: 550 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) { 551 struct pispbe_node *n = &pispbe->node[i]; 552 553 if (!buf[i]) 554 continue; 555 556 /* Return the buffer to the ready_list queue */ 557 spin_lock_irqsave(&n->ready_lock, flags); 558 list_add(&buf[i]->ready_list, &n->ready_queue); 559 spin_unlock_irqrestore(&n->ready_lock, flags); 560 } 561 562 memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job)); 563 564 return -ENODEV; 565 } 566 567 static void pispbe_schedule(struct pispbe_dev *pispbe, bool clear_hw_busy) 568 { 569 struct pispbe_job_descriptor job; 570 unsigned long flags; 571 int ret; 572 573 spin_lock_irqsave(&pispbe->hw_lock, flags); 574 575 if (clear_hw_busy) 576 pispbe->hw_busy = false; 577 578 if (pispbe->hw_busy) 579 goto unlock_and_return; 580 581 ret = pispbe_prepare_job(pispbe, &job); 582 if (ret) 583 goto unlock_and_return; 584 585 /* 586 * We can kick the job off without the hw_lock, as this can 587 * never run again until hw_busy is cleared, which will happen 588 * only when the following job has been queued and an interrupt 589 * is rised. 590 */ 591 pispbe->hw_busy = true; 592 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 593 594 if (job.config->num_tiles <= 0 || 595 job.config->num_tiles > PISP_BACK_END_NUM_TILES || 596 !((job.hw_enables.bayer_enables | job.hw_enables.rgb_enables) & 597 PISP_BE_BAYER_ENABLE_INPUT)) { 598 /* 599 * Bad job. We can't let it proceed as it could lock up 600 * the hardware, or worse! 601 * 602 * For now, just force num_tiles to 0, which causes the 603 * H/W to do something bizarre but survivable. It 604 * increments (started,done) counters by more than 1, 605 * but we seem to survive... 606 */ 607 dev_dbg(pispbe->dev, "Bad job: invalid number of tiles: %u\n", 608 job.config->num_tiles); 609 job.config->num_tiles = 0; 610 } 611 612 pispbe_queue_job(pispbe, &job); 613 614 return; 615 616 unlock_and_return: 617 /* No job has been queued, just release the lock and return. */ 618 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 619 } 620 621 static void pispbe_isr_jobdone(struct pispbe_dev *pispbe, 622 struct pispbe_job *job) 623 { 624 struct pispbe_buffer **buf = job->buf; 625 u64 ts = ktime_get_ns(); 626 627 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) { 628 if (buf[i]) { 629 buf[i]->vb.vb2_buf.timestamp = ts; 630 buf[i]->vb.sequence = pispbe->sequence; 631 vb2_buffer_done(&buf[i]->vb.vb2_buf, 632 VB2_BUF_STATE_DONE); 633 } 634 } 635 636 pispbe->sequence++; 637 } 638 639 static irqreturn_t pispbe_isr(int irq, void *dev) 640 { 641 struct pispbe_dev *pispbe = (struct pispbe_dev *)dev; 642 bool can_queue_another = false; 643 u8 started, done; 644 u32 u; 645 646 u = pispbe_rd(pispbe, PISP_BE_INTERRUPT_STATUS_REG); 647 if (u == 0) 648 return IRQ_NONE; 649 650 pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, u); 651 u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG); 652 done = (uint8_t)u; 653 started = (uint8_t)(u >> 8); 654 655 /* 656 * Be aware that done can go up by 2 and started by 1 when: a job that 657 * we previously saw "start" now finishes, and we then queued a new job 658 * which we see both start and finish "simultaneously". 659 */ 660 if (pispbe->running_job.valid && pispbe->done != done) { 661 pispbe_isr_jobdone(pispbe, &pispbe->running_job); 662 memset(&pispbe->running_job, 0, sizeof(pispbe->running_job)); 663 pispbe->done++; 664 } 665 666 if (pispbe->started != started) { 667 pispbe->started++; 668 can_queue_another = 1; 669 670 if (pispbe->done != done && pispbe->queued_job.valid) { 671 pispbe_isr_jobdone(pispbe, &pispbe->queued_job); 672 pispbe->done++; 673 } else { 674 pispbe->running_job = pispbe->queued_job; 675 } 676 677 memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job)); 678 } 679 680 if (pispbe->done != done || pispbe->started != started) { 681 dev_dbg(pispbe->dev, 682 "Job counters not matching: done = %u, expected %u - started = %u, expected %u\n", 683 pispbe->done, done, pispbe->started, started); 684 pispbe->started = started; 685 pispbe->done = done; 686 } 687 688 /* check if there's more to do before going to sleep */ 689 pispbe_schedule(pispbe, can_queue_another); 690 691 return IRQ_HANDLED; 692 } 693 694 static int pisp_be_validate_config(struct pispbe_dev *pispbe, 695 struct pisp_be_tiles_config *config) 696 { 697 u32 bayer_enables = config->config.global.bayer_enables; 698 u32 rgb_enables = config->config.global.rgb_enables; 699 struct device *dev = pispbe->dev; 700 struct v4l2_format *fmt; 701 unsigned int bpl, size; 702 703 if (!(bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) == 704 !(rgb_enables & PISP_BE_RGB_ENABLE_INPUT)) { 705 dev_dbg(dev, "%s: Not one input enabled\n", __func__); 706 return -EIO; 707 } 708 709 /* Ensure output config strides and buffer sizes match the V4L2 formats. */ 710 fmt = &pispbe->node[TDN_OUTPUT_NODE].format; 711 if (bayer_enables & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) { 712 bpl = config->config.tdn_output_format.stride; 713 size = bpl * config->config.tdn_output_format.height; 714 715 if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) { 716 dev_dbg(dev, "%s: bpl mismatch on tdn_output\n", 717 __func__); 718 return -EINVAL; 719 } 720 721 if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) { 722 dev_dbg(dev, "%s: size mismatch on tdn_output\n", 723 __func__); 724 return -EINVAL; 725 } 726 } 727 728 fmt = &pispbe->node[STITCH_OUTPUT_NODE].format; 729 if (bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) { 730 bpl = config->config.stitch_output_format.stride; 731 size = bpl * config->config.stitch_output_format.height; 732 733 if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) { 734 dev_dbg(dev, "%s: bpl mismatch on stitch_output\n", 735 __func__); 736 return -EINVAL; 737 } 738 739 if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) { 740 dev_dbg(dev, "%s: size mismatch on stitch_output\n", 741 __func__); 742 return -EINVAL; 743 } 744 } 745 746 for (unsigned int j = 0; j < PISP_BACK_END_NUM_OUTPUTS; j++) { 747 if (!(rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT(j))) 748 continue; 749 750 if (config->config.output_format[j].image.format & 751 PISP_IMAGE_FORMAT_WALLPAPER_ROLL) 752 continue; /* TODO: Size checks for wallpaper formats */ 753 754 fmt = &pispbe->node[OUTPUT0_NODE + j].format; 755 for (unsigned int i = 0; i < fmt->fmt.pix_mp.num_planes; i++) { 756 bpl = !i ? config->config.output_format[j].image.stride 757 : config->config.output_format[j].image.stride2; 758 size = bpl * config->config.output_format[j].image.height; 759 760 if (config->config.output_format[j].image.format & 761 PISP_IMAGE_FORMAT_SAMPLING_420) 762 size >>= 1; 763 764 if (fmt->fmt.pix_mp.plane_fmt[i].bytesperline < bpl) { 765 dev_dbg(dev, "%s: bpl mismatch on output %d\n", 766 __func__, j); 767 return -EINVAL; 768 } 769 770 if (fmt->fmt.pix_mp.plane_fmt[i].sizeimage < size) { 771 dev_dbg(dev, "%s: size mismatch on output\n", 772 __func__); 773 return -EINVAL; 774 } 775 } 776 } 777 778 return 0; 779 } 780 781 static int pispbe_node_queue_setup(struct vb2_queue *q, unsigned int *nbuffers, 782 unsigned int *nplanes, unsigned int sizes[], 783 struct device *alloc_devs[]) 784 { 785 struct pispbe_node *node = vb2_get_drv_priv(q); 786 struct pispbe_dev *pispbe = node->pispbe; 787 unsigned int num_planes = NODE_IS_MPLANE(node) ? 788 node->format.fmt.pix_mp.num_planes : 1; 789 790 if (*nplanes) { 791 if (*nplanes != num_planes) 792 return -EINVAL; 793 794 for (unsigned int i = 0; i < *nplanes; i++) { 795 unsigned int size = NODE_IS_MPLANE(node) ? 796 node->format.fmt.pix_mp.plane_fmt[i].sizeimage : 797 node->format.fmt.meta.buffersize; 798 799 if (sizes[i] < size) 800 return -EINVAL; 801 } 802 803 return 0; 804 } 805 806 *nplanes = num_planes; 807 for (unsigned int i = 0; i < *nplanes; i++) { 808 unsigned int size = NODE_IS_MPLANE(node) ? 809 node->format.fmt.pix_mp.plane_fmt[i].sizeimage : 810 node->format.fmt.meta.buffersize; 811 sizes[i] = size; 812 } 813 814 dev_dbg(pispbe->dev, 815 "Image (or metadata) size %u, nbuffers %u for node %s\n", 816 sizes[0], *nbuffers, NODE_NAME(node)); 817 818 return 0; 819 } 820 821 static int pispbe_node_buffer_prepare(struct vb2_buffer *vb) 822 { 823 struct pispbe_node *node = vb2_get_drv_priv(vb->vb2_queue); 824 struct pispbe_dev *pispbe = node->pispbe; 825 unsigned int num_planes = NODE_IS_MPLANE(node) ? 826 node->format.fmt.pix_mp.num_planes : 1; 827 828 for (unsigned int i = 0; i < num_planes; i++) { 829 unsigned long size = NODE_IS_MPLANE(node) ? 830 node->format.fmt.pix_mp.plane_fmt[i].sizeimage : 831 node->format.fmt.meta.buffersize; 832 833 if (vb2_plane_size(vb, i) < size) { 834 dev_dbg(pispbe->dev, 835 "data will not fit into plane %d (%lu < %lu)\n", 836 i, vb2_plane_size(vb, i), size); 837 return -EINVAL; 838 } 839 840 vb2_set_plane_payload(vb, i, size); 841 } 842 843 if (node->id == CONFIG_NODE) { 844 void *dst = &node->pispbe->config[vb->index]; 845 void *src = vb2_plane_vaddr(vb, 0); 846 847 memcpy(dst, src, sizeof(struct pisp_be_tiles_config)); 848 849 return pisp_be_validate_config(pispbe, dst); 850 } 851 852 return 0; 853 } 854 855 static void pispbe_node_buffer_queue(struct vb2_buffer *buf) 856 { 857 struct vb2_v4l2_buffer *vbuf = 858 container_of(buf, struct vb2_v4l2_buffer, vb2_buf); 859 struct pispbe_buffer *buffer = 860 container_of(vbuf, struct pispbe_buffer, vb); 861 struct pispbe_node *node = vb2_get_drv_priv(buf->vb2_queue); 862 struct pispbe_dev *pispbe = node->pispbe; 863 unsigned long flags; 864 865 dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node)); 866 spin_lock_irqsave(&node->ready_lock, flags); 867 list_add_tail(&buffer->ready_list, &node->ready_queue); 868 spin_unlock_irqrestore(&node->ready_lock, flags); 869 870 /* 871 * Every time we add a buffer, check if there's now some work for the hw 872 * to do. 873 */ 874 pispbe_schedule(pispbe, false); 875 } 876 877 static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count) 878 { 879 struct pispbe_node *node = vb2_get_drv_priv(q); 880 struct pispbe_dev *pispbe = node->pispbe; 881 struct pispbe_buffer *buf, *tmp; 882 unsigned long flags; 883 int ret; 884 885 ret = pm_runtime_resume_and_get(pispbe->dev); 886 if (ret < 0) 887 goto err_return_buffers; 888 889 spin_lock_irqsave(&pispbe->hw_lock, flags); 890 node->pispbe->streaming_map |= BIT(node->id); 891 node->pispbe->sequence = 0; 892 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 893 894 dev_dbg(pispbe->dev, "%s: for node %s (count %u)\n", 895 __func__, NODE_NAME(node), count); 896 dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n", 897 node->pispbe->streaming_map); 898 899 /* Maybe we're ready to run. */ 900 pispbe_schedule(pispbe, false); 901 902 return 0; 903 904 err_return_buffers: 905 spin_lock_irqsave(&pispbe->hw_lock, flags); 906 list_for_each_entry_safe(buf, tmp, &node->ready_queue, ready_list) { 907 list_del(&buf->ready_list); 908 vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED); 909 } 910 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 911 912 return ret; 913 } 914 915 static void pispbe_node_stop_streaming(struct vb2_queue *q) 916 { 917 struct pispbe_node *node = vb2_get_drv_priv(q); 918 struct pispbe_dev *pispbe = node->pispbe; 919 struct pispbe_buffer *buf; 920 unsigned long flags; 921 922 /* 923 * Now this is a bit awkward. In a simple M2M device we could just wait 924 * for all queued jobs to complete, but here there's a risk that a 925 * partial set of buffers was queued and cannot be run. For now, just 926 * cancel all buffers stuck in the "ready queue", then wait for any 927 * running job. 928 * 929 * This may return buffers out of order. 930 */ 931 dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node)); 932 spin_lock_irqsave(&pispbe->hw_lock, flags); 933 do { 934 unsigned long flags1; 935 936 spin_lock_irqsave(&node->ready_lock, flags1); 937 buf = list_first_entry_or_null(&node->ready_queue, 938 struct pispbe_buffer, 939 ready_list); 940 if (buf) { 941 list_del(&buf->ready_list); 942 vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); 943 } 944 spin_unlock_irqrestore(&node->ready_lock, flags1); 945 } while (buf); 946 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 947 948 vb2_wait_for_all_buffers(&node->queue); 949 950 spin_lock_irqsave(&pispbe->hw_lock, flags); 951 pispbe->streaming_map &= ~BIT(node->id); 952 spin_unlock_irqrestore(&pispbe->hw_lock, flags); 953 954 pm_runtime_mark_last_busy(pispbe->dev); 955 pm_runtime_put_autosuspend(pispbe->dev); 956 957 dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n", 958 pispbe->streaming_map); 959 } 960 961 static const struct vb2_ops pispbe_node_queue_ops = { 962 .queue_setup = pispbe_node_queue_setup, 963 .buf_prepare = pispbe_node_buffer_prepare, 964 .buf_queue = pispbe_node_buffer_queue, 965 .start_streaming = pispbe_node_start_streaming, 966 .stop_streaming = pispbe_node_stop_streaming, 967 .wait_prepare = vb2_ops_wait_prepare, 968 .wait_finish = vb2_ops_wait_finish, 969 }; 970 971 static const struct v4l2_file_operations pispbe_fops = { 972 .owner = THIS_MODULE, 973 .open = v4l2_fh_open, 974 .release = vb2_fop_release, 975 .poll = vb2_fop_poll, 976 .unlocked_ioctl = video_ioctl2, 977 .mmap = vb2_fop_mmap 978 }; 979 980 static int pispbe_node_querycap(struct file *file, void *priv, 981 struct v4l2_capability *cap) 982 { 983 struct pispbe_node *node = video_drvdata(file); 984 struct pispbe_dev *pispbe = node->pispbe; 985 986 strscpy(cap->driver, PISPBE_NAME, sizeof(cap->driver)); 987 strscpy(cap->card, PISPBE_NAME, sizeof(cap->card)); 988 989 dev_dbg(pispbe->dev, "Caps for node %s: %x and %x (dev %x)\n", 990 NODE_NAME(node), cap->capabilities, cap->device_caps, 991 node->vfd.device_caps); 992 993 return 0; 994 } 995 996 static int pispbe_node_g_fmt_vid_cap(struct file *file, void *priv, 997 struct v4l2_format *f) 998 { 999 struct pispbe_node *node = video_drvdata(file); 1000 struct pispbe_dev *pispbe = node->pispbe; 1001 1002 if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) { 1003 dev_dbg(pispbe->dev, 1004 "Cannot get capture fmt for output node %s\n", 1005 NODE_NAME(node)); 1006 return -EINVAL; 1007 } 1008 1009 *f = node->format; 1010 dev_dbg(pispbe->dev, "Get capture format for node %s\n", 1011 NODE_NAME(node)); 1012 1013 return 0; 1014 } 1015 1016 static int pispbe_node_g_fmt_vid_out(struct file *file, void *priv, 1017 struct v4l2_format *f) 1018 { 1019 struct pispbe_node *node = video_drvdata(file); 1020 struct pispbe_dev *pispbe = node->pispbe; 1021 1022 if (NODE_IS_CAPTURE(node) || NODE_IS_META(node)) { 1023 dev_dbg(pispbe->dev, 1024 "Cannot get capture fmt for output node %s\n", 1025 NODE_NAME(node)); 1026 return -EINVAL; 1027 } 1028 1029 *f = node->format; 1030 dev_dbg(pispbe->dev, "Get output format for node %s\n", 1031 NODE_NAME(node)); 1032 1033 return 0; 1034 } 1035 1036 static int pispbe_node_g_fmt_meta_out(struct file *file, void *priv, 1037 struct v4l2_format *f) 1038 { 1039 struct pispbe_node *node = video_drvdata(file); 1040 struct pispbe_dev *pispbe = node->pispbe; 1041 1042 if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) { 1043 dev_dbg(pispbe->dev, 1044 "Cannot get capture fmt for meta output node %s\n", 1045 NODE_NAME(node)); 1046 return -EINVAL; 1047 } 1048 1049 *f = node->format; 1050 dev_dbg(pispbe->dev, "Get output format for meta node %s\n", 1051 NODE_NAME(node)); 1052 1053 return 0; 1054 } 1055 1056 static const struct pisp_be_format *pispbe_find_fmt(unsigned int fourcc) 1057 { 1058 for (unsigned int i = 0; i < ARRAY_SIZE(supported_formats); i++) { 1059 if (supported_formats[i].fourcc == fourcc) 1060 return &supported_formats[i]; 1061 } 1062 1063 return NULL; 1064 } 1065 1066 static void pispbe_set_plane_params(struct v4l2_format *f, 1067 const struct pisp_be_format *fmt) 1068 { 1069 unsigned int nplanes = f->fmt.pix_mp.num_planes; 1070 unsigned int total_plane_factor = 0; 1071 1072 for (unsigned int i = 0; i < PISPBE_MAX_PLANES; i++) 1073 total_plane_factor += fmt->plane_factor[i]; 1074 1075 for (unsigned int i = 0; i < nplanes; i++) { 1076 struct v4l2_plane_pix_format *p = &f->fmt.pix_mp.plane_fmt[i]; 1077 unsigned int bpl, plane_size; 1078 1079 bpl = (f->fmt.pix_mp.width * fmt->bit_depth) >> 3; 1080 bpl = ALIGN(max(p->bytesperline, bpl), fmt->align); 1081 1082 plane_size = bpl * f->fmt.pix_mp.height * 1083 (nplanes > 1 ? fmt->plane_factor[i] : total_plane_factor); 1084 /* 1085 * The shift is to divide out the plane_factor fixed point 1086 * scaling of 8. 1087 */ 1088 plane_size = max(p->sizeimage, plane_size >> 3); 1089 1090 p->bytesperline = bpl; 1091 p->sizeimage = plane_size; 1092 } 1093 } 1094 1095 static void pispbe_try_format(struct v4l2_format *f, struct pispbe_node *node) 1096 { 1097 struct pispbe_dev *pispbe = node->pispbe; 1098 u32 pixfmt = f->fmt.pix_mp.pixelformat; 1099 const struct pisp_be_format *fmt; 1100 bool is_rgb; 1101 1102 dev_dbg(pispbe->dev, 1103 "%s: [%s] req %ux%u %p4cc, planes %d\n", 1104 __func__, NODE_NAME(node), f->fmt.pix_mp.width, 1105 f->fmt.pix_mp.height, &pixfmt, 1106 f->fmt.pix_mp.num_planes); 1107 1108 fmt = pispbe_find_fmt(pixfmt); 1109 if (!fmt) { 1110 dev_dbg(pispbe->dev, 1111 "%s: [%s] Format not found, defaulting to YUV420\n", 1112 __func__, NODE_NAME(node)); 1113 fmt = pispbe_find_fmt(V4L2_PIX_FMT_YUV420); 1114 } 1115 1116 f->fmt.pix_mp.pixelformat = fmt->fourcc; 1117 f->fmt.pix_mp.num_planes = fmt->num_planes; 1118 f->fmt.pix_mp.field = V4L2_FIELD_NONE; 1119 f->fmt.pix_mp.width = max(min(f->fmt.pix_mp.width, 65536u), 1120 PISP_BACK_END_MIN_TILE_WIDTH); 1121 f->fmt.pix_mp.height = max(min(f->fmt.pix_mp.height, 65536u), 1122 PISP_BACK_END_MIN_TILE_HEIGHT); 1123 1124 /* 1125 * Fill in the actual colour space when the requested one was 1126 * not supported. This also catches the case when the "default" 1127 * colour space was requested (as that's never in the mask). 1128 */ 1129 if (!(V4L2_COLORSPACE_MASK(f->fmt.pix_mp.colorspace) & 1130 fmt->colorspace_mask)) 1131 f->fmt.pix_mp.colorspace = fmt->colorspace_default; 1132 1133 /* In all cases, we only support the defaults for these: */ 1134 f->fmt.pix_mp.ycbcr_enc = 1135 V4L2_MAP_YCBCR_ENC_DEFAULT(f->fmt.pix_mp.colorspace); 1136 f->fmt.pix_mp.xfer_func = 1137 V4L2_MAP_XFER_FUNC_DEFAULT(f->fmt.pix_mp.colorspace); 1138 1139 is_rgb = f->fmt.pix_mp.colorspace == V4L2_COLORSPACE_SRGB; 1140 f->fmt.pix_mp.quantization = 1141 V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, f->fmt.pix_mp.colorspace, 1142 f->fmt.pix_mp.ycbcr_enc); 1143 1144 /* Set plane size and bytes/line for each plane. */ 1145 pispbe_set_plane_params(f, fmt); 1146 1147 for (unsigned int i = 0; i < f->fmt.pix_mp.num_planes; i++) { 1148 dev_dbg(pispbe->dev, 1149 "%s: [%s] calc plane %d, %ux%u, depth %u, bpl %u size %u\n", 1150 __func__, NODE_NAME(node), i, f->fmt.pix_mp.width, 1151 f->fmt.pix_mp.height, fmt->bit_depth, 1152 f->fmt.pix_mp.plane_fmt[i].bytesperline, 1153 f->fmt.pix_mp.plane_fmt[i].sizeimage); 1154 } 1155 } 1156 1157 static int pispbe_node_try_fmt_vid_cap(struct file *file, void *priv, 1158 struct v4l2_format *f) 1159 { 1160 struct pispbe_node *node = video_drvdata(file); 1161 struct pispbe_dev *pispbe = node->pispbe; 1162 1163 if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) { 1164 dev_dbg(pispbe->dev, 1165 "Cannot set capture fmt for output node %s\n", 1166 NODE_NAME(node)); 1167 return -EINVAL; 1168 } 1169 1170 pispbe_try_format(f, node); 1171 1172 return 0; 1173 } 1174 1175 static int pispbe_node_try_fmt_vid_out(struct file *file, void *priv, 1176 struct v4l2_format *f) 1177 { 1178 struct pispbe_node *node = video_drvdata(file); 1179 struct pispbe_dev *pispbe = node->pispbe; 1180 1181 if (!NODE_IS_OUTPUT(node) || NODE_IS_META(node)) { 1182 dev_dbg(pispbe->dev, 1183 "Cannot set capture fmt for output node %s\n", 1184 NODE_NAME(node)); 1185 return -EINVAL; 1186 } 1187 1188 pispbe_try_format(f, node); 1189 1190 return 0; 1191 } 1192 1193 static int pispbe_node_try_fmt_meta_out(struct file *file, void *priv, 1194 struct v4l2_format *f) 1195 { 1196 struct pispbe_node *node = video_drvdata(file); 1197 struct pispbe_dev *pispbe = node->pispbe; 1198 1199 if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) { 1200 dev_dbg(pispbe->dev, 1201 "Cannot set capture fmt for meta output node %s\n", 1202 NODE_NAME(node)); 1203 return -EINVAL; 1204 } 1205 1206 f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG; 1207 f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config); 1208 1209 return 0; 1210 } 1211 1212 static int pispbe_node_s_fmt_vid_cap(struct file *file, void *priv, 1213 struct v4l2_format *f) 1214 { 1215 struct pispbe_node *node = video_drvdata(file); 1216 struct pispbe_dev *pispbe = node->pispbe; 1217 int ret; 1218 1219 ret = pispbe_node_try_fmt_vid_cap(file, priv, f); 1220 if (ret < 0) 1221 return ret; 1222 1223 if (vb2_is_busy(&node->queue)) 1224 return -EBUSY; 1225 1226 node->format = *f; 1227 node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat); 1228 1229 dev_dbg(pispbe->dev, "Set capture format for node %s to %p4cc\n", 1230 NODE_NAME(node), &f->fmt.pix_mp.pixelformat); 1231 1232 return 0; 1233 } 1234 1235 static int pispbe_node_s_fmt_vid_out(struct file *file, void *priv, 1236 struct v4l2_format *f) 1237 { 1238 struct pispbe_node *node = video_drvdata(file); 1239 struct pispbe_dev *pispbe = node->pispbe; 1240 int ret; 1241 1242 ret = pispbe_node_try_fmt_vid_out(file, priv, f); 1243 if (ret < 0) 1244 return ret; 1245 1246 if (vb2_is_busy(&node->queue)) 1247 return -EBUSY; 1248 1249 node->format = *f; 1250 node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat); 1251 1252 dev_dbg(pispbe->dev, "Set output format for node %s to %p4cc\n", 1253 NODE_NAME(node), &f->fmt.pix_mp.pixelformat); 1254 1255 return 0; 1256 } 1257 1258 static int pispbe_node_s_fmt_meta_out(struct file *file, void *priv, 1259 struct v4l2_format *f) 1260 { 1261 struct pispbe_node *node = video_drvdata(file); 1262 struct pispbe_dev *pispbe = node->pispbe; 1263 int ret; 1264 1265 ret = pispbe_node_try_fmt_meta_out(file, priv, f); 1266 if (ret < 0) 1267 return ret; 1268 1269 if (vb2_is_busy(&node->queue)) 1270 return -EBUSY; 1271 1272 node->format = *f; 1273 node->pisp_format = &meta_out_supported_formats[0]; 1274 1275 dev_dbg(pispbe->dev, "Set output format for meta node %s to %p4cc\n", 1276 NODE_NAME(node), &f->fmt.meta.dataformat); 1277 1278 return 0; 1279 } 1280 1281 static int pispbe_node_enum_fmt(struct file *file, void *priv, 1282 struct v4l2_fmtdesc *f) 1283 { 1284 struct pispbe_node *node = video_drvdata(file); 1285 1286 if (f->type != node->queue.type) 1287 return -EINVAL; 1288 1289 if (NODE_IS_META(node)) { 1290 if (f->index) 1291 return -EINVAL; 1292 1293 f->pixelformat = V4L2_META_FMT_RPI_BE_CFG; 1294 f->flags = 0; 1295 return 0; 1296 } 1297 1298 if (f->index >= ARRAY_SIZE(supported_formats)) 1299 return -EINVAL; 1300 1301 f->pixelformat = supported_formats[f->index].fourcc; 1302 f->flags = 0; 1303 1304 return 0; 1305 } 1306 1307 static int pispbe_enum_framesizes(struct file *file, void *priv, 1308 struct v4l2_frmsizeenum *fsize) 1309 { 1310 struct pispbe_node *node = video_drvdata(file); 1311 struct pispbe_dev *pispbe = node->pispbe; 1312 1313 if (NODE_IS_META(node) || fsize->index) 1314 return -EINVAL; 1315 1316 if (!pispbe_find_fmt(fsize->pixel_format)) { 1317 dev_dbg(pispbe->dev, "Invalid pixel code: %x\n", 1318 fsize->pixel_format); 1319 return -EINVAL; 1320 } 1321 1322 fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE; 1323 fsize->stepwise.min_width = 32; 1324 fsize->stepwise.max_width = 65535; 1325 fsize->stepwise.step_width = 2; 1326 1327 fsize->stepwise.min_height = 32; 1328 fsize->stepwise.max_height = 65535; 1329 fsize->stepwise.step_height = 2; 1330 1331 return 0; 1332 } 1333 1334 static const struct v4l2_ioctl_ops pispbe_node_ioctl_ops = { 1335 .vidioc_querycap = pispbe_node_querycap, 1336 .vidioc_g_fmt_vid_cap_mplane = pispbe_node_g_fmt_vid_cap, 1337 .vidioc_g_fmt_vid_out_mplane = pispbe_node_g_fmt_vid_out, 1338 .vidioc_g_fmt_meta_out = pispbe_node_g_fmt_meta_out, 1339 .vidioc_try_fmt_vid_cap_mplane = pispbe_node_try_fmt_vid_cap, 1340 .vidioc_try_fmt_vid_out_mplane = pispbe_node_try_fmt_vid_out, 1341 .vidioc_try_fmt_meta_out = pispbe_node_try_fmt_meta_out, 1342 .vidioc_s_fmt_vid_cap_mplane = pispbe_node_s_fmt_vid_cap, 1343 .vidioc_s_fmt_vid_out_mplane = pispbe_node_s_fmt_vid_out, 1344 .vidioc_s_fmt_meta_out = pispbe_node_s_fmt_meta_out, 1345 .vidioc_enum_fmt_vid_cap = pispbe_node_enum_fmt, 1346 .vidioc_enum_fmt_vid_out = pispbe_node_enum_fmt, 1347 .vidioc_enum_fmt_meta_out = pispbe_node_enum_fmt, 1348 .vidioc_enum_framesizes = pispbe_enum_framesizes, 1349 .vidioc_create_bufs = vb2_ioctl_create_bufs, 1350 .vidioc_prepare_buf = vb2_ioctl_prepare_buf, 1351 .vidioc_querybuf = vb2_ioctl_querybuf, 1352 .vidioc_qbuf = vb2_ioctl_qbuf, 1353 .vidioc_dqbuf = vb2_ioctl_dqbuf, 1354 .vidioc_expbuf = vb2_ioctl_expbuf, 1355 .vidioc_reqbufs = vb2_ioctl_reqbufs, 1356 .vidioc_streamon = vb2_ioctl_streamon, 1357 .vidioc_streamoff = vb2_ioctl_streamoff, 1358 }; 1359 1360 static const struct video_device pispbe_videodev = { 1361 .name = PISPBE_NAME, 1362 .vfl_dir = VFL_DIR_M2M, /* gets overwritten */ 1363 .fops = &pispbe_fops, 1364 .ioctl_ops = &pispbe_node_ioctl_ops, 1365 .minor = -1, 1366 .release = video_device_release_empty, 1367 }; 1368 1369 static void pispbe_node_def_fmt(struct pispbe_node *node) 1370 { 1371 if (NODE_IS_META(node) && NODE_IS_OUTPUT(node)) { 1372 /* Config node */ 1373 struct v4l2_format *f = &node->format; 1374 1375 f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG; 1376 f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config); 1377 f->type = node->buf_type; 1378 } else { 1379 struct v4l2_format f = { 1380 .fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420, 1381 .fmt.pix_mp.width = 1920, 1382 .fmt.pix_mp.height = 1080, 1383 .type = node->buf_type, 1384 }; 1385 pispbe_try_format(&f, node); 1386 node->format = f; 1387 } 1388 1389 node->pisp_format = pispbe_find_fmt(node->format.fmt.pix_mp.pixelformat); 1390 } 1391 1392 /* 1393 * Initialise a struct pispbe_node and register it as /dev/video<N> 1394 * to represent one of the PiSP Back End's input or output streams. 1395 */ 1396 static int pispbe_init_node(struct pispbe_dev *pispbe, unsigned int id) 1397 { 1398 bool output = NODE_DESC_IS_OUTPUT(&node_desc[id]); 1399 struct pispbe_node *node = &pispbe->node[id]; 1400 struct media_entity *entity = &node->vfd.entity; 1401 struct video_device *vdev = &node->vfd; 1402 struct vb2_queue *q = &node->queue; 1403 int ret; 1404 1405 node->id = id; 1406 node->pispbe = pispbe; 1407 node->buf_type = node_desc[id].buf_type; 1408 1409 mutex_init(&node->node_lock); 1410 mutex_init(&node->queue_lock); 1411 INIT_LIST_HEAD(&node->ready_queue); 1412 spin_lock_init(&node->ready_lock); 1413 1414 node->format.type = node->buf_type; 1415 pispbe_node_def_fmt(node); 1416 1417 q->type = node->buf_type; 1418 q->io_modes = VB2_MMAP | VB2_DMABUF; 1419 q->mem_ops = &vb2_dma_contig_memops; 1420 q->drv_priv = node; 1421 q->ops = &pispbe_node_queue_ops; 1422 q->buf_struct_size = sizeof(struct pispbe_buffer); 1423 q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; 1424 q->dev = pispbe->dev; 1425 /* get V4L2 to handle node->queue locking */ 1426 q->lock = &node->queue_lock; 1427 1428 ret = vb2_queue_init(q); 1429 if (ret < 0) { 1430 dev_err(pispbe->dev, "vb2_queue_init failed\n"); 1431 goto err_mutex_destroy; 1432 } 1433 1434 *vdev = pispbe_videodev; /* default initialization */ 1435 strscpy(vdev->name, node_desc[id].ent_name, sizeof(vdev->name)); 1436 vdev->v4l2_dev = &pispbe->v4l2_dev; 1437 vdev->vfl_dir = output ? VFL_DIR_TX : VFL_DIR_RX; 1438 /* get V4L2 to serialise our ioctls */ 1439 vdev->lock = &node->node_lock; 1440 vdev->queue = &node->queue; 1441 vdev->device_caps = V4L2_CAP_STREAMING | node_desc[id].caps; 1442 1443 node->pad.flags = output ? MEDIA_PAD_FL_SOURCE : MEDIA_PAD_FL_SINK; 1444 ret = media_entity_pads_init(entity, 1, &node->pad); 1445 if (ret) { 1446 dev_err(pispbe->dev, 1447 "Failed to register media pads for %s device node\n", 1448 NODE_NAME(node)); 1449 goto err_unregister_queue; 1450 } 1451 1452 ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1); 1453 if (ret) { 1454 dev_err(pispbe->dev, 1455 "Failed to register video %s device node\n", 1456 NODE_NAME(node)); 1457 goto err_unregister_queue; 1458 } 1459 video_set_drvdata(vdev, node); 1460 1461 if (output) 1462 ret = media_create_pad_link(entity, 0, &pispbe->sd.entity, 1463 id, MEDIA_LNK_FL_IMMUTABLE | 1464 MEDIA_LNK_FL_ENABLED); 1465 else 1466 ret = media_create_pad_link(&pispbe->sd.entity, id, entity, 1467 0, MEDIA_LNK_FL_IMMUTABLE | 1468 MEDIA_LNK_FL_ENABLED); 1469 if (ret) 1470 goto err_unregister_video_dev; 1471 1472 dev_dbg(pispbe->dev, "%s device node registered as /dev/video%d\n", 1473 NODE_NAME(node), node->vfd.num); 1474 1475 return 0; 1476 1477 err_unregister_video_dev: 1478 video_unregister_device(&node->vfd); 1479 err_unregister_queue: 1480 vb2_queue_release(&node->queue); 1481 err_mutex_destroy: 1482 mutex_destroy(&node->node_lock); 1483 mutex_destroy(&node->queue_lock); 1484 return ret; 1485 } 1486 1487 static const struct v4l2_subdev_pad_ops pispbe_pad_ops = { 1488 .link_validate = v4l2_subdev_link_validate_default, 1489 }; 1490 1491 static const struct v4l2_subdev_ops pispbe_sd_ops = { 1492 .pad = &pispbe_pad_ops, 1493 }; 1494 1495 static int pispbe_init_subdev(struct pispbe_dev *pispbe) 1496 { 1497 struct v4l2_subdev *sd = &pispbe->sd; 1498 int ret; 1499 1500 v4l2_subdev_init(sd, &pispbe_sd_ops); 1501 sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER; 1502 sd->owner = THIS_MODULE; 1503 sd->dev = pispbe->dev; 1504 strscpy(sd->name, PISPBE_NAME, sizeof(sd->name)); 1505 1506 for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) 1507 pispbe->pad[i].flags = 1508 NODE_DESC_IS_OUTPUT(&node_desc[i]) ? 1509 MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE; 1510 1511 ret = media_entity_pads_init(&sd->entity, PISPBE_NUM_NODES, 1512 pispbe->pad); 1513 if (ret) 1514 goto error; 1515 1516 ret = v4l2_device_register_subdev(&pispbe->v4l2_dev, sd); 1517 if (ret) 1518 goto error; 1519 1520 return 0; 1521 1522 error: 1523 media_entity_cleanup(&sd->entity); 1524 return ret; 1525 } 1526 1527 static int pispbe_init_devices(struct pispbe_dev *pispbe) 1528 { 1529 struct v4l2_device *v4l2_dev; 1530 struct media_device *mdev; 1531 unsigned int num_regist; 1532 int ret; 1533 1534 /* Register v4l2_device and media_device */ 1535 mdev = &pispbe->mdev; 1536 mdev->hw_revision = pispbe->hw_version; 1537 mdev->dev = pispbe->dev; 1538 strscpy(mdev->model, PISPBE_NAME, sizeof(mdev->model)); 1539 media_device_init(mdev); 1540 1541 v4l2_dev = &pispbe->v4l2_dev; 1542 v4l2_dev->mdev = &pispbe->mdev; 1543 strscpy(v4l2_dev->name, PISPBE_NAME, sizeof(v4l2_dev->name)); 1544 1545 ret = v4l2_device_register(pispbe->dev, v4l2_dev); 1546 if (ret) 1547 goto err_media_dev_cleanup; 1548 1549 /* Register the PISPBE subdevice. */ 1550 ret = pispbe_init_subdev(pispbe); 1551 if (ret) 1552 goto err_unregister_v4l2; 1553 1554 /* Create device video nodes */ 1555 for (num_regist = 0; num_regist < PISPBE_NUM_NODES; num_regist++) { 1556 ret = pispbe_init_node(pispbe, num_regist); 1557 if (ret) 1558 goto err_unregister_nodes; 1559 } 1560 1561 ret = media_device_register(mdev); 1562 if (ret) 1563 goto err_unregister_nodes; 1564 1565 pispbe->config = 1566 dma_alloc_coherent(pispbe->dev, 1567 sizeof(struct pisp_be_tiles_config) * 1568 PISP_BE_NUM_CONFIG_BUFFERS, 1569 &pispbe->config_dma_addr, GFP_KERNEL); 1570 if (!pispbe->config) { 1571 dev_err(pispbe->dev, "Unable to allocate cached config buffers.\n"); 1572 ret = -ENOMEM; 1573 goto err_unregister_mdev; 1574 } 1575 1576 return 0; 1577 1578 err_unregister_mdev: 1579 media_device_unregister(mdev); 1580 err_unregister_nodes: 1581 while (num_regist-- > 0) { 1582 video_unregister_device(&pispbe->node[num_regist].vfd); 1583 vb2_queue_release(&pispbe->node[num_regist].queue); 1584 } 1585 v4l2_device_unregister_subdev(&pispbe->sd); 1586 media_entity_cleanup(&pispbe->sd.entity); 1587 err_unregister_v4l2: 1588 v4l2_device_unregister(v4l2_dev); 1589 err_media_dev_cleanup: 1590 media_device_cleanup(mdev); 1591 return ret; 1592 } 1593 1594 static void pispbe_destroy_devices(struct pispbe_dev *pispbe) 1595 { 1596 if (pispbe->config) { 1597 dma_free_coherent(pispbe->dev, 1598 sizeof(struct pisp_be_tiles_config) * 1599 PISP_BE_NUM_CONFIG_BUFFERS, 1600 pispbe->config, 1601 pispbe->config_dma_addr); 1602 } 1603 1604 dev_dbg(pispbe->dev, "Unregister from media controller\n"); 1605 1606 v4l2_device_unregister_subdev(&pispbe->sd); 1607 media_entity_cleanup(&pispbe->sd.entity); 1608 media_device_unregister(&pispbe->mdev); 1609 1610 for (int i = PISPBE_NUM_NODES - 1; i >= 0; i--) { 1611 video_unregister_device(&pispbe->node[i].vfd); 1612 vb2_queue_release(&pispbe->node[i].queue); 1613 mutex_destroy(&pispbe->node[i].node_lock); 1614 mutex_destroy(&pispbe->node[i].queue_lock); 1615 } 1616 1617 media_device_cleanup(&pispbe->mdev); 1618 v4l2_device_unregister(&pispbe->v4l2_dev); 1619 } 1620 1621 static int pispbe_runtime_suspend(struct device *dev) 1622 { 1623 struct pispbe_dev *pispbe = dev_get_drvdata(dev); 1624 1625 clk_disable_unprepare(pispbe->clk); 1626 1627 return 0; 1628 } 1629 1630 static int pispbe_runtime_resume(struct device *dev) 1631 { 1632 struct pispbe_dev *pispbe = dev_get_drvdata(dev); 1633 int ret; 1634 1635 ret = clk_prepare_enable(pispbe->clk); 1636 if (ret) { 1637 dev_err(dev, "Unable to enable clock\n"); 1638 return ret; 1639 } 1640 1641 dev_dbg(dev, "%s: Enabled clock, rate=%lu\n", 1642 __func__, clk_get_rate(pispbe->clk)); 1643 1644 return 0; 1645 } 1646 1647 static int pispbe_hw_init(struct pispbe_dev *pispbe) 1648 { 1649 u32 u; 1650 1651 /* Check the HW is present and has a known version */ 1652 u = pispbe_rd(pispbe, PISP_BE_VERSION_REG); 1653 dev_dbg(pispbe->dev, "pispbe_probe: HW version: 0x%08x", u); 1654 pispbe->hw_version = u; 1655 if ((u & ~PISP_BE_VERSION_MINOR_BITS) != PISP_BE_VERSION_2712) 1656 return -ENODEV; 1657 1658 /* Clear leftover interrupts */ 1659 pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, 0xFFFFFFFFu); 1660 u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG); 1661 dev_dbg(pispbe->dev, "pispbe_probe: BatchStatus: 0x%08x", u); 1662 1663 pispbe->done = (uint8_t)u; 1664 pispbe->started = (uint8_t)(u >> 8); 1665 u = pispbe_rd(pispbe, PISP_BE_STATUS_REG); 1666 dev_dbg(pispbe->dev, "pispbe_probe: Status: 0x%08x", u); 1667 1668 if (u != 0 || pispbe->done != pispbe->started) { 1669 dev_err(pispbe->dev, "pispbe_probe: HW is stuck or busy\n"); 1670 return -EBUSY; 1671 } 1672 1673 /* 1674 * AXI QOS=0, CACHE=4'b0010, PROT=3'b011 1675 * Also set "chicken bits" 22:20 which enable sub-64-byte bursts 1676 * and AXI AWID/BID variability (on versions which support this). 1677 */ 1678 pispbe_wr(pispbe, PISP_BE_AXI_REG, 0x32703200u); 1679 1680 /* Enable both interrupt flags */ 1681 pispbe_wr(pispbe, PISP_BE_INTERRUPT_EN_REG, 0x00000003u); 1682 1683 return 0; 1684 } 1685 1686 /* Probe the ISP-BE hardware block, as a single platform device. */ 1687 static int pispbe_probe(struct platform_device *pdev) 1688 { 1689 struct pispbe_dev *pispbe; 1690 int ret; 1691 1692 pispbe = devm_kzalloc(&pdev->dev, sizeof(*pispbe), GFP_KERNEL); 1693 if (!pispbe) 1694 return -ENOMEM; 1695 1696 dev_set_drvdata(&pdev->dev, pispbe); 1697 pispbe->dev = &pdev->dev; 1698 platform_set_drvdata(pdev, pispbe); 1699 1700 pispbe->be_reg_base = devm_platform_ioremap_resource(pdev, 0); 1701 if (IS_ERR(pispbe->be_reg_base)) { 1702 dev_err(&pdev->dev, "Failed to get ISP-BE registers address\n"); 1703 return PTR_ERR(pispbe->be_reg_base); 1704 } 1705 1706 pispbe->irq = platform_get_irq(pdev, 0); 1707 if (pispbe->irq <= 0) 1708 return -EINVAL; 1709 1710 ret = devm_request_irq(&pdev->dev, pispbe->irq, pispbe_isr, 0, 1711 PISPBE_NAME, pispbe); 1712 if (ret) { 1713 dev_err(&pdev->dev, "Unable to request interrupt\n"); 1714 return ret; 1715 } 1716 1717 ret = dma_set_mask_and_coherent(pispbe->dev, DMA_BIT_MASK(36)); 1718 if (ret) 1719 return ret; 1720 1721 pispbe->clk = devm_clk_get(&pdev->dev, NULL); 1722 if (IS_ERR(pispbe->clk)) 1723 return dev_err_probe(&pdev->dev, PTR_ERR(pispbe->clk), 1724 "Failed to get clock"); 1725 1726 /* Hardware initialisation */ 1727 pm_runtime_set_autosuspend_delay(pispbe->dev, 200); 1728 pm_runtime_use_autosuspend(pispbe->dev); 1729 pm_runtime_enable(pispbe->dev); 1730 1731 ret = pispbe_runtime_resume(pispbe->dev); 1732 if (ret) 1733 goto pm_runtime_disable_err; 1734 1735 pispbe->hw_busy = false; 1736 spin_lock_init(&pispbe->hw_lock); 1737 ret = pispbe_hw_init(pispbe); 1738 if (ret) 1739 goto pm_runtime_suspend_err; 1740 1741 ret = pispbe_init_devices(pispbe); 1742 if (ret) 1743 goto disable_devs_err; 1744 1745 pm_runtime_mark_last_busy(pispbe->dev); 1746 pm_runtime_put_autosuspend(pispbe->dev); 1747 1748 return 0; 1749 1750 disable_devs_err: 1751 pispbe_destroy_devices(pispbe); 1752 pm_runtime_suspend_err: 1753 pispbe_runtime_suspend(pispbe->dev); 1754 pm_runtime_disable_err: 1755 pm_runtime_dont_use_autosuspend(pispbe->dev); 1756 pm_runtime_disable(pispbe->dev); 1757 1758 return ret; 1759 } 1760 1761 static void pispbe_remove(struct platform_device *pdev) 1762 { 1763 struct pispbe_dev *pispbe = platform_get_drvdata(pdev); 1764 1765 pispbe_destroy_devices(pispbe); 1766 1767 pispbe_runtime_suspend(pispbe->dev); 1768 pm_runtime_dont_use_autosuspend(pispbe->dev); 1769 pm_runtime_disable(pispbe->dev); 1770 } 1771 1772 static const struct dev_pm_ops pispbe_pm_ops = { 1773 SET_RUNTIME_PM_OPS(pispbe_runtime_suspend, pispbe_runtime_resume, NULL) 1774 }; 1775 1776 static const struct of_device_id pispbe_of_match[] = { 1777 { 1778 .compatible = "raspberrypi,pispbe", 1779 }, 1780 { /* sentinel */ }, 1781 }; 1782 MODULE_DEVICE_TABLE(of, pispbe_of_match); 1783 1784 static struct platform_driver pispbe_pdrv = { 1785 .probe = pispbe_probe, 1786 .remove_new = pispbe_remove, 1787 .driver = { 1788 .name = PISPBE_NAME, 1789 .of_match_table = pispbe_of_match, 1790 .pm = &pispbe_pm_ops, 1791 }, 1792 }; 1793 1794 module_platform_driver(pispbe_pdrv); 1795 1796 MODULE_DESCRIPTION("PiSP Back End driver"); 1797 MODULE_AUTHOR("David Plowman <david.plowman@raspberrypi.com>"); 1798 MODULE_AUTHOR("Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>"); 1799 MODULE_LICENSE("GPL"); 1800