1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 - 2025 Intel Corporation 4 */ 5 6 #include <linux/auxiliary_bus.h> 7 #include <linux/bitfield.h> 8 #include <linux/bits.h> 9 #include <linux/bug.h> 10 #include <linux/completion.h> 11 #include <linux/container_of.h> 12 #include <linux/delay.h> 13 #include <linux/device.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/err.h> 16 #include <linux/firmware.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/pci.h> 21 #include <linux/pm_runtime.h> 22 #include <linux/pm_qos.h> 23 #include <linux/slab.h> 24 #include <linux/spinlock.h> 25 #include <linux/string.h> 26 #include <linux/types.h> 27 28 #include <media/ipu-bridge.h> 29 #include <media/media-entity.h> 30 #include <media/v4l2-async.h> 31 #include <media/v4l2-device.h> 32 #include <media/v4l2-fwnode.h> 33 #include <media/v4l2-subdev.h> 34 35 #include "abi/ipu7_fw_isys_abi.h" 36 37 #include "ipu7-bus.h" 38 #include "ipu7-buttress-regs.h" 39 #include "ipu7-cpd.h" 40 #include "ipu7-dma.h" 41 #include "ipu7-fw-isys.h" 42 #include "ipu7-mmu.h" 43 #include "ipu7-isys.h" 44 #include "ipu7-isys-csi2.h" 45 #include "ipu7-isys-csi-phy.h" 46 #include "ipu7-isys-csi2-regs.h" 47 #include "ipu7-isys-video.h" 48 #include "ipu7-platform-regs.h" 49 50 #define ISYS_PM_QOS_VALUE 300 51 52 static int 53 isys_complete_ext_device_registration(struct ipu7_isys *isys, 54 struct v4l2_subdev *sd, 55 struct ipu7_isys_csi2_config *csi2) 56 { 57 struct device *dev = &isys->adev->auxdev.dev; 58 unsigned int i; 59 int ret; 60 61 v4l2_set_subdev_hostdata(sd, csi2); 62 63 for (i = 0; i < sd->entity.num_pads; i++) { 64 if (sd->entity.pads[i].flags & MEDIA_PAD_FL_SOURCE) 65 break; 66 } 67 68 if (i == sd->entity.num_pads) { 69 dev_warn(dev, "no source pad in external entity\n"); 70 ret = -ENOENT; 71 goto skip_unregister_subdev; 72 } 73 74 ret = media_create_pad_link(&sd->entity, i, 75 &isys->csi2[csi2->port].asd.sd.entity, 76 0, MEDIA_LNK_FL_ENABLED | 77 MEDIA_LNK_FL_IMMUTABLE); 78 if (ret) { 79 dev_warn(dev, "can't create link\n"); 80 goto skip_unregister_subdev; 81 } 82 83 isys->csi2[csi2->port].nlanes = csi2->nlanes; 84 if (csi2->bus_type == V4L2_MBUS_CSI2_DPHY) 85 isys->csi2[csi2->port].phy_mode = PHY_MODE_DPHY; 86 else 87 isys->csi2[csi2->port].phy_mode = PHY_MODE_CPHY; 88 89 return 0; 90 91 skip_unregister_subdev: 92 v4l2_device_unregister_subdev(sd); 93 return ret; 94 } 95 96 static void isys_stream_init(struct ipu7_isys *isys) 97 { 98 unsigned int i; 99 100 for (i = 0; i < IPU_ISYS_MAX_STREAMS; i++) { 101 mutex_init(&isys->streams[i].mutex); 102 init_completion(&isys->streams[i].stream_open_completion); 103 init_completion(&isys->streams[i].stream_close_completion); 104 init_completion(&isys->streams[i].stream_start_completion); 105 init_completion(&isys->streams[i].stream_stop_completion); 106 INIT_LIST_HEAD(&isys->streams[i].queues); 107 isys->streams[i].isys = isys; 108 isys->streams[i].stream_handle = i; 109 isys->streams[i].vc = INVALID_VC_ID; 110 } 111 } 112 113 static int isys_fw_log_init(struct ipu7_isys *isys) 114 { 115 struct device *dev = &isys->adev->auxdev.dev; 116 struct isys_fw_log *fw_log; 117 void *log_buf; 118 119 if (isys->fw_log) 120 return 0; 121 122 fw_log = devm_kzalloc(dev, sizeof(*fw_log), GFP_KERNEL); 123 if (!fw_log) 124 return -ENOMEM; 125 126 mutex_init(&fw_log->mutex); 127 128 log_buf = devm_kzalloc(dev, FW_LOG_BUF_SIZE, GFP_KERNEL); 129 if (!log_buf) 130 return -ENOMEM; 131 132 fw_log->head = log_buf; 133 fw_log->addr = log_buf; 134 fw_log->count = 0; 135 fw_log->size = 0; 136 137 isys->fw_log = fw_log; 138 139 return 0; 140 } 141 142 /* The .bound() notifier callback when a match is found */ 143 static int isys_notifier_bound(struct v4l2_async_notifier *notifier, 144 struct v4l2_subdev *sd, 145 struct v4l2_async_connection *asc) 146 { 147 struct ipu7_isys *isys = container_of(notifier, 148 struct ipu7_isys, notifier); 149 struct sensor_async_sd *s_asd = 150 container_of(asc, struct sensor_async_sd, asc); 151 struct device *dev = &isys->adev->auxdev.dev; 152 int ret; 153 154 ret = ipu_bridge_instantiate_vcm(sd->dev); 155 if (ret) { 156 dev_err(dev, "instantiate vcm failed\n"); 157 return ret; 158 } 159 160 dev_info(dev, "bind %s nlanes is %d port is %d\n", 161 sd->name, s_asd->csi2.nlanes, s_asd->csi2.port); 162 isys_complete_ext_device_registration(isys, sd, &s_asd->csi2); 163 164 return v4l2_device_register_subdev_nodes(&isys->v4l2_dev); 165 } 166 167 static int isys_notifier_complete(struct v4l2_async_notifier *notifier) 168 { 169 struct ipu7_isys *isys = container_of(notifier, 170 struct ipu7_isys, notifier); 171 172 dev_info(&isys->adev->auxdev.dev, 173 "All sensor registration completed.\n"); 174 175 return v4l2_device_register_subdev_nodes(&isys->v4l2_dev); 176 } 177 178 static const struct v4l2_async_notifier_operations isys_async_ops = { 179 .bound = isys_notifier_bound, 180 .complete = isys_notifier_complete, 181 }; 182 183 static int isys_notifier_init(struct ipu7_isys *isys) 184 { 185 const struct ipu7_isys_internal_csi2_pdata *csi2 = 186 &isys->pdata->ipdata->csi2; 187 struct ipu7_device *isp = isys->adev->isp; 188 struct device *dev = &isp->pdev->dev; 189 unsigned int i; 190 int ret; 191 192 v4l2_async_nf_init(&isys->notifier, &isys->v4l2_dev); 193 194 for (i = 0; i < csi2->nports; i++) { 195 struct v4l2_fwnode_endpoint vep = { 196 .bus_type = V4L2_MBUS_UNKNOWN 197 }; 198 struct sensor_async_sd *s_asd; 199 struct fwnode_handle *ep; 200 201 ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), i, 0, 202 FWNODE_GRAPH_ENDPOINT_NEXT); 203 if (!ep) 204 continue; 205 206 ret = v4l2_fwnode_endpoint_parse(ep, &vep); 207 if (ret) 208 goto err_parse; 209 210 if (vep.bus_type != V4L2_MBUS_CSI2_DPHY && 211 vep.bus_type != V4L2_MBUS_CSI2_CPHY) { 212 ret = -EINVAL; 213 dev_err(dev, "unsupported bus type %d!\n", 214 vep.bus_type); 215 goto err_parse; 216 } 217 218 s_asd = v4l2_async_nf_add_fwnode_remote(&isys->notifier, ep, 219 struct 220 sensor_async_sd); 221 if (IS_ERR(s_asd)) { 222 ret = PTR_ERR(s_asd); 223 goto err_parse; 224 } 225 226 s_asd->csi2.port = vep.base.port; 227 s_asd->csi2.nlanes = vep.bus.mipi_csi2.num_data_lanes; 228 s_asd->csi2.bus_type = vep.bus_type; 229 230 fwnode_handle_put(ep); 231 232 continue; 233 234 err_parse: 235 fwnode_handle_put(ep); 236 return ret; 237 } 238 239 if (list_empty(&isys->notifier.waiting_list)) { 240 /* isys probe could continue with async subdevs missing */ 241 dev_warn(dev, "no subdev found in graph\n"); 242 return 0; 243 } 244 245 isys->notifier.ops = &isys_async_ops; 246 ret = v4l2_async_nf_register(&isys->notifier); 247 if (ret) { 248 dev_err(dev, "failed to register async notifier(%d)\n", ret); 249 v4l2_async_nf_cleanup(&isys->notifier); 250 } 251 252 return ret; 253 } 254 255 static void isys_notifier_cleanup(struct ipu7_isys *isys) 256 { 257 v4l2_async_nf_unregister(&isys->notifier); 258 v4l2_async_nf_cleanup(&isys->notifier); 259 } 260 261 static void isys_unregister_video_devices(struct ipu7_isys *isys) 262 { 263 const struct ipu7_isys_internal_csi2_pdata *csi2_pdata = 264 &isys->pdata->ipdata->csi2; 265 unsigned int i, j; 266 267 for (i = 0; i < csi2_pdata->nports; i++) 268 for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++) 269 ipu7_isys_video_cleanup(&isys->csi2[i].av[j]); 270 } 271 272 static int isys_register_video_devices(struct ipu7_isys *isys) 273 { 274 const struct ipu7_isys_internal_csi2_pdata *csi2_pdata = 275 &isys->pdata->ipdata->csi2; 276 unsigned int i, j; 277 int ret; 278 279 for (i = 0; i < csi2_pdata->nports; i++) { 280 for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++) { 281 struct ipu7_isys_video *av = &isys->csi2[i].av[j]; 282 283 snprintf(av->vdev.name, sizeof(av->vdev.name), 284 IPU_ISYS_ENTITY_PREFIX " ISYS Capture %u", 285 i * IPU7_NR_OF_CSI2_SRC_PADS + j); 286 av->isys = isys; 287 av->aq.vbq.buf_struct_size = 288 sizeof(struct ipu7_isys_video_buffer); 289 290 ret = ipu7_isys_video_init(av); 291 if (ret) 292 goto fail; 293 } 294 } 295 296 return 0; 297 298 fail: 299 i = i + 1U; 300 while (i--) { 301 while (j--) 302 ipu7_isys_video_cleanup(&isys->csi2[i].av[j]); 303 j = IPU7_NR_OF_CSI2_SRC_PADS; 304 } 305 306 return ret; 307 } 308 309 static void isys_csi2_unregister_subdevices(struct ipu7_isys *isys) 310 { 311 const struct ipu7_isys_internal_csi2_pdata *csi2 = 312 &isys->pdata->ipdata->csi2; 313 unsigned int i; 314 315 for (i = 0; i < csi2->nports; i++) 316 ipu7_isys_csi2_cleanup(&isys->csi2[i]); 317 } 318 319 static int isys_csi2_register_subdevices(struct ipu7_isys *isys) 320 { 321 const struct ipu7_isys_internal_csi2_pdata *csi2_pdata = 322 &isys->pdata->ipdata->csi2; 323 unsigned int i; 324 int ret; 325 326 for (i = 0; i < csi2_pdata->nports; i++) { 327 ret = ipu7_isys_csi2_init(&isys->csi2[i], isys, 328 isys->pdata->base + 329 csi2_pdata->offsets[i], i); 330 if (ret) 331 goto fail; 332 } 333 334 isys->isr_csi2_mask = IPU7_CSI_RX_LEGACY_IRQ_MASK; 335 336 return 0; 337 338 fail: 339 while (i--) 340 ipu7_isys_csi2_cleanup(&isys->csi2[i]); 341 342 return ret; 343 } 344 345 static int isys_csi2_create_media_links(struct ipu7_isys *isys) 346 { 347 const struct ipu7_isys_internal_csi2_pdata *csi2_pdata = 348 &isys->pdata->ipdata->csi2; 349 struct device *dev = &isys->adev->auxdev.dev; 350 struct media_entity *sd; 351 unsigned int i, j; 352 int ret; 353 354 for (i = 0; i < csi2_pdata->nports; i++) { 355 sd = &isys->csi2[i].asd.sd.entity; 356 357 for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++) { 358 struct ipu7_isys_video *av = &isys->csi2[i].av[j]; 359 360 ret = media_create_pad_link(sd, IPU7_CSI2_PAD_SRC + j, 361 &av->vdev.entity, 0, 0); 362 if (ret) { 363 dev_err(dev, "CSI2 can't create link\n"); 364 return ret; 365 } 366 367 av->csi2 = &isys->csi2[i]; 368 } 369 } 370 371 return 0; 372 } 373 374 static int isys_register_devices(struct ipu7_isys *isys) 375 { 376 struct device *dev = &isys->adev->auxdev.dev; 377 struct pci_dev *pdev = isys->adev->isp->pdev; 378 int ret; 379 380 media_device_pci_init(&isys->media_dev, 381 pdev, IPU_MEDIA_DEV_MODEL_NAME); 382 383 strscpy(isys->v4l2_dev.name, isys->media_dev.model, 384 sizeof(isys->v4l2_dev.name)); 385 386 ret = media_device_register(&isys->media_dev); 387 if (ret < 0) 388 goto out_media_device_unregister; 389 390 isys->v4l2_dev.mdev = &isys->media_dev; 391 isys->v4l2_dev.ctrl_handler = NULL; 392 393 ret = v4l2_device_register(dev, &isys->v4l2_dev); 394 if (ret < 0) 395 goto out_media_device_unregister; 396 397 ret = isys_register_video_devices(isys); 398 if (ret) 399 goto out_v4l2_device_unregister; 400 401 ret = isys_csi2_register_subdevices(isys); 402 if (ret) 403 goto out_video_unregister_device; 404 405 ret = isys_csi2_create_media_links(isys); 406 if (ret) 407 goto out_csi2_unregister_subdevices; 408 409 ret = isys_notifier_init(isys); 410 if (ret) 411 goto out_csi2_unregister_subdevices; 412 413 return 0; 414 415 out_csi2_unregister_subdevices: 416 isys_csi2_unregister_subdevices(isys); 417 418 out_video_unregister_device: 419 isys_unregister_video_devices(isys); 420 421 out_v4l2_device_unregister: 422 v4l2_device_unregister(&isys->v4l2_dev); 423 424 out_media_device_unregister: 425 media_device_unregister(&isys->media_dev); 426 media_device_cleanup(&isys->media_dev); 427 428 dev_err(dev, "failed to register isys devices\n"); 429 430 return ret; 431 } 432 433 static void isys_unregister_devices(struct ipu7_isys *isys) 434 { 435 isys_unregister_video_devices(isys); 436 isys_csi2_unregister_subdevices(isys); 437 v4l2_device_unregister(&isys->v4l2_dev); 438 media_device_unregister(&isys->media_dev); 439 media_device_cleanup(&isys->media_dev); 440 } 441 442 static void enable_csi2_legacy_irq(struct ipu7_isys *isys, bool enable) 443 { 444 u32 offset = IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE; 445 void __iomem *base = isys->pdata->base; 446 u32 mask = isys->isr_csi2_mask; 447 448 if (!enable) { 449 writel(mask, base + offset + IRQ_CTL_CLEAR); 450 writel(0, base + offset + IRQ_CTL_ENABLE); 451 return; 452 } 453 454 writel(mask, base + offset + IRQ_CTL_EDGE); 455 writel(mask, base + offset + IRQ_CTL_CLEAR); 456 writel(mask, base + offset + IRQ_CTL_MASK); 457 writel(mask, base + offset + IRQ_CTL_ENABLE); 458 } 459 460 static void enable_to_sw_irq(struct ipu7_isys *isys, bool enable) 461 { 462 void __iomem *base = isys->pdata->base; 463 u32 mask = IS_UC_TO_SW_IRQ_MASK; 464 u32 offset = IS_UC_CTRL_BASE; 465 466 if (!enable) { 467 writel(0, base + offset + TO_SW_IRQ_CNTL_ENABLE); 468 return; 469 } 470 471 writel(mask, base + offset + TO_SW_IRQ_CNTL_CLEAR); 472 writel(mask, base + offset + TO_SW_IRQ_CNTL_MASK_N); 473 writel(mask, base + offset + TO_SW_IRQ_CNTL_ENABLE); 474 } 475 476 void ipu7_isys_setup_hw(struct ipu7_isys *isys) 477 { 478 u32 offset; 479 void __iomem *base = isys->pdata->base; 480 481 /* soft reset */ 482 offset = IS_IO_GPREGS_BASE; 483 484 writel(0x0, base + offset + CLK_EN_TXCLKESC); 485 /* Update if ISYS freq updated (0: 400/1, 1:400/2, 63:400/64) */ 486 writel(0x0, base + offset + CLK_DIV_FACTOR_IS_CLK); 487 /* correct the initial printf configuration */ 488 writel(0x200, base + IS_UC_CTRL_BASE + PRINTF_AXI_CNTL); 489 490 enable_to_sw_irq(isys, 1); 491 enable_csi2_legacy_irq(isys, 1); 492 } 493 494 static void isys_cleanup_hw(struct ipu7_isys *isys) 495 { 496 enable_csi2_legacy_irq(isys, 0); 497 enable_to_sw_irq(isys, 0); 498 } 499 500 static int isys_runtime_pm_resume(struct device *dev) 501 { 502 struct ipu7_bus_device *adev = to_ipu7_bus_device(dev); 503 struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev); 504 struct ipu7_device *isp = adev->isp; 505 unsigned long flags; 506 int ret; 507 508 if (!isys) 509 return 0; 510 511 ret = ipu7_mmu_hw_init(adev->mmu); 512 if (ret) 513 return ret; 514 515 cpu_latency_qos_update_request(&isys->pm_qos, ISYS_PM_QOS_VALUE); 516 517 ret = ipu_buttress_start_tsc_sync(isp); 518 if (ret) 519 return ret; 520 521 spin_lock_irqsave(&isys->power_lock, flags); 522 isys->power = 1; 523 spin_unlock_irqrestore(&isys->power_lock, flags); 524 525 return 0; 526 } 527 528 static int isys_runtime_pm_suspend(struct device *dev) 529 { 530 struct ipu7_bus_device *adev = to_ipu7_bus_device(dev); 531 struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev); 532 unsigned long flags; 533 534 if (!isys) 535 return 0; 536 537 isys_cleanup_hw(isys); 538 539 spin_lock_irqsave(&isys->power_lock, flags); 540 isys->power = 0; 541 spin_unlock_irqrestore(&isys->power_lock, flags); 542 543 cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE); 544 545 ipu7_mmu_hw_cleanup(adev->mmu); 546 547 return 0; 548 } 549 550 static int isys_suspend(struct device *dev) 551 { 552 struct ipu7_isys *isys = dev_get_drvdata(dev); 553 554 /* If stream is open, refuse to suspend */ 555 if (isys->stream_opened) 556 return -EBUSY; 557 558 return 0; 559 } 560 561 static int isys_resume(struct device *dev) 562 { 563 return 0; 564 } 565 566 static const struct dev_pm_ops isys_pm_ops = { 567 .runtime_suspend = isys_runtime_pm_suspend, 568 .runtime_resume = isys_runtime_pm_resume, 569 .suspend = isys_suspend, 570 .resume = isys_resume, 571 }; 572 573 static void isys_remove(struct auxiliary_device *auxdev) 574 { 575 struct ipu7_isys *isys = dev_get_drvdata(&auxdev->dev); 576 struct isys_fw_msgs *fwmsg, *safe; 577 struct ipu7_bus_device *adev = auxdev_to_adev(auxdev); 578 579 for (int i = 0; i < IPU_ISYS_MAX_STREAMS; i++) 580 mutex_destroy(&isys->streams[i].mutex); 581 582 list_for_each_entry_safe(fwmsg, safe, &isys->framebuflist, head) 583 ipu7_dma_free(adev, sizeof(struct isys_fw_msgs), 584 fwmsg, fwmsg->dma_addr, 0); 585 586 list_for_each_entry_safe(fwmsg, safe, &isys->framebuflist_fw, head) 587 ipu7_dma_free(adev, sizeof(struct isys_fw_msgs), 588 fwmsg, fwmsg->dma_addr, 0); 589 590 isys_notifier_cleanup(isys); 591 isys_unregister_devices(isys); 592 593 cpu_latency_qos_remove_request(&isys->pm_qos); 594 595 mutex_destroy(&isys->stream_mutex); 596 mutex_destroy(&isys->mutex); 597 } 598 599 static int alloc_fw_msg_bufs(struct ipu7_isys *isys, int amount) 600 { 601 struct ipu7_bus_device *adev = isys->adev; 602 struct isys_fw_msgs *addr; 603 dma_addr_t dma_addr; 604 unsigned long flags; 605 unsigned int i; 606 607 for (i = 0; i < amount; i++) { 608 addr = ipu7_dma_alloc(adev, sizeof(struct isys_fw_msgs), 609 &dma_addr, GFP_KERNEL, 0); 610 if (!addr) 611 break; 612 addr->dma_addr = dma_addr; 613 614 spin_lock_irqsave(&isys->listlock, flags); 615 list_add(&addr->head, &isys->framebuflist); 616 spin_unlock_irqrestore(&isys->listlock, flags); 617 } 618 619 if (i == amount) 620 return 0; 621 622 spin_lock_irqsave(&isys->listlock, flags); 623 while (!list_empty(&isys->framebuflist)) { 624 addr = list_first_entry(&isys->framebuflist, 625 struct isys_fw_msgs, head); 626 list_del(&addr->head); 627 spin_unlock_irqrestore(&isys->listlock, flags); 628 ipu7_dma_free(adev, sizeof(struct isys_fw_msgs), 629 addr, addr->dma_addr, 0); 630 spin_lock_irqsave(&isys->listlock, flags); 631 } 632 spin_unlock_irqrestore(&isys->listlock, flags); 633 634 return -ENOMEM; 635 } 636 637 struct isys_fw_msgs *ipu7_get_fw_msg_buf(struct ipu7_isys_stream *stream) 638 { 639 struct device *dev = &stream->isys->adev->auxdev.dev; 640 struct ipu7_isys *isys = stream->isys; 641 struct isys_fw_msgs *msg; 642 unsigned long flags; 643 int ret; 644 645 spin_lock_irqsave(&isys->listlock, flags); 646 if (list_empty(&isys->framebuflist)) { 647 spin_unlock_irqrestore(&isys->listlock, flags); 648 dev_dbg(dev, "Frame buffer list empty\n"); 649 650 ret = alloc_fw_msg_bufs(isys, 5); 651 if (ret < 0) 652 return NULL; 653 654 spin_lock_irqsave(&isys->listlock, flags); 655 if (list_empty(&isys->framebuflist)) { 656 spin_unlock_irqrestore(&isys->listlock, flags); 657 dev_err(dev, "Frame list empty\n"); 658 return NULL; 659 } 660 } 661 msg = list_last_entry(&isys->framebuflist, struct isys_fw_msgs, head); 662 list_move(&msg->head, &isys->framebuflist_fw); 663 spin_unlock_irqrestore(&isys->listlock, flags); 664 memset(&msg->fw_msg, 0, sizeof(msg->fw_msg)); 665 666 return msg; 667 } 668 669 void ipu7_cleanup_fw_msg_bufs(struct ipu7_isys *isys) 670 { 671 struct isys_fw_msgs *fwmsg, *fwmsg0; 672 unsigned long flags; 673 674 spin_lock_irqsave(&isys->listlock, flags); 675 list_for_each_entry_safe(fwmsg, fwmsg0, &isys->framebuflist_fw, head) 676 list_move(&fwmsg->head, &isys->framebuflist); 677 spin_unlock_irqrestore(&isys->listlock, flags); 678 } 679 680 void ipu7_put_fw_msg_buf(struct ipu7_isys *isys, uintptr_t data) 681 { 682 struct isys_fw_msgs *msg; 683 void *ptr = (void *)data; 684 unsigned long flags; 685 686 if (WARN_ON_ONCE(!ptr)) 687 return; 688 689 spin_lock_irqsave(&isys->listlock, flags); 690 msg = container_of(ptr, struct isys_fw_msgs, fw_msg.dummy); 691 list_move(&msg->head, &isys->framebuflist); 692 spin_unlock_irqrestore(&isys->listlock, flags); 693 } 694 695 static int isys_probe(struct auxiliary_device *auxdev, 696 const struct auxiliary_device_id *auxdev_id) 697 { 698 const struct ipu7_isys_internal_csi2_pdata *csi2_pdata; 699 struct ipu7_bus_device *adev = auxdev_to_adev(auxdev); 700 struct ipu7_device *isp = adev->isp; 701 struct ipu7_isys *isys; 702 int ret = 0; 703 704 if (!isp->ipu7_bus_ready_to_probe) 705 return -EPROBE_DEFER; 706 707 isys = devm_kzalloc(&auxdev->dev, sizeof(*isys), GFP_KERNEL); 708 if (!isys) 709 return -ENOMEM; 710 711 ret = pm_runtime_resume_and_get(&auxdev->dev); 712 if (ret < 0) 713 return ret; 714 715 adev->auxdrv_data = 716 (const struct ipu7_auxdrv_data *)auxdev_id->driver_data; 717 adev->auxdrv = to_auxiliary_drv(auxdev->dev.driver); 718 isys->adev = adev; 719 isys->pdata = adev->pdata; 720 721 INIT_LIST_HEAD(&isys->requests); 722 csi2_pdata = &isys->pdata->ipdata->csi2; 723 724 isys->csi2 = devm_kcalloc(&auxdev->dev, csi2_pdata->nports, 725 sizeof(*isys->csi2), GFP_KERNEL); 726 if (!isys->csi2) { 727 ret = -ENOMEM; 728 goto out_runtime_put; 729 } 730 731 ret = ipu7_mmu_hw_init(adev->mmu); 732 if (ret) 733 goto out_runtime_put; 734 735 spin_lock_init(&isys->streams_lock); 736 spin_lock_init(&isys->power_lock); 737 isys->power = 0; 738 739 mutex_init(&isys->mutex); 740 mutex_init(&isys->stream_mutex); 741 742 spin_lock_init(&isys->listlock); 743 INIT_LIST_HEAD(&isys->framebuflist); 744 INIT_LIST_HEAD(&isys->framebuflist_fw); 745 746 dev_set_drvdata(&auxdev->dev, isys); 747 748 isys->icache_prefetch = 0; 749 isys->phy_rext_cal = 0; 750 751 isys_stream_init(isys); 752 753 cpu_latency_qos_add_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE); 754 ret = alloc_fw_msg_bufs(isys, 20); 755 if (ret < 0) 756 goto out_cleanup_isys; 757 758 ret = ipu7_fw_isys_init(isys); 759 if (ret) 760 goto out_cleanup_isys; 761 762 ret = isys_register_devices(isys); 763 if (ret) 764 goto out_cleanup_fw; 765 766 ret = isys_fw_log_init(isys); 767 if (ret) 768 goto out_cleanup; 769 770 ipu7_mmu_hw_cleanup(adev->mmu); 771 pm_runtime_put(&auxdev->dev); 772 773 return 0; 774 775 out_cleanup: 776 isys_unregister_devices(isys); 777 out_cleanup_fw: 778 ipu7_fw_isys_release(isys); 779 out_cleanup_isys: 780 cpu_latency_qos_remove_request(&isys->pm_qos); 781 782 for (unsigned int i = 0; i < IPU_ISYS_MAX_STREAMS; i++) 783 mutex_destroy(&isys->streams[i].mutex); 784 785 mutex_destroy(&isys->mutex); 786 mutex_destroy(&isys->stream_mutex); 787 788 ipu7_mmu_hw_cleanup(adev->mmu); 789 790 out_runtime_put: 791 pm_runtime_put(&auxdev->dev); 792 793 return ret; 794 } 795 796 struct ipu7_csi2_error { 797 const char *error_string; 798 bool is_info_only; 799 }; 800 801 /* 802 * Strings corresponding to CSI-2 receiver errors are here. 803 * Corresponding macros are defined in the header file. 804 */ 805 static const struct ipu7_csi2_error dphy_rx_errors[] = { 806 { "Error handler FIFO full", false }, 807 { "Reserved Short Packet encoding detected", true }, 808 { "Reserved Long Packet encoding detected", true }, 809 { "Received packet is too short", false}, 810 { "Received packet is too long", false}, 811 { "Short packet discarded due to errors", false }, 812 { "Long packet discarded due to errors", false }, 813 { "CSI Combo Rx interrupt", false }, 814 { "IDI CDC FIFO overflow(remaining bits are reserved as 0)", false }, 815 { "Received NULL packet", true }, 816 { "Received blanking packet", true }, 817 { "Tie to 0", true }, 818 { } 819 }; 820 821 static void ipu7_isys_register_errors(struct ipu7_isys_csi2 *csi2) 822 { 823 u32 offset = IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(csi2->port); 824 u32 status = readl(csi2->base + offset + IRQ_CTL_STATUS); 825 u32 mask = IPU7_CSI_RX_ERROR_IRQ_MASK; 826 827 if (!status) 828 return; 829 830 dev_dbg(&csi2->isys->adev->auxdev.dev, "csi2-%u error status 0x%08x\n", 831 csi2->port, status); 832 833 writel(status & mask, csi2->base + offset + IRQ_CTL_CLEAR); 834 csi2->receiver_errors |= status & mask; 835 } 836 837 static void ipu7_isys_csi2_error(struct ipu7_isys_csi2 *csi2) 838 { 839 struct ipu7_csi2_error const *errors; 840 unsigned int i; 841 u32 status; 842 843 /* Register errors once more in case of error interrupts are disabled */ 844 ipu7_isys_register_errors(csi2); 845 status = csi2->receiver_errors; 846 csi2->receiver_errors = 0; 847 errors = dphy_rx_errors; 848 849 for (i = 0; i < CSI_RX_NUM_ERRORS_IN_IRQ; i++) { 850 if (status & BIT(i)) 851 dev_err_ratelimited(&csi2->isys->adev->auxdev.dev, 852 "csi2-%i error: %s\n", 853 csi2->port, 854 errors[i].error_string); 855 } 856 } 857 858 struct resp_to_msg { 859 enum ipu7_insys_resp_type type; 860 const char *msg; 861 }; 862 863 static const struct resp_to_msg is_fw_msg[] = { 864 {IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE, 865 "IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE"}, 866 {IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK, 867 "IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK"}, 868 {IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK, 869 "IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK"}, 870 {IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK, 871 "IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK"}, 872 {IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK, 873 "IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK"}, 874 {IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK, 875 "IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK"}, 876 {IPU_INSYS_RESP_TYPE_PIN_DATA_READY, 877 "IPU_INSYS_RESP_TYPE_PIN_DATA_READY"}, 878 {IPU_INSYS_RESP_TYPE_FRAME_SOF, "IPU_INSYS_RESP_TYPE_FRAME_SOF"}, 879 {IPU_INSYS_RESP_TYPE_FRAME_EOF, "IPU_INSYS_RESP_TYPE_FRAME_EOF"}, 880 {IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE, 881 "IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE"}, 882 {IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE, 883 "IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE"}, 884 {N_IPU_INSYS_RESP_TYPE, "N_IPU_INSYS_RESP_TYPE"}, 885 }; 886 887 int isys_isr_one(struct ipu7_bus_device *adev) 888 { 889 struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev); 890 struct ipu7_isys_stream *stream = NULL; 891 struct device *dev = &adev->auxdev.dev; 892 struct ipu7_isys_csi2 *csi2 = NULL; 893 struct ia_gofo_msg_err err_info; 894 struct ipu7_insys_resp *resp; 895 u64 ts; 896 897 if (!isys->adev->syscom) 898 return 1; 899 900 resp = ipu7_fw_isys_get_resp(isys); 901 if (!resp) 902 return 1; 903 if (resp->type >= N_IPU_INSYS_RESP_TYPE) { 904 dev_err(dev, "Unknown response type %u stream %u\n", 905 resp->type, resp->stream_id); 906 ipu7_fw_isys_put_resp(isys); 907 return 1; 908 } 909 910 err_info = resp->error_info; 911 ts = ((u64)resp->timestamp[1] << 32) | resp->timestamp[0]; 912 if (err_info.err_group == INSYS_MSG_ERR_GROUP_CAPTURE && 913 err_info.err_code == INSYS_MSG_ERR_CAPTURE_SYNC_FRAME_DROP) { 914 /* receive a sp w/o command, firmware drop it */ 915 dev_dbg(dev, "FRAME DROP: %02u %s stream %u\n", 916 resp->type, is_fw_msg[resp->type].msg, 917 resp->stream_id); 918 dev_dbg(dev, "\tpin %u buf_id %llx frame %u\n", 919 resp->pin_id, resp->buf_id, resp->frame_id); 920 dev_dbg(dev, "\terror group %u code %u details [%u %u]\n", 921 err_info.err_group, err_info.err_code, 922 err_info.err_detail[0], err_info.err_detail[1]); 923 } else if (!IA_GOFO_MSG_ERR_IS_OK(err_info)) { 924 dev_err(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n", 925 resp->type, is_fw_msg[resp->type].msg, resp->stream_id, 926 resp->pin_id, resp->buf_id, resp->frame_id); 927 dev_err(dev, "\terror group %u code %u details [%u %u]\n", 928 err_info.err_group, err_info.err_code, 929 err_info.err_detail[0], err_info.err_detail[1]); 930 } else { 931 dev_dbg(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n", 932 resp->type, is_fw_msg[resp->type].msg, resp->stream_id, 933 resp->pin_id, resp->buf_id, resp->frame_id); 934 dev_dbg(dev, "\tts %llu\n", ts); 935 } 936 937 if (resp->stream_id >= IPU_ISYS_MAX_STREAMS) { 938 dev_err(dev, "bad stream handle %u\n", 939 resp->stream_id); 940 goto leave; 941 } 942 943 stream = ipu7_isys_query_stream_by_handle(isys, resp->stream_id); 944 if (!stream) { 945 dev_err(dev, "stream of stream_handle %u is unused\n", 946 resp->stream_id); 947 goto leave; 948 } 949 950 stream->error = err_info.err_code; 951 952 if (stream->asd) 953 csi2 = ipu7_isys_subdev_to_csi2(stream->asd); 954 955 switch (resp->type) { 956 case IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE: 957 complete(&stream->stream_open_completion); 958 break; 959 case IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK: 960 complete(&stream->stream_close_completion); 961 break; 962 case IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK: 963 complete(&stream->stream_start_completion); 964 break; 965 case IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK: 966 complete(&stream->stream_stop_completion); 967 break; 968 case IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK: 969 complete(&stream->stream_stop_completion); 970 break; 971 case IPU_INSYS_RESP_TYPE_PIN_DATA_READY: 972 /* 973 * firmware only release the capture msg until software 974 * get pin_data_ready event 975 */ 976 ipu7_put_fw_msg_buf(ipu7_bus_get_drvdata(adev), resp->buf_id); 977 if (resp->pin_id < IPU_INSYS_OUTPUT_PINS && 978 stream->output_pins[resp->pin_id].pin_ready) 979 stream->output_pins[resp->pin_id].pin_ready(stream, 980 resp); 981 else 982 dev_err(dev, "No handler for pin %u ready\n", 983 resp->pin_id); 984 if (csi2) 985 ipu7_isys_csi2_error(csi2); 986 987 break; 988 case IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK: 989 break; 990 case IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE: 991 case IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE: 992 break; 993 case IPU_INSYS_RESP_TYPE_FRAME_SOF: 994 if (csi2) 995 ipu7_isys_csi2_sof_event_by_stream(stream); 996 997 stream->seq[stream->seq_index].sequence = 998 atomic_read(&stream->sequence) - 1U; 999 stream->seq[stream->seq_index].timestamp = ts; 1000 dev_dbg(dev, 1001 "SOF: stream %u frame %u (index %u), ts 0x%16.16llx\n", 1002 resp->stream_id, resp->frame_id, 1003 stream->seq[stream->seq_index].sequence, ts); 1004 stream->seq_index = (stream->seq_index + 1U) 1005 % IPU_ISYS_MAX_PARALLEL_SOF; 1006 break; 1007 case IPU_INSYS_RESP_TYPE_FRAME_EOF: 1008 if (csi2) 1009 ipu7_isys_csi2_eof_event_by_stream(stream); 1010 1011 dev_dbg(dev, "eof: stream %d(index %u) ts 0x%16.16llx\n", 1012 resp->stream_id, 1013 stream->seq[stream->seq_index].sequence, ts); 1014 break; 1015 default: 1016 dev_err(dev, "Unknown response type %u stream %u\n", 1017 resp->type, resp->stream_id); 1018 break; 1019 } 1020 1021 ipu7_isys_put_stream(stream); 1022 leave: 1023 ipu7_fw_isys_put_resp(isys); 1024 1025 return 0; 1026 } 1027 1028 static void ipu7_isys_csi2_isr(struct ipu7_isys_csi2 *csi2) 1029 { 1030 struct device *dev = &csi2->isys->adev->auxdev.dev; 1031 struct ipu7_device *isp = csi2->isys->adev->isp; 1032 struct ipu7_isys_stream *s; 1033 u32 sync, offset; 1034 u32 fe = 0; 1035 u8 vc; 1036 1037 ipu7_isys_register_errors(csi2); 1038 1039 offset = IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(csi2->port); 1040 sync = readl(csi2->base + offset + IRQ_CTL_STATUS); 1041 writel(sync, csi2->base + offset + IRQ_CTL_CLEAR); 1042 dev_dbg(dev, "csi2-%u sync status 0x%08x\n", csi2->port, sync); 1043 1044 if (!is_ipu7(isp->hw_ver)) { 1045 fe = readl(csi2->base + offset + IRQ1_CTL_STATUS); 1046 writel(fe, csi2->base + offset + IRQ1_CTL_CLEAR); 1047 dev_dbg(dev, "csi2-%u FE status 0x%08x\n", csi2->port, fe); 1048 } 1049 1050 for (vc = 0; vc < IPU7_NR_OF_CSI2_VC && (sync || fe); vc++) { 1051 s = ipu7_isys_query_stream_by_source(csi2->isys, 1052 csi2->asd.source, vc); 1053 if (!s) 1054 continue; 1055 1056 if (!is_ipu7(isp->hw_ver)) { 1057 if (sync & IPU7P5_CSI_RX_SYNC_FS_VC & (1U << vc)) 1058 ipu7_isys_csi2_sof_event_by_stream(s); 1059 1060 if (fe & IPU7P5_CSI_RX_SYNC_FE_VC & (1U << vc)) 1061 ipu7_isys_csi2_eof_event_by_stream(s); 1062 } else { 1063 if (sync & IPU7_CSI_RX_SYNC_FS_VC & (1U << (vc * 2))) 1064 ipu7_isys_csi2_sof_event_by_stream(s); 1065 1066 if (sync & IPU7_CSI_RX_SYNC_FE_VC & (2U << (vc * 2))) 1067 ipu7_isys_csi2_eof_event_by_stream(s); 1068 } 1069 } 1070 } 1071 1072 static irqreturn_t isys_isr(struct ipu7_bus_device *adev) 1073 { 1074 struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev); 1075 u32 status_csi, status_sw, csi_offset, sw_offset; 1076 struct device *dev = &isys->adev->auxdev.dev; 1077 void __iomem *base = isys->pdata->base; 1078 1079 spin_lock(&isys->power_lock); 1080 if (!isys->power) { 1081 spin_unlock(&isys->power_lock); 1082 return IRQ_NONE; 1083 } 1084 1085 csi_offset = IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE; 1086 sw_offset = IS_BASE; 1087 1088 status_csi = readl(base + csi_offset + IRQ_CTL_STATUS); 1089 status_sw = readl(base + sw_offset + TO_SW_IRQ_CNTL_STATUS); 1090 if (!status_csi && !status_sw) { 1091 spin_unlock(&isys->power_lock); 1092 return IRQ_NONE; 1093 } 1094 1095 if (status_csi) 1096 dev_dbg(dev, "status csi 0x%08x\n", status_csi); 1097 if (status_sw) 1098 dev_dbg(dev, "status to_sw 0x%08x\n", status_sw); 1099 1100 do { 1101 writel(status_sw, base + sw_offset + TO_SW_IRQ_CNTL_CLEAR); 1102 writel(status_csi, base + csi_offset + IRQ_CTL_CLEAR); 1103 1104 if (isys->isr_csi2_mask & status_csi) { 1105 unsigned int i; 1106 1107 for (i = 0; i < isys->pdata->ipdata->csi2.nports; i++) { 1108 /* irq from not enabled port */ 1109 if (!isys->csi2[i].base) 1110 continue; 1111 if (status_csi & isys->csi2[i].legacy_irq_mask) 1112 ipu7_isys_csi2_isr(&isys->csi2[i]); 1113 } 1114 } 1115 1116 if (!isys_isr_one(adev)) 1117 status_sw = TO_SW_IRQ_FW; 1118 else 1119 status_sw = 0; 1120 1121 status_csi = readl(base + csi_offset + IRQ_CTL_STATUS); 1122 status_sw |= readl(base + sw_offset + TO_SW_IRQ_CNTL_STATUS); 1123 } while ((status_csi & isys->isr_csi2_mask) || 1124 (status_sw & TO_SW_IRQ_FW)); 1125 1126 writel(TO_SW_IRQ_MASK, base + sw_offset + TO_SW_IRQ_CNTL_MASK_N); 1127 1128 spin_unlock(&isys->power_lock); 1129 1130 return IRQ_HANDLED; 1131 } 1132 1133 static const struct ipu7_auxdrv_data ipu7_isys_auxdrv_data = { 1134 .isr = isys_isr, 1135 .isr_threaded = NULL, 1136 .wake_isr_thread = false, 1137 }; 1138 1139 static const struct auxiliary_device_id ipu7_isys_id_table[] = { 1140 { 1141 .name = "intel_ipu7.isys", 1142 .driver_data = (kernel_ulong_t)&ipu7_isys_auxdrv_data, 1143 }, 1144 { } 1145 }; 1146 MODULE_DEVICE_TABLE(auxiliary, ipu7_isys_id_table); 1147 1148 static struct auxiliary_driver isys_driver = { 1149 .name = IPU_ISYS_NAME, 1150 .probe = isys_probe, 1151 .remove = isys_remove, 1152 .id_table = ipu7_isys_id_table, 1153 .driver = { 1154 .pm = &isys_pm_ops, 1155 }, 1156 }; 1157 1158 module_auxiliary_driver(isys_driver); 1159 1160 MODULE_AUTHOR("Bingbu Cao <bingbu.cao@intel.com>"); 1161 MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>"); 1162 MODULE_AUTHOR("Qingwu Zhang <qingwu.zhang@intel.com>"); 1163 MODULE_LICENSE("GPL"); 1164 MODULE_DESCRIPTION("Intel ipu7 input system driver"); 1165 MODULE_IMPORT_NS("INTEL_IPU7"); 1166 MODULE_IMPORT_NS("INTEL_IPU_BRIDGE"); 1167