1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/pci.h> 3 #include <linux/types.h> 4 #include <linux/iopoll.h> 5 #include <linux/bitfield.h> 6 #include <linux/module.h> 7 #include <linux/init.h> 8 #include <linux/kthread.h> 9 #include <linux/interrupt.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/ktime.h> 13 #include <linux/math64.h> 14 #include <linux/pm.h> 15 #include <linux/freezer.h> 16 #include <linux/pci_regs.h> 17 18 #include <media/v4l2-ctrls.h> 19 20 #include "hws.h" 21 #include "hws_reg.h" 22 #include "hws_video.h" 23 #include "hws_irq.h" 24 #include "hws_v4l2_ioctl.h" 25 26 #define DRV_NAME "hws" 27 #define HWS_BUSY_POLL_DELAY_US 10 28 #define HWS_BUSY_POLL_TIMEOUT_US 1000000 29 30 static unsigned long long hws_elapsed_us(u64 start_ns) 31 { 32 return div_u64(ktime_get_mono_fast_ns() - start_ns, 1000); 33 } 34 35 /* register layout inside HWS_REG_DEVICE_INFO */ 36 #define DEVINFO_VER GENMASK(7, 0) 37 #define DEVINFO_SUBVER GENMASK(15, 8) 38 #define DEVINFO_YV12 GENMASK(31, 28) 39 #define DEVINFO_HWKEY GENMASK(27, 24) 40 #define DEVINFO_PORTID GENMASK(25, 24) /* low 2 bits of HW-key */ 41 42 #define MAKE_ENTRY(__vend, __chip, __subven, __subdev, __configptr) \ 43 { .vendor = (__vend), \ 44 .device = (__chip), \ 45 .subvendor = (__subven), \ 46 .subdevice = (__subdev), \ 47 .driver_data = (unsigned long)(__configptr) } 48 49 /* 50 * PCI IDs for HWS family cards. 51 * 52 * The subsystem IDs are fixed at 0x8888:0x0007 for this family. Some boards 53 * enumerate with vendor ID 0x8888 or 0x1f33. Exact SKU names are not fully 54 * pinned down yet; update these comments when vendor documentation or INF 55 * strings are available. 56 */ 57 static const struct pci_device_id hws_pci_table[] = { 58 /* HWS family, SKU unknown. */ 59 MAKE_ENTRY(0x8888, 0x9534, 0x8888, 0x0007, NULL), 60 MAKE_ENTRY(0x1F33, 0x8534, 0x8888, 0x0007, NULL), 61 MAKE_ENTRY(0x1F33, 0x8554, 0x8888, 0x0007, NULL), 62 63 /* HWS 2x2 HDMI family. */ 64 MAKE_ENTRY(0x8888, 0x8524, 0x8888, 0x0007, NULL), 65 /* HWS 2x2 SDI family. */ 66 MAKE_ENTRY(0x1F33, 0x6524, 0x8888, 0x0007, NULL), 67 68 /* HWS X4 HDMI family. */ 69 MAKE_ENTRY(0x8888, 0x8504, 0x8888, 0x0007, NULL), 70 /* HWS X4 SDI family. */ 71 MAKE_ENTRY(0x8888, 0x6504, 0x8888, 0x0007, NULL), 72 73 /* HWS family, SKU unknown. */ 74 MAKE_ENTRY(0x8888, 0x8532, 0x8888, 0x0007, NULL), 75 MAKE_ENTRY(0x8888, 0x8512, 0x8888, 0x0007, NULL), 76 MAKE_ENTRY(0x8888, 0x8501, 0x8888, 0x0007, NULL), 77 MAKE_ENTRY(0x1F33, 0x6502, 0x8888, 0x0007, NULL), 78 79 /* HWS X4 HDMI family (alternate vendor ID). */ 80 MAKE_ENTRY(0x1F33, 0x8504, 0x8888, 0x0007, NULL), 81 /* HWS 2x2 HDMI family (alternate vendor ID). */ 82 MAKE_ENTRY(0x1F33, 0x8524, 0x8888, 0x0007, NULL), 83 84 {} 85 }; 86 87 static void enable_pcie_relaxed_ordering(struct pci_dev *dev) 88 { 89 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); 90 } 91 92 static void hws_configure_hardware_capabilities(struct hws_pcie_dev *hdev) 93 { 94 u16 id = hdev->device_id; 95 96 /* select per-chip channel counts */ 97 switch (id) { 98 case 0x9534: 99 case 0x6524: 100 case 0x8524: 101 case 0x8504: 102 case 0x6504: 103 hdev->cur_max_video_ch = 4; 104 break; 105 case 0x8532: 106 hdev->cur_max_video_ch = 2; 107 break; 108 case 0x8512: 109 case 0x6502: 110 hdev->cur_max_video_ch = 2; 111 break; 112 case 0x8501: 113 hdev->cur_max_video_ch = 1; 114 break; 115 default: 116 hdev->cur_max_video_ch = 4; 117 break; 118 } 119 120 /* universal buffer capacity */ 121 hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE; 122 123 /* decide hardware-version and program DMA max size if needed */ 124 if (hdev->device_ver > 121) { 125 if (id == 0x8501 && hdev->device_ver == 122) { 126 hdev->hw_ver = 0; 127 } else { 128 hdev->hw_ver = 1; 129 u32 dma_max = (u32)(MAX_VIDEO_SCALER_SIZE / 16); 130 131 writel(dma_max, hdev->bar0_base + HWS_REG_DMA_MAX_SIZE); 132 /* readback to flush posted MMIO write */ 133 (void)readl(hdev->bar0_base + HWS_REG_DMA_MAX_SIZE); 134 } 135 } else { 136 hdev->hw_ver = 0; 137 } 138 } 139 140 static void hws_stop_device(struct hws_pcie_dev *hws); 141 142 static void hws_log_lifecycle_snapshot(struct hws_pcie_dev *hws, 143 const char *action, 144 const char *phase) 145 { 146 struct device *dev; 147 u32 int_en, int_status, vcap, sys_status, dec_mode; 148 149 if (!hws || !hws->pdev) 150 return; 151 152 dev = &hws->pdev->dev; 153 if (!hws->bar0_base) { 154 dev_dbg(dev, 155 "lifecycle:%s:%s bar0-unmapped suspended=%d start_run=%d pci_lost=%d irq=%d\n", 156 action, phase, READ_ONCE(hws->suspended), hws->start_run, 157 hws->pci_lost, hws->irq); 158 return; 159 } 160 161 int_en = readl(hws->bar0_base + INT_EN_REG_BASE); 162 int_status = readl(hws->bar0_base + HWS_REG_INT_STATUS); 163 vcap = readl(hws->bar0_base + HWS_REG_VCAP_ENABLE); 164 sys_status = readl(hws->bar0_base + HWS_REG_SYS_STATUS); 165 dec_mode = readl(hws->bar0_base + HWS_REG_DEC_MODE); 166 167 dev_dbg(dev, 168 "lifecycle:%s:%s suspended=%d start_run=%d pci_lost=%d irq=%d INT_EN=0x%08x INT_STATUS=0x%08x VCAP=0x%08x SYS=0x%08x DEC=0x%08x\n", 169 action, phase, READ_ONCE(hws->suspended), hws->start_run, 170 hws->pci_lost, hws->irq, int_en, int_status, vcap, 171 sys_status, dec_mode); 172 } 173 174 static int read_chip_id(struct hws_pcie_dev *hdev) 175 { 176 u32 reg; 177 /* mirror PCI IDs for later switches */ 178 hdev->device_id = hdev->pdev->device; 179 hdev->vendor_id = hdev->pdev->vendor; 180 181 reg = readl(hdev->bar0_base + HWS_REG_DEVICE_INFO); 182 183 hdev->device_ver = FIELD_GET(DEVINFO_VER, reg); 184 hdev->sub_ver = FIELD_GET(DEVINFO_SUBVER, reg); 185 hdev->support_yv12 = FIELD_GET(DEVINFO_YV12, reg); 186 hdev->port_id = FIELD_GET(DEVINFO_PORTID, reg); 187 188 hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE; 189 hdev->max_channels = 4; 190 hdev->buf_allocated = false; 191 hdev->main_task = NULL; 192 hdev->start_run = false; 193 hdev->pci_lost = 0; 194 195 writel(0x00, hdev->bar0_base + HWS_REG_DEC_MODE); 196 writel(0x10, hdev->bar0_base + HWS_REG_DEC_MODE); 197 198 hws_configure_hardware_capabilities(hdev); 199 200 dev_info(&hdev->pdev->dev, 201 "chip detected: ver=%u subver=%u port=%u yv12=%u\n", 202 hdev->device_ver, hdev->sub_ver, hdev->port_id, 203 hdev->support_yv12); 204 205 return 0; 206 } 207 208 static int main_ks_thread_handle(void *data) 209 { 210 struct hws_pcie_dev *pdx = data; 211 212 set_freezable(); 213 214 while (!kthread_should_stop()) { 215 /* If we're suspending, don't touch hardware; just sleep/freeze. */ 216 if (READ_ONCE(pdx->suspended)) { 217 try_to_freeze(); 218 schedule_timeout_interruptible(msecs_to_jiffies(1000)); 219 continue; 220 } 221 222 /* avoid MMIO when suspended (guarded above) */ 223 check_video_format(pdx); 224 225 try_to_freeze(); /* cooperate with freezer each loop */ 226 227 /* Sleep 1s or until signaled to wake/stop */ 228 schedule_timeout_interruptible(msecs_to_jiffies(1000)); 229 } 230 231 dev_dbg(&pdx->pdev->dev, "%s: exiting\n", __func__); 232 return 0; 233 } 234 235 static void hws_stop_kthread_action(void *data) 236 { 237 struct hws_pcie_dev *hws = data; 238 struct task_struct *t; 239 u64 start_ns; 240 241 if (!hws) 242 return; 243 244 t = READ_ONCE(hws->main_task); 245 if (!IS_ERR_OR_NULL(t)) { 246 start_ns = ktime_get_mono_fast_ns(); 247 dev_dbg(&hws->pdev->dev, 248 "lifecycle:kthread-stop:begin task=%s[%d]\n", 249 t->comm, t->pid); 250 WRITE_ONCE(hws->main_task, NULL); 251 kthread_stop(t); 252 dev_dbg(&hws->pdev->dev, 253 "lifecycle:kthread-stop:done (%lluus)\n", 254 hws_elapsed_us(start_ns)); 255 } 256 } 257 258 static int hws_alloc_seed_buffers(struct hws_pcie_dev *hws) 259 { 260 int ch; 261 /* 64 KiB is plenty for a safe dummy; hardware needs 64-byte alignment. */ 262 const size_t need = ALIGN(64 * 1024, 64); 263 264 for (ch = 0; ch < hws->cur_max_video_ch; ch++) { 265 #if defined(CONFIG_HAS_DMA) /* normal on PCIe platforms */ 266 void *cpu = dma_alloc_coherent(&hws->pdev->dev, need, 267 &hws->scratch_vid[ch].dma, 268 GFP_KERNEL); 269 #else 270 void *cpu = NULL; 271 #endif 272 if (!cpu) { 273 dev_warn(&hws->pdev->dev, 274 "scratch: dma_alloc_coherent failed ch=%d\n", ch); 275 /* not fatal: free earlier ones and continue without seeding */ 276 while (--ch >= 0) { 277 if (hws->scratch_vid[ch].cpu) 278 dma_free_coherent(&hws->pdev->dev, 279 hws->scratch_vid[ch].size, 280 hws->scratch_vid[ch].cpu, 281 hws->scratch_vid[ch].dma); 282 hws->scratch_vid[ch].cpu = NULL; 283 hws->scratch_vid[ch].size = 0; 284 } 285 return -ENOMEM; 286 } 287 hws->scratch_vid[ch].cpu = cpu; 288 hws->scratch_vid[ch].size = need; 289 } 290 return 0; 291 } 292 293 static void hws_free_seed_buffers(struct hws_pcie_dev *hws) 294 { 295 int ch; 296 297 for (ch = 0; ch < hws->cur_max_video_ch; ch++) { 298 if (hws->scratch_vid[ch].cpu) { 299 dma_free_coherent(&hws->pdev->dev, 300 hws->scratch_vid[ch].size, 301 hws->scratch_vid[ch].cpu, 302 hws->scratch_vid[ch].dma); 303 hws->scratch_vid[ch].cpu = NULL; 304 hws->scratch_vid[ch].size = 0; 305 } 306 } 307 } 308 309 static void hws_seed_channel(struct hws_pcie_dev *hws, int ch) 310 { 311 dma_addr_t paddr = hws->scratch_vid[ch].dma; 312 u32 lo = lower_32_bits(paddr); 313 u32 hi = upper_32_bits(paddr); 314 u32 pci_addr = lo & PCI_E_BAR_ADD_LOWMASK; 315 316 lo &= PCI_E_BAR_ADD_MASK; 317 318 /* Program 64-bit BAR remap entry for this channel (table @ 0x208 + ch * 8) */ 319 writel_relaxed(hi, hws->bar0_base + 320 PCI_ADDR_TABLE_BASE + 0x208 + ch * 8); 321 writel_relaxed(lo, hws->bar0_base + 322 PCI_ADDR_TABLE_BASE + 0x208 + ch * 8 + 323 PCIE_BARADDROFSIZE); 324 325 /* Program capture engine per-channel base/half */ 326 writel_relaxed((ch + 1) * PCIEBAR_AXI_BASE + pci_addr, 327 hws->bar0_base + CVBS_IN_BUF_BASE + 328 ch * PCIE_BARADDROFSIZE); 329 330 /* Half size: use either the current format's half or half of scratch. */ 331 { 332 u32 half = hws->video[ch].pix.half_size ? 333 hws->video[ch].pix.half_size : 334 (u32)(hws->scratch_vid[ch].size / 2); 335 336 writel_relaxed(half / 16, 337 hws->bar0_base + CVBS_IN_BUF_BASE2 + 338 ch * PCIE_BARADDROFSIZE); 339 } 340 341 (void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush posted writes */ 342 } 343 344 static void hws_seed_all_channels(struct hws_pcie_dev *hws) 345 { 346 int ch; 347 348 for (ch = 0; ch < hws->cur_max_video_ch; ch++) { 349 if (hws->scratch_vid[ch].cpu) 350 hws_seed_channel(hws, ch); 351 } 352 } 353 354 static void hws_irq_mask_gate(struct hws_pcie_dev *hws) 355 { 356 writel(0x00000000, hws->bar0_base + INT_EN_REG_BASE); 357 (void)readl(hws->bar0_base + INT_EN_REG_BASE); 358 } 359 360 static void hws_irq_unmask_gate(struct hws_pcie_dev *hws) 361 { 362 writel(HWS_INT_EN_MASK, hws->bar0_base + INT_EN_REG_BASE); 363 (void)readl(hws->bar0_base + INT_EN_REG_BASE); 364 } 365 366 static void hws_irq_clear_pending(struct hws_pcie_dev *hws) 367 { 368 u32 st = readl(hws->bar0_base + HWS_REG_INT_STATUS); 369 370 if (st) { 371 writel(st, hws->bar0_base + HWS_REG_INT_STATUS); /* W1C */ 372 (void)readl(hws->bar0_base + HWS_REG_INT_STATUS); 373 } 374 } 375 376 static void hws_block_hotpaths(struct hws_pcie_dev *hws) 377 { 378 WRITE_ONCE(hws->suspended, true); 379 if (hws->irq >= 0) 380 disable_irq(hws->irq); 381 382 if (!hws->bar0_base) 383 return; 384 385 hws_irq_mask_gate(hws); 386 hws_irq_clear_pending(hws); 387 } 388 389 static int hws_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) 390 { 391 struct hws_pcie_dev *hws; 392 int i, ret, irq; 393 unsigned long irqf = 0; 394 bool v4l2_registered = false; 395 396 /* devres-backed device object */ 397 hws = devm_kzalloc(&pdev->dev, sizeof(*hws), GFP_KERNEL); 398 if (!hws) 399 return -ENOMEM; 400 401 hws->pdev = pdev; 402 hws->irq = -1; 403 hws->suspended = false; 404 pci_set_drvdata(pdev, hws); 405 406 /* 1) Enable device + bus mastering (managed) */ 407 ret = pcim_enable_device(pdev); 408 if (ret) 409 return dev_err_probe(&pdev->dev, ret, "pcim_enable_device\n"); 410 pci_set_master(pdev); 411 412 /* 2) Map BAR0 (managed) */ 413 ret = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME); 414 if (ret) 415 return dev_err_probe(&pdev->dev, ret, "pcim_iomap_regions BAR0\n"); 416 hws->bar0_base = pcim_iomap_table(pdev)[0]; 417 418 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 419 if (ret) { 420 dev_warn(&pdev->dev, 421 "64-bit DMA mask unavailable, falling back to 32-bit (%d)\n", 422 ret); 423 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 424 if (ret) 425 return dev_err_probe(&pdev->dev, ret, 426 "No suitable DMA configuration\n"); 427 } else { 428 dev_dbg(&pdev->dev, "Using 64-bit DMA mask\n"); 429 } 430 431 /* 3) Apply optional PCIe tuning. */ 432 enable_pcie_relaxed_ordering(pdev); 433 #ifdef CONFIG_ARCH_TI816X 434 pcie_set_readrq(pdev, 128); 435 #endif 436 437 /* 4) Identify chip & capabilities */ 438 read_chip_id(hws); 439 dev_info(&pdev->dev, "Device VID=0x%04x DID=0x%04x\n", 440 pdev->vendor, pdev->device); 441 hws_init_video_sys(hws, false); 442 443 /* 5) Init channels (video state, locks, vb2, ctrls) */ 444 for (i = 0; i < hws->max_channels; i++) { 445 ret = hws_video_init_channel(hws, i); 446 if (ret) { 447 dev_err(&pdev->dev, "video channel init failed (ch=%d)\n", i); 448 goto err_unwind_channels; 449 } 450 } 451 452 /* 6) Allocate scratch DMA and seed BAR table + channel base/half (legacy SetDMAAddress) */ 453 ret = hws_alloc_seed_buffers(hws); 454 if (!ret) 455 hws_seed_all_channels(hws); 456 457 /* 7) Start-run sequence. */ 458 hws_init_video_sys(hws, false); 459 460 /* A) Force legacy INTx; legacy used request_irq(pdev->irq, ..., IRQF_SHARED) */ 461 pci_intx(pdev, 1); 462 irqf = IRQF_SHARED; 463 irq = pdev->irq; 464 hws->irq = irq; 465 dev_info(&pdev->dev, "IRQ mode: legacy INTx (shared), irq=%d\n", irq); 466 467 /* B) Mask the device's global/bridge gate (INT_EN_REG_BASE) */ 468 hws_irq_mask_gate(hws); 469 470 /* C) Clear any sticky pending interrupt status (W1C) before we arm the line */ 471 hws_irq_clear_pending(hws); 472 473 /* D) Request the legacy shared interrupt line (no vectors/MSI/MSI-X) */ 474 ret = devm_request_irq(&pdev->dev, irq, hws_irq_handler, irqf, 475 dev_name(&pdev->dev), hws); 476 if (ret) { 477 dev_err(&pdev->dev, "request_irq(%d) failed: %d\n", irq, ret); 478 goto err_unwind_channels; 479 } 480 481 /* E) Set the global interrupt enable bit in main control register */ 482 { 483 u32 ctl_reg = readl(hws->bar0_base + HWS_REG_CTL); 484 485 ctl_reg |= HWS_CTL_IRQ_ENABLE_BIT; 486 writel(ctl_reg, hws->bar0_base + HWS_REG_CTL); 487 (void)readl(hws->bar0_base + HWS_REG_CTL); /* flush write */ 488 dev_info(&pdev->dev, "Global IRQ enable bit set in control register\n"); 489 } 490 491 /* F) Open the global gate just like legacy did */ 492 hws_irq_unmask_gate(hws); 493 dev_info(&pdev->dev, "INT_EN_GATE readback=0x%08x\n", 494 readl(hws->bar0_base + INT_EN_REG_BASE)); 495 496 /* 11) Register V4L2 */ 497 ret = hws_video_register(hws); 498 if (ret) { 499 dev_err(&pdev->dev, "video_register: %d\n", ret); 500 goto err_unwind_channels; 501 } 502 v4l2_registered = true; 503 504 /* 12) Background monitor thread (managed) */ 505 hws->main_task = kthread_run(main_ks_thread_handle, hws, "hws-mon"); 506 if (IS_ERR(hws->main_task)) { 507 ret = PTR_ERR(hws->main_task); 508 hws->main_task = NULL; 509 dev_err(&pdev->dev, "kthread_run: %d\n", ret); 510 goto err_unregister_va; 511 } 512 ret = devm_add_action_or_reset(&pdev->dev, hws_stop_kthread_action, hws); 513 if (ret) { 514 dev_err(&pdev->dev, "devm_add_action kthread_stop: %d\n", ret); 515 goto err_unregister_va; /* reset already stopped the thread */ 516 } 517 518 /* 13) Final: show the line is armed */ 519 dev_info(&pdev->dev, "irq handler installed on irq=%d\n", irq); 520 return 0; 521 522 err_unregister_va: 523 hws_stop_device(hws); 524 hws_video_unregister(hws); 525 hws_free_seed_buffers(hws); 526 return ret; 527 err_unwind_channels: 528 hws_free_seed_buffers(hws); 529 if (!v4l2_registered) { 530 while (--i >= 0) 531 hws_video_cleanup_channel(hws, i); 532 } 533 return ret; 534 } 535 536 static int hws_check_busy(struct hws_pcie_dev *pdx) 537 { 538 void __iomem *reg = pdx->bar0_base + HWS_REG_SYS_STATUS; 539 u32 val; 540 int ret; 541 542 /* poll until !(val & BUSY_BIT), sleeping HWS_BUSY_POLL_DELAY_US between reads */ 543 ret = readl_poll_timeout(reg, val, !(val & HWS_SYS_DMA_BUSY_BIT), 544 HWS_BUSY_POLL_DELAY_US, 545 HWS_BUSY_POLL_TIMEOUT_US); 546 if (ret) { 547 dev_err(&pdx->pdev->dev, 548 "SYS_STATUS busy bit never cleared (0x%08x)\n", val); 549 return -ETIMEDOUT; 550 } 551 552 return 0; 553 } 554 555 static void hws_stop_dsp(struct hws_pcie_dev *hws) 556 { 557 u32 status; 558 559 /* Read the decoder mode/status register */ 560 status = readl(hws->bar0_base + HWS_REG_DEC_MODE); 561 dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status); 562 563 /* If the device looks unplugged/stuck, bail out */ 564 if (status == 0xFFFFFFFF) 565 return; 566 567 /* Tell the DSP to stop */ 568 writel(0x10, hws->bar0_base + HWS_REG_DEC_MODE); 569 570 if (hws_check_busy(hws)) 571 dev_warn(&hws->pdev->dev, "DSP busy timeout on stop\n"); 572 /* Disable video capture engine in the DSP */ 573 writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE); 574 } 575 576 /* Publish stop so ISR/BH will not touch video buffers anymore. */ 577 static void hws_publish_stop_flags(struct hws_pcie_dev *hws) 578 { 579 unsigned int i; 580 581 for (i = 0; i < hws->cur_max_video_ch; ++i) { 582 struct hws_video *v = &hws->video[i]; 583 584 WRITE_ONCE(v->cap_active, false); 585 WRITE_ONCE(v->stop_requested, true); 586 } 587 588 smp_wmb(); /* make flags visible before we touch MMIO/queues */ 589 } 590 591 /* Drain engines + ISR/BH after flags are published. */ 592 static void hws_drain_after_stop(struct hws_pcie_dev *hws) 593 { 594 u32 ackmask = 0; 595 unsigned int i; 596 u64 start_ns = ktime_get_mono_fast_ns(); 597 598 /* Mask device enables: no new DMA starts. */ 599 writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE); 600 (void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush */ 601 602 /* Let any in-flight DMAs finish (best-effort). */ 603 (void)hws_check_busy(hws); 604 605 /* Ack any latched VDONE. */ 606 for (i = 0; i < hws->cur_max_video_ch; ++i) 607 ackmask |= HWS_INT_VDONE_BIT(i); 608 if (ackmask) { 609 writel(ackmask, hws->bar0_base + HWS_REG_INT_STATUS); 610 (void)readl(hws->bar0_base + HWS_REG_INT_STATUS); 611 } 612 613 /* Ensure no hard IRQ is still running. */ 614 if (hws->irq >= 0) 615 synchronize_irq(hws->irq); 616 617 dev_dbg(&hws->pdev->dev, "lifecycle:drain-after-stop:done (%lluus)\n", 618 hws_elapsed_us(start_ns)); 619 } 620 621 static void hws_stop_device(struct hws_pcie_dev *hws) 622 { 623 u32 status = readl(hws->bar0_base + HWS_REG_SYS_STATUS); 624 u64 start_ns = ktime_get_mono_fast_ns(); 625 bool live = status != 0xFFFFFFFF; 626 627 dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status); 628 if (!live) { 629 hws->pci_lost = true; 630 goto out; 631 } 632 hws_log_lifecycle_snapshot(hws, "stop-device", "begin"); 633 634 /* Make ISR/BH a no-op, then drain engines/IRQ. */ 635 hws_publish_stop_flags(hws); 636 hws_drain_after_stop(hws); 637 638 /* 1) Stop the on-board DSP */ 639 hws_stop_dsp(hws); 640 641 out: 642 hws->start_run = false; 643 if (live) 644 hws_log_lifecycle_snapshot(hws, "stop-device", "end"); 645 else 646 dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:device-lost\n"); 647 dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:done (%lluus)\n", 648 hws_elapsed_us(start_ns)); 649 dev_dbg(&hws->pdev->dev, "%s: complete\n", __func__); 650 } 651 652 static int hws_quiesce_for_transition(struct hws_pcie_dev *hws, 653 const char *action, 654 bool stop_thread) 655 { 656 struct device *dev = &hws->pdev->dev; 657 u64 start_ns = ktime_get_mono_fast_ns(); 658 u64 step_ns; 659 int vret; 660 661 hws_log_lifecycle_snapshot(hws, action, "begin"); 662 663 step_ns = ktime_get_mono_fast_ns(); 664 hws_block_hotpaths(hws); 665 dev_dbg(dev, "lifecycle:%s:block-hotpaths (%lluus)\n", action, 666 hws_elapsed_us(step_ns)); 667 hws_log_lifecycle_snapshot(hws, action, "blocked"); 668 669 if (stop_thread) { 670 step_ns = ktime_get_mono_fast_ns(); 671 hws_stop_kthread_action(hws); 672 dev_dbg(dev, "lifecycle:%s:stop-kthread (%lluus)\n", action, 673 hws_elapsed_us(step_ns)); 674 } 675 676 step_ns = ktime_get_mono_fast_ns(); 677 vret = hws_video_quiesce(hws, action); 678 dev_dbg(dev, "lifecycle:%s:video-quiesce ret=%d (%lluus)\n", action, 679 vret, hws_elapsed_us(step_ns)); 680 if (vret) 681 dev_warn(dev, "lifecycle:%s video quiesce returned %d\n", 682 action, vret); 683 684 step_ns = ktime_get_mono_fast_ns(); 685 hws_stop_device(hws); 686 dev_dbg(dev, "lifecycle:%s:stop-device (%lluus)\n", action, 687 hws_elapsed_us(step_ns)); 688 hws_log_lifecycle_snapshot(hws, action, "end"); 689 dev_dbg(dev, "lifecycle:%s:quiesce-done ret=%d (%lluus)\n", action, 690 vret, hws_elapsed_us(start_ns)); 691 692 return vret; 693 } 694 695 static void hws_remove(struct pci_dev *pdev) 696 { 697 struct hws_pcie_dev *hws = pci_get_drvdata(pdev); 698 u64 start_ns; 699 700 if (!hws) 701 return; 702 703 start_ns = ktime_get_mono_fast_ns(); 704 dev_info(&pdev->dev, "lifecycle:remove begin\n"); 705 hws_log_lifecycle_snapshot(hws, "remove", "begin"); 706 707 /* Stop the monitor thread before tearing down V4L2/vb2 objects. */ 708 hws_block_hotpaths(hws); 709 hws_stop_kthread_action(hws); 710 711 /* Stop hardware and capture cleanly. */ 712 hws_stop_device(hws); 713 714 /* Unregister V4L2 resources. */ 715 hws_video_unregister(hws); 716 717 /* Release seeded DMA buffers */ 718 hws_free_seed_buffers(hws); 719 /* kthread is stopped by the devm action registered in probe. */ 720 hws_log_lifecycle_snapshot(hws, "remove", "end"); 721 dev_info(&pdev->dev, "lifecycle:remove done (%lluus)\n", 722 hws_elapsed_us(start_ns)); 723 } 724 725 #ifdef CONFIG_PM_SLEEP 726 static int hws_pm_suspend(struct device *dev) 727 { 728 struct pci_dev *pdev = to_pci_dev(dev); 729 struct hws_pcie_dev *hws = pci_get_drvdata(pdev); 730 int vret; 731 u64 start_ns = ktime_get_mono_fast_ns(); 732 u64 step_ns; 733 734 dev_info(dev, "lifecycle:pm_suspend begin\n"); 735 vret = hws_quiesce_for_transition(hws, "pm_suspend", false); 736 737 step_ns = ktime_get_mono_fast_ns(); 738 pci_save_state(pdev); 739 pci_clear_master(pdev); 740 pci_disable_device(pdev); 741 pci_set_power_state(pdev, PCI_D3hot); 742 dev_dbg(dev, "lifecycle:pm_suspend:pci-d3hot (%lluus)\n", 743 hws_elapsed_us(step_ns)); 744 dev_info(dev, "lifecycle:pm_suspend done ret=%d (%lluus)\n", vret, 745 hws_elapsed_us(start_ns)); 746 747 return 0; 748 } 749 750 static int hws_pm_resume(struct device *dev) 751 { 752 struct pci_dev *pdev = to_pci_dev(dev); 753 struct hws_pcie_dev *hws = pci_get_drvdata(pdev); 754 int ret; 755 u64 start_ns = ktime_get_mono_fast_ns(); 756 u64 step_ns; 757 758 dev_info(dev, "lifecycle:pm_resume begin\n"); 759 760 /* Back to D0 and re-enable the function */ 761 step_ns = ktime_get_mono_fast_ns(); 762 pci_set_power_state(pdev, PCI_D0); 763 764 ret = pci_enable_device(pdev); 765 if (ret) { 766 dev_err(dev, "pci_enable_device: %d\n", ret); 767 return ret; 768 } 769 pci_restore_state(pdev); 770 pci_set_master(pdev); 771 dev_dbg(dev, "lifecycle:pm_resume:pci-enable (%lluus)\n", 772 hws_elapsed_us(step_ns)); 773 774 /* Reapply any PCIe tuning lost across D3 */ 775 enable_pcie_relaxed_ordering(pdev); 776 777 /* Reinitialize chip-side capabilities / registers */ 778 step_ns = ktime_get_mono_fast_ns(); 779 read_chip_id(hws); 780 /* Re-seed BAR remaps/DMA windows and restart the capture core */ 781 hws_seed_all_channels(hws); 782 hws_init_video_sys(hws, true); 783 hws_irq_clear_pending(hws); 784 dev_dbg(dev, "lifecycle:pm_resume:chip-reinit (%lluus)\n", 785 hws_elapsed_us(step_ns)); 786 787 /* IRQs can be re-enabled now that MMIO is sane */ 788 step_ns = ktime_get_mono_fast_ns(); 789 if (hws->irq >= 0) 790 enable_irq(hws->irq); 791 792 WRITE_ONCE(hws->suspended, false); 793 dev_dbg(dev, "lifecycle:pm_resume:irq-unsuspend (%lluus)\n", 794 hws_elapsed_us(step_ns)); 795 796 /* vb2: nothing mandatory; userspace will STREAMON again when ready */ 797 step_ns = ktime_get_mono_fast_ns(); 798 hws_video_pm_resume(hws); 799 dev_dbg(dev, "lifecycle:pm_resume:video-resume (%lluus)\n", 800 hws_elapsed_us(step_ns)); 801 hws_log_lifecycle_snapshot(hws, "pm_resume", "end"); 802 dev_info(dev, "lifecycle:pm_resume done (%lluus)\n", 803 hws_elapsed_us(start_ns)); 804 805 return 0; 806 } 807 808 static SIMPLE_DEV_PM_OPS(hws_pm_ops, hws_pm_suspend, hws_pm_resume); 809 # define HWS_PM_OPS (&hws_pm_ops) 810 #else 811 # define HWS_PM_OPS NULL 812 #endif 813 814 static void hws_shutdown(struct pci_dev *pdev) 815 { 816 struct hws_pcie_dev *hws = pci_get_drvdata(pdev); 817 int vret = 0; 818 u64 start_ns = ktime_get_mono_fast_ns(); 819 u64 step_ns; 820 821 if (!hws) 822 return; 823 824 dev_info(&pdev->dev, "lifecycle:pci_shutdown begin\n"); 825 vret = hws_quiesce_for_transition(hws, "pci_shutdown", true); 826 827 step_ns = ktime_get_mono_fast_ns(); 828 pci_clear_master(pdev); 829 dev_dbg(&pdev->dev, "lifecycle:pci_shutdown:clear-master (%lluus)\n", 830 hws_elapsed_us(step_ns)); 831 dev_info(&pdev->dev, "lifecycle:pci_shutdown done ret=%d (%lluus)\n", 832 vret, hws_elapsed_us(start_ns)); 833 } 834 835 static struct pci_driver hws_pci_driver = { 836 .name = KBUILD_MODNAME, 837 .id_table = hws_pci_table, 838 .probe = hws_probe, 839 .remove = hws_remove, 840 .shutdown = hws_shutdown, 841 .driver = { 842 .pm = HWS_PM_OPS, 843 }, 844 }; 845 846 MODULE_DEVICE_TABLE(pci, hws_pci_table); 847 848 static int __init pcie_hws_init(void) 849 { 850 return pci_register_driver(&hws_pci_driver); 851 } 852 853 static void __exit pcie_hws_exit(void) 854 { 855 pci_unregister_driver(&hws_pci_driver); 856 } 857 858 module_init(pcie_hws_init); 859 module_exit(pcie_hws_exit); 860 861 MODULE_DESCRIPTION(DRV_NAME); 862 MODULE_AUTHOR("Ben Hoff <hoff.benjamin.k@gmail.com>"); 863 MODULE_AUTHOR("Sales <sales@avmatrix.com>"); 864 MODULE_LICENSE("GPL"); 865 MODULE_IMPORT_NS("DMA_BUF"); 866