1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/compiler.h> 3 #include <linux/moduleparam.h> 4 #include <linux/io.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/interrupt.h> 7 #include <linux/minmax.h> 8 #include <linux/string.h> 9 10 #include <media/videobuf2-dma-contig.h> 11 12 #include "hws_irq.h" 13 #include "hws_reg.h" 14 #include "hws_video.h" 15 #include "hws.h" 16 17 #define MAX_INT_LOOPS 100 18 19 static bool hws_toggle_debug; 20 module_param_named(toggle_debug, hws_toggle_debug, bool, 0644); 21 MODULE_PARM_DESC(toggle_debug, 22 "Read toggle registers in IRQ handler for debug logging"); 23 24 static int hws_arm_next(struct hws_pcie_dev *hws, u32 ch) 25 { 26 struct hws_video *v = &hws->video[ch]; 27 unsigned long flags; 28 struct hwsvideo_buffer *buf; 29 30 dev_dbg(&hws->pdev->dev, 31 "arm_next(ch=%u): stop=%d cap=%d queued=%d\n", 32 ch, READ_ONCE(v->stop_requested), READ_ONCE(v->cap_active), 33 !list_empty(&v->capture_queue)); 34 35 if (READ_ONCE(hws->suspended)) { 36 dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): suspended\n", ch); 37 return -EBUSY; 38 } 39 40 if (READ_ONCE(v->stop_requested) || !READ_ONCE(v->cap_active)) { 41 dev_dbg(&hws->pdev->dev, 42 "arm_next(ch=%u): stop=%d cap=%d -> cancel\n", ch, 43 v->stop_requested, v->cap_active); 44 return -ECANCELED; 45 } 46 47 spin_lock_irqsave(&v->irq_lock, flags); 48 if (list_empty(&v->capture_queue)) { 49 spin_unlock_irqrestore(&v->irq_lock, flags); 50 dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): queue empty\n", ch); 51 return -EAGAIN; 52 } 53 54 buf = list_first_entry(&v->capture_queue, struct hwsvideo_buffer, list); 55 list_del_init(&buf->list); /* keep buffer safe for later cleanup */ 56 if (v->queued_count) 57 v->queued_count--; 58 v->active = buf; 59 spin_unlock_irqrestore(&v->irq_lock, flags); 60 dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): picked buffer %p\n", ch, 61 buf); 62 63 /* Publish descriptor(s) before doorbell/MMIO kicks. */ 64 wmb(); 65 66 /* Avoid MMIO during suspend */ 67 if (READ_ONCE(hws->suspended)) { 68 unsigned long f; 69 70 dev_dbg(&hws->pdev->dev, 71 "arm_next(ch=%u): suspended after pick\n", ch); 72 spin_lock_irqsave(&v->irq_lock, f); 73 if (v->active) { 74 list_add(&buf->list, &v->capture_queue); 75 v->queued_count++; 76 v->active = NULL; 77 } 78 spin_unlock_irqrestore(&v->irq_lock, f); 79 return -EBUSY; 80 } 81 82 /* Also program the DMA address register directly */ 83 { 84 dma_addr_t dma_addr = 85 vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0); 86 hws_program_dma_for_addr(hws, ch, dma_addr); 87 iowrite32(lower_32_bits(dma_addr), 88 hws->bar0_base + HWS_REG_DMA_ADDR(ch)); 89 } 90 91 dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): programmed buffer %p\n", ch, 92 buf); 93 spin_lock_irqsave(&v->irq_lock, flags); 94 hws_prime_next_locked(v); 95 spin_unlock_irqrestore(&v->irq_lock, flags); 96 return 0; 97 } 98 99 static void hws_video_handle_vdone(struct hws_video *v) 100 { 101 struct hws_pcie_dev *hws = v->parent; 102 unsigned int ch = v->channel_index; 103 struct hwsvideo_buffer *done; 104 unsigned long flags; 105 bool promoted = false; 106 107 dev_dbg(&hws->pdev->dev, 108 "bh_video(ch=%u): stop=%d cap=%d active=%p\n", 109 ch, READ_ONCE(v->stop_requested), READ_ONCE(v->cap_active), 110 v->active); 111 112 int ret; 113 114 dev_dbg(&hws->pdev->dev, 115 "bh_video(ch=%u): entry stop=%d cap=%d\n", ch, 116 v->stop_requested, v->cap_active); 117 if (READ_ONCE(hws->suspended)) 118 return; 119 120 if (READ_ONCE(v->stop_requested) || !READ_ONCE(v->cap_active)) 121 return; 122 123 spin_lock_irqsave(&v->irq_lock, flags); 124 done = v->active; 125 if (done && v->next_prepared) { 126 v->active = v->next_prepared; 127 v->next_prepared = NULL; 128 promoted = true; 129 } 130 spin_unlock_irqrestore(&v->irq_lock, flags); 131 132 /* 1) Complete the buffer the HW just finished (if any) */ 133 if (done) { 134 struct vb2_v4l2_buffer *vb2v = &done->vb; 135 size_t expected = v->pix.sizeimage; 136 size_t plane_size = vb2_plane_size(&vb2v->vb2_buf, 0); 137 138 if (expected > plane_size) { 139 dev_warn_ratelimited(&hws->pdev->dev, 140 "bh_video(ch=%u): sizeimage %zu > plane %zu, dropping seq=%u\n", 141 ch, expected, plane_size, 142 (u32)atomic_read(&v->sequence_number) + 1); 143 vb2_buffer_done(&vb2v->vb2_buf, VB2_BUF_STATE_ERROR); 144 goto arm_next; 145 } 146 vb2_set_plane_payload(&vb2v->vb2_buf, 0, expected); 147 148 dma_rmb(); /* device writes visible before userspace sees it */ 149 150 vb2v->sequence = (u32)atomic_inc_return(&v->sequence_number); 151 vb2v->vb2_buf.timestamp = ktime_get_ns(); 152 dev_dbg(&hws->pdev->dev, 153 "bh_video(ch=%u): DONE buf=%p seq=%u half_seen=%d toggle=%u\n", 154 ch, done, vb2v->sequence, v->half_seen, 155 v->last_buf_half_toggle); 156 157 if (!promoted) 158 v->active = NULL; /* channel no longer owns this buffer */ 159 vb2_buffer_done(&vb2v->vb2_buf, VB2_BUF_STATE_DONE); 160 } 161 162 if (READ_ONCE(hws->suspended)) 163 return; 164 165 if (promoted) { 166 dev_dbg(&hws->pdev->dev, 167 "bh_video(ch=%u): promoted pre-armed buffer active=%p\n", 168 ch, v->active); 169 spin_lock_irqsave(&v->irq_lock, flags); 170 hws_prime_next_locked(v); 171 spin_unlock_irqrestore(&v->irq_lock, flags); 172 return; 173 } 174 175 arm_next: 176 /* 2) Immediately arm the next queued buffer (if present) */ 177 ret = hws_arm_next(hws, ch); 178 if (ret == -EAGAIN) { 179 dev_dbg(&hws->pdev->dev, 180 "bh_video(ch=%u): no queued buffer to arm\n", ch); 181 return; 182 } 183 dev_dbg(&hws->pdev->dev, 184 "bh_video(ch=%u): armed next buffer, active=%p\n", ch, 185 v->active); 186 /* On success the engine now points at v->active's DMA address */ 187 } 188 189 irqreturn_t hws_irq_handler(int irq, void *info) 190 { 191 struct hws_pcie_dev *pdx = info; 192 u32 int_state; 193 194 dev_dbg(&pdx->pdev->dev, "irq: entry\n"); 195 if (pdx->bar0_base) { 196 dev_dbg(&pdx->pdev->dev, 197 "irq: INT_EN=0x%08x INT_STATUS=0x%08x\n", 198 readl(pdx->bar0_base + INT_EN_REG_BASE), 199 readl(pdx->bar0_base + HWS_REG_INT_STATUS)); 200 } 201 202 /* Fast path: if suspended, quietly ack and exit */ 203 if (READ_ONCE(pdx->suspended)) { 204 int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS); 205 if (int_state) { 206 writel(int_state, pdx->bar0_base + HWS_REG_INT_STATUS); 207 (void)readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS); 208 } 209 return int_state ? IRQ_HANDLED : IRQ_NONE; 210 } 211 int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS); 212 if (!int_state || int_state == 0xFFFFFFFF) { 213 dev_dbg(&pdx->pdev->dev, 214 "irq: spurious or device-gone int_state=0x%08x\n", 215 int_state); 216 return IRQ_NONE; 217 } 218 dev_dbg(&pdx->pdev->dev, "irq: entry INT_STATUS=0x%08x\n", int_state); 219 220 /* Loop until all pending bits are serviced (max 100 iterations) */ 221 for (u32 cnt = 0; int_state && cnt < MAX_INT_LOOPS; ++cnt) { 222 for (unsigned int ch = 0; ch < pdx->cur_max_video_ch; ++ch) { 223 u32 vbit = HWS_INT_VDONE_BIT(ch); 224 225 if (!(int_state & vbit)) 226 continue; 227 228 if (READ_ONCE(pdx->video[ch].cap_active) && 229 !READ_ONCE(pdx->video[ch].stop_requested)) { 230 if (hws_toggle_debug) { 231 u32 toggle = 232 readl_relaxed(pdx->bar0_base + 233 HWS_REG_VBUF_TOGGLE(ch)) & 0x01; 234 WRITE_ONCE(pdx->video[ch].last_buf_half_toggle, 235 toggle); 236 } 237 dma_rmb(); 238 WRITE_ONCE(pdx->video[ch].half_seen, true); 239 dev_dbg(&pdx->pdev->dev, 240 "irq: VDONE ch=%u toggle=%u handling inline (cap=%d)\n", 241 ch, 242 READ_ONCE(pdx->video[ch].last_buf_half_toggle), 243 READ_ONCE(pdx->video[ch].cap_active)); 244 hws_video_handle_vdone(&pdx->video[ch]); 245 } else { 246 dev_dbg(&pdx->pdev->dev, 247 "irq: VDONE ch=%u ignored (cap=%d stop=%d)\n", 248 ch, 249 READ_ONCE(pdx->video[ch].cap_active), 250 READ_ONCE(pdx->video[ch].stop_requested)); 251 } 252 253 writel(vbit, pdx->bar0_base + HWS_REG_INT_STATUS); 254 (void)readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS); 255 } 256 257 /* Re-read in case new interrupt bits popped while processing */ 258 int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS); 259 dev_dbg(&pdx->pdev->dev, 260 "irq: loop cnt=%u new INT_STATUS=0x%08x\n", cnt, 261 int_state); 262 if (cnt + 1 == MAX_INT_LOOPS) 263 dev_warn_ratelimited(&pdx->pdev->dev, 264 "IRQ storm? status=0x%08x\n", 265 int_state); 266 } 267 268 return IRQ_HANDLED; 269 } 270