1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2014-2018 Broadcom */ 3 4 /** 5 * DOC: Interrupt management for the V3D engine 6 * 7 * When we take a bin, render, TFU done, or CSD done interrupt, we 8 * need to signal the fence for that job so that the scheduler can 9 * queue up the next one and unblock any waiters. 10 * 11 * When we take the binner out of memory interrupt, we need to 12 * allocate some new memory and pass it to the binner so that the 13 * current job can make progress. 14 */ 15 16 #include <linux/platform_device.h> 17 #include <linux/sched/clock.h> 18 19 #include <drm/drm_print.h> 20 21 #include "v3d_drv.h" 22 #include "v3d_regs.h" 23 #include "v3d_trace.h" 24 25 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM | \ 26 V3D_INT_FLDONE | \ 27 V3D_INT_FRDONE | \ 28 V3D_INT_CSDDONE(ver) | \ 29 (ver < 71 ? V3D_INT_GMPV : 0))) 30 31 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV | \ 32 V3D_HUB_INT_MMU_PTI | \ 33 V3D_HUB_INT_MMU_CAP | \ 34 V3D_HUB_INT_TFUC | \ 35 (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0))) 36 37 static irqreturn_t 38 v3d_hub_irq(int irq, void *arg); 39 40 static void 41 v3d_overflow_mem_work(struct work_struct *work) 42 { 43 struct v3d_dev *v3d = 44 container_of(work, struct v3d_dev, overflow_mem_work); 45 struct drm_device *dev = &v3d->drm; 46 struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024); 47 struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; 48 struct v3d_bin_job *bin_job; 49 struct drm_gem_object *obj; 50 unsigned long irqflags; 51 52 if (IS_ERR(bo)) { 53 drm_err(dev, "Couldn't allocate binner overflow mem\n"); 54 return; 55 } 56 obj = &bo->base.base; 57 58 /* We lost a race, and our work task came in after the bin job 59 * completed and exited. This can happen because the HW 60 * signals OOM before it's fully OOM, so the binner might just 61 * barely complete. 62 * 63 * If we lose the race and our work task comes in after a new 64 * bin job got scheduled, that's fine. We'll just give them 65 * some binner pool anyway. 66 */ 67 spin_lock_irqsave(&queue->queue_lock, irqflags); 68 bin_job = (struct v3d_bin_job *)queue->active_job; 69 70 if (!bin_job) { 71 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 72 goto out; 73 } 74 75 drm_gem_object_get(obj); 76 list_add_tail(&bo->unref_head, &bin_job->render->unref_list); 77 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 78 79 v3d_mmu_flush_all(v3d); 80 81 V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT); 82 V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size); 83 84 out: 85 drm_gem_object_put(obj); 86 } 87 88 static void 89 v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q, 90 void (*trace_irq)(struct drm_device *, uint64_t)) 91 { 92 struct v3d_queue_state *queue = &v3d->queue[q]; 93 struct v3d_job *job = queue->active_job; 94 struct v3d_fence *fence = to_v3d_fence(job->irq_fence); 95 96 v3d_perfmon_stop(v3d, job->perfmon, true); 97 98 v3d_job_update_stats(job); 99 trace_irq(&v3d->drm, fence->seqno); 100 101 queue->active_job = NULL; 102 dma_fence_signal(&fence->base); 103 } 104 105 static irqreturn_t 106 v3d_irq(int irq, void *arg) 107 { 108 struct v3d_dev *v3d = arg; 109 u32 intsts; 110 irqreturn_t status = IRQ_NONE; 111 112 intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS); 113 114 /* Acknowledge the interrupts we're handling here. */ 115 V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts); 116 117 if (intsts & V3D_INT_OUTOMEM) { 118 /* Note that the OOM status is edge signaled, so the 119 * interrupt won't happen again until the we actually 120 * add more memory. Also, as of V3D 4.1, FLDONE won't 121 * be reported until any OOM state has been cleared. 122 */ 123 schedule_work(&v3d->overflow_mem_work); 124 status = IRQ_HANDLED; 125 } 126 127 if (intsts & V3D_INT_FLDONE) { 128 v3d_irq_signal_fence(v3d, V3D_BIN, trace_v3d_bcl_irq); 129 status = IRQ_HANDLED; 130 } 131 132 if (intsts & V3D_INT_FRDONE) { 133 v3d_irq_signal_fence(v3d, V3D_RENDER, trace_v3d_rcl_irq); 134 status = IRQ_HANDLED; 135 } 136 137 if (intsts & V3D_INT_CSDDONE(v3d->ver)) { 138 v3d_irq_signal_fence(v3d, V3D_CSD, trace_v3d_csd_irq); 139 status = IRQ_HANDLED; 140 } 141 142 /* We shouldn't be triggering these if we have GMP in 143 * always-allowed mode. 144 */ 145 if (v3d->ver < V3D_GEN_71 && (intsts & V3D_INT_GMPV)) 146 drm_err(&v3d->drm, "GMP violation\n"); 147 148 /* V3D 4.2 wires the hub and core IRQs together, so if we & 149 * didn't see the common one then check hub for MMU IRQs. 150 */ 151 if (v3d->single_irq_line && status == IRQ_NONE) 152 return v3d_hub_irq(irq, arg); 153 154 return status; 155 } 156 157 static irqreturn_t 158 v3d_hub_irq(int irq, void *arg) 159 { 160 struct v3d_dev *v3d = arg; 161 u32 intsts; 162 irqreturn_t status = IRQ_NONE; 163 164 intsts = V3D_READ(V3D_HUB_INT_STS); 165 166 /* Acknowledge the interrupts we're handling here. */ 167 V3D_WRITE(V3D_HUB_INT_CLR, intsts); 168 169 if (intsts & V3D_HUB_INT_TFUC) { 170 v3d_irq_signal_fence(v3d, V3D_TFU, trace_v3d_tfu_irq); 171 status = IRQ_HANDLED; 172 } 173 174 if (intsts & (V3D_HUB_INT_MMU_WRV | 175 V3D_HUB_INT_MMU_PTI | 176 V3D_HUB_INT_MMU_CAP)) { 177 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID); 178 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) << 179 (v3d->va_width - 32)); 180 static const struct { 181 u32 begin; 182 u32 end; 183 const char *client; 184 } v3d41_axi_ids[] = { 185 {0x00, 0x20, "L2T"}, 186 {0x20, 0x21, "PTB"}, 187 {0x40, 0x41, "PSE"}, 188 {0x60, 0x80, "TLB"}, 189 {0x80, 0x88, "CLE"}, 190 {0xA0, 0xA1, "TFU"}, 191 {0xC0, 0xE0, "MMU"}, 192 {0xE0, 0xE1, "GMP"}, 193 }, v3d71_axi_ids[] = { 194 {0x00, 0x30, "L2T"}, 195 {0x30, 0x38, "CLE"}, 196 {0x38, 0x39, "PTB"}, 197 {0x39, 0x3A, "PSE"}, 198 {0x3A, 0x3B, "CSD"}, 199 {0x40, 0x60, "TLB"}, 200 {0x60, 0x70, "MMU"}, 201 {0x7C, 0x7E, "TFU"}, 202 {0x7F, 0x80, "GMP"}, 203 }; 204 const char *client = "?"; 205 206 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL)); 207 208 if (v3d->ver >= V3D_GEN_71) { 209 size_t i; 210 211 axi_id = axi_id & 0x7F; 212 for (i = 0; i < ARRAY_SIZE(v3d71_axi_ids); i++) { 213 if (axi_id >= v3d71_axi_ids[i].begin && 214 axi_id < v3d71_axi_ids[i].end) { 215 client = v3d71_axi_ids[i].client; 216 break; 217 } 218 } 219 } else if (v3d->ver >= V3D_GEN_41) { 220 size_t i; 221 222 axi_id = axi_id & 0xFF; 223 for (i = 0; i < ARRAY_SIZE(v3d41_axi_ids); i++) { 224 if (axi_id >= v3d41_axi_ids[i].begin && 225 axi_id < v3d41_axi_ids[i].end) { 226 client = v3d41_axi_ids[i].client; 227 break; 228 } 229 } 230 } 231 232 drm_dbg(&v3d->drm, "MMU error from client %s (0x%x) at 0x%llx%s%s%s\n", 233 client, axi_id, (long long)vio_addr, 234 ((intsts & V3D_HUB_INT_MMU_WRV) ? 235 ", write violation" : ""), 236 ((intsts & V3D_HUB_INT_MMU_PTI) ? 237 ", pte invalid" : ""), 238 ((intsts & V3D_HUB_INT_MMU_CAP) ? 239 ", cap exceeded" : "")); 240 status = IRQ_HANDLED; 241 } 242 243 if (v3d->ver >= V3D_GEN_71 && (intsts & V3D_V7_HUB_INT_GMPV)) { 244 drm_err(&v3d->drm, "GMP Violation\n"); 245 status = IRQ_HANDLED; 246 } 247 248 return status; 249 } 250 251 int 252 v3d_irq_init(struct v3d_dev *v3d) 253 { 254 int irq, ret; 255 256 INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work); 257 258 irq = platform_get_irq_optional(v3d_to_pdev(v3d), 1); 259 if (irq == -EPROBE_DEFER) 260 return irq; 261 if (irq > 0) { 262 v3d->irq[V3D_CORE_IRQ] = irq; 263 264 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 265 v3d_irq, IRQF_SHARED, 266 "v3d_core0", v3d); 267 if (ret) 268 goto fail; 269 270 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 271 if (irq < 0) 272 return irq; 273 v3d->irq[V3D_HUB_IRQ] = irq; 274 275 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_HUB_IRQ], 276 v3d_hub_irq, IRQF_SHARED, 277 "v3d_hub", v3d); 278 if (ret) 279 goto fail; 280 } else { 281 v3d->single_irq_line = true; 282 283 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 284 if (irq < 0) 285 return irq; 286 v3d->irq[V3D_CORE_IRQ] = irq; 287 288 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 289 v3d_irq, IRQF_SHARED, 290 "v3d", v3d); 291 if (ret) 292 goto fail; 293 } 294 295 return 0; 296 297 fail: 298 if (ret != -EPROBE_DEFER) 299 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret); 300 return ret; 301 } 302 303 void 304 v3d_irq_enable(struct v3d_dev *v3d) 305 { 306 int core; 307 308 /* Clear any pending interrupts someone might have left around for us. */ 309 for (core = 0; core < v3d->cores; core++) 310 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 311 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 312 313 /* Enable our set of interrupts, masking out any others. */ 314 for (core = 0; core < v3d->cores; core++) { 315 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver)); 316 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver)); 317 } 318 319 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver)); 320 V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver)); 321 } 322 323 void 324 v3d_irq_disable(struct v3d_dev *v3d) 325 { 326 int core; 327 328 /* Disable all interrupts. */ 329 for (core = 0; core < v3d->cores; core++) 330 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0); 331 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0); 332 333 /* Finish any interrupt handler still in flight. */ 334 for (int i = 0; i < V3D_MAX_IRQS; i++) { 335 if (v3d->irq[i]) 336 synchronize_irq(v3d->irq[i]); 337 } 338 339 /* Clear any pending interrupts we might have left. */ 340 for (core = 0; core < v3d->cores; core++) 341 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 342 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 343 344 cancel_work_sync(&v3d->overflow_mem_work); 345 } 346 347 /** Reinitializes interrupt registers when a GPU reset is performed. */ 348 void v3d_irq_reset(struct v3d_dev *v3d) 349 { 350 v3d_irq_enable(v3d); 351 } 352