1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2014-2018 Broadcom */ 3 4 /** 5 * DOC: Interrupt management for the V3D engine 6 * 7 * When we take a bin, render, TFU done, or CSD done interrupt, we 8 * need to signal the fence for that job so that the scheduler can 9 * queue up the next one and unblock any waiters. 10 * 11 * When we take the binner out of memory interrupt, we need to 12 * allocate some new memory and pass it to the binner so that the 13 * current job can make progress. 14 */ 15 16 #include <linux/platform_device.h> 17 #include <linux/sched/clock.h> 18 19 #include "v3d_drv.h" 20 #include "v3d_regs.h" 21 #include "v3d_trace.h" 22 23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM | \ 24 V3D_INT_FLDONE | \ 25 V3D_INT_FRDONE | \ 26 V3D_INT_CSDDONE(ver) | \ 27 (ver < 71 ? V3D_INT_GMPV : 0))) 28 29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV | \ 30 V3D_HUB_INT_MMU_PTI | \ 31 V3D_HUB_INT_MMU_CAP | \ 32 V3D_HUB_INT_TFUC | \ 33 (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0))) 34 35 static irqreturn_t 36 v3d_hub_irq(int irq, void *arg); 37 38 static void 39 v3d_overflow_mem_work(struct work_struct *work) 40 { 41 struct v3d_dev *v3d = 42 container_of(work, struct v3d_dev, overflow_mem_work); 43 struct drm_device *dev = &v3d->drm; 44 struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024); 45 struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; 46 struct v3d_bin_job *bin_job; 47 struct drm_gem_object *obj; 48 unsigned long irqflags; 49 50 if (IS_ERR(bo)) { 51 DRM_ERROR("Couldn't allocate binner overflow mem\n"); 52 return; 53 } 54 obj = &bo->base.base; 55 56 /* We lost a race, and our work task came in after the bin job 57 * completed and exited. This can happen because the HW 58 * signals OOM before it's fully OOM, so the binner might just 59 * barely complete. 60 * 61 * If we lose the race and our work task comes in after a new 62 * bin job got scheduled, that's fine. We'll just give them 63 * some binner pool anyway. 64 */ 65 spin_lock_irqsave(&queue->queue_lock, irqflags); 66 bin_job = (struct v3d_bin_job *)queue->active_job; 67 68 if (!bin_job) { 69 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 70 goto out; 71 } 72 73 drm_gem_object_get(obj); 74 list_add_tail(&bo->unref_head, &bin_job->render->unref_list); 75 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 76 77 v3d_mmu_flush_all(v3d); 78 79 V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT); 80 V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size); 81 82 out: 83 drm_gem_object_put(obj); 84 } 85 86 static void 87 v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q, 88 void (*trace_irq)(struct drm_device *, uint64_t)) 89 { 90 struct v3d_queue_state *queue = &v3d->queue[q]; 91 struct v3d_fence *fence = to_v3d_fence(queue->active_job->irq_fence); 92 93 v3d_job_update_stats(queue->active_job, q); 94 trace_irq(&v3d->drm, fence->seqno); 95 96 queue->active_job = NULL; 97 dma_fence_signal(&fence->base); 98 } 99 100 static irqreturn_t 101 v3d_irq(int irq, void *arg) 102 { 103 struct v3d_dev *v3d = arg; 104 u32 intsts; 105 irqreturn_t status = IRQ_NONE; 106 107 intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS); 108 109 /* Acknowledge the interrupts we're handling here. */ 110 V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts); 111 112 if (intsts & V3D_INT_OUTOMEM) { 113 /* Note that the OOM status is edge signaled, so the 114 * interrupt won't happen again until the we actually 115 * add more memory. Also, as of V3D 4.1, FLDONE won't 116 * be reported until any OOM state has been cleared. 117 */ 118 schedule_work(&v3d->overflow_mem_work); 119 status = IRQ_HANDLED; 120 } 121 122 if (intsts & V3D_INT_FLDONE) { 123 v3d_irq_signal_fence(v3d, V3D_BIN, trace_v3d_bcl_irq); 124 status = IRQ_HANDLED; 125 } 126 127 if (intsts & V3D_INT_FRDONE) { 128 v3d_irq_signal_fence(v3d, V3D_RENDER, trace_v3d_rcl_irq); 129 status = IRQ_HANDLED; 130 } 131 132 if (intsts & V3D_INT_CSDDONE(v3d->ver)) { 133 v3d_irq_signal_fence(v3d, V3D_CSD, trace_v3d_csd_irq); 134 status = IRQ_HANDLED; 135 } 136 137 /* We shouldn't be triggering these if we have GMP in 138 * always-allowed mode. 139 */ 140 if (v3d->ver < V3D_GEN_71 && (intsts & V3D_INT_GMPV)) 141 dev_err(v3d->drm.dev, "GMP violation\n"); 142 143 /* V3D 4.2 wires the hub and core IRQs together, so if we & 144 * didn't see the common one then check hub for MMU IRQs. 145 */ 146 if (v3d->single_irq_line && status == IRQ_NONE) 147 return v3d_hub_irq(irq, arg); 148 149 return status; 150 } 151 152 static irqreturn_t 153 v3d_hub_irq(int irq, void *arg) 154 { 155 struct v3d_dev *v3d = arg; 156 u32 intsts; 157 irqreturn_t status = IRQ_NONE; 158 159 intsts = V3D_READ(V3D_HUB_INT_STS); 160 161 /* Acknowledge the interrupts we're handling here. */ 162 V3D_WRITE(V3D_HUB_INT_CLR, intsts); 163 164 if (intsts & V3D_HUB_INT_TFUC) { 165 v3d_irq_signal_fence(v3d, V3D_TFU, trace_v3d_tfu_irq); 166 status = IRQ_HANDLED; 167 } 168 169 if (intsts & (V3D_HUB_INT_MMU_WRV | 170 V3D_HUB_INT_MMU_PTI | 171 V3D_HUB_INT_MMU_CAP)) { 172 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID); 173 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) << 174 (v3d->va_width - 32)); 175 static const struct { 176 u32 begin; 177 u32 end; 178 const char *client; 179 } v3d41_axi_ids[] = { 180 {0x00, 0x20, "L2T"}, 181 {0x20, 0x21, "PTB"}, 182 {0x40, 0x41, "PSE"}, 183 {0x60, 0x80, "TLB"}, 184 {0x80, 0x88, "CLE"}, 185 {0xA0, 0xA1, "TFU"}, 186 {0xC0, 0xE0, "MMU"}, 187 {0xE0, 0xE1, "GMP"}, 188 }, v3d71_axi_ids[] = { 189 {0x00, 0x30, "L2T"}, 190 {0x30, 0x38, "CLE"}, 191 {0x38, 0x39, "PTB"}, 192 {0x39, 0x3A, "PSE"}, 193 {0x3A, 0x3B, "CSD"}, 194 {0x40, 0x60, "TLB"}, 195 {0x60, 0x70, "MMU"}, 196 {0x7C, 0x7E, "TFU"}, 197 {0x7F, 0x80, "GMP"}, 198 }; 199 const char *client = "?"; 200 201 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL)); 202 203 if (v3d->ver >= V3D_GEN_71) { 204 size_t i; 205 206 axi_id = axi_id & 0x7F; 207 for (i = 0; i < ARRAY_SIZE(v3d71_axi_ids); i++) { 208 if (axi_id >= v3d71_axi_ids[i].begin && 209 axi_id < v3d71_axi_ids[i].end) { 210 client = v3d71_axi_ids[i].client; 211 break; 212 } 213 } 214 } else if (v3d->ver >= V3D_GEN_41) { 215 size_t i; 216 217 axi_id = axi_id & 0xFF; 218 for (i = 0; i < ARRAY_SIZE(v3d41_axi_ids); i++) { 219 if (axi_id >= v3d41_axi_ids[i].begin && 220 axi_id < v3d41_axi_ids[i].end) { 221 client = v3d41_axi_ids[i].client; 222 break; 223 } 224 } 225 } 226 227 dev_err(v3d->drm.dev, "MMU error from client %s (0x%x) at 0x%llx%s%s%s\n", 228 client, axi_id, (long long)vio_addr, 229 ((intsts & V3D_HUB_INT_MMU_WRV) ? 230 ", write violation" : ""), 231 ((intsts & V3D_HUB_INT_MMU_PTI) ? 232 ", pte invalid" : ""), 233 ((intsts & V3D_HUB_INT_MMU_CAP) ? 234 ", cap exceeded" : "")); 235 status = IRQ_HANDLED; 236 } 237 238 if (v3d->ver >= V3D_GEN_71 && (intsts & V3D_V7_HUB_INT_GMPV)) { 239 dev_err(v3d->drm.dev, "GMP Violation\n"); 240 status = IRQ_HANDLED; 241 } 242 243 return status; 244 } 245 246 int 247 v3d_irq_init(struct v3d_dev *v3d) 248 { 249 int irq, ret, core; 250 251 INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work); 252 253 /* Clear any pending interrupts someone might have left around 254 * for us. 255 */ 256 for (core = 0; core < v3d->cores; core++) 257 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 258 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 259 260 irq = platform_get_irq_optional(v3d_to_pdev(v3d), 1); 261 if (irq == -EPROBE_DEFER) 262 return irq; 263 if (irq > 0) { 264 v3d->irq[V3D_CORE_IRQ] = irq; 265 266 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 267 v3d_irq, IRQF_SHARED, 268 "v3d_core0", v3d); 269 if (ret) 270 goto fail; 271 272 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 273 if (irq < 0) 274 return irq; 275 v3d->irq[V3D_HUB_IRQ] = irq; 276 277 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_HUB_IRQ], 278 v3d_hub_irq, IRQF_SHARED, 279 "v3d_hub", v3d); 280 if (ret) 281 goto fail; 282 } else { 283 v3d->single_irq_line = true; 284 285 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 286 if (irq < 0) 287 return irq; 288 v3d->irq[V3D_CORE_IRQ] = irq; 289 290 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 291 v3d_irq, IRQF_SHARED, 292 "v3d", v3d); 293 if (ret) 294 goto fail; 295 } 296 297 v3d_irq_enable(v3d); 298 return 0; 299 300 fail: 301 if (ret != -EPROBE_DEFER) 302 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret); 303 return ret; 304 } 305 306 void 307 v3d_irq_enable(struct v3d_dev *v3d) 308 { 309 int core; 310 311 /* Enable our set of interrupts, masking out any others. */ 312 for (core = 0; core < v3d->cores; core++) { 313 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver)); 314 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver)); 315 } 316 317 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver)); 318 V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver)); 319 } 320 321 void 322 v3d_irq_disable(struct v3d_dev *v3d) 323 { 324 int core; 325 326 /* Disable all interrupts. */ 327 for (core = 0; core < v3d->cores; core++) 328 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0); 329 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0); 330 331 /* Finish any interrupt handler still in flight. */ 332 for (int i = 0; i < V3D_MAX_IRQS; i++) { 333 if (v3d->irq[i]) 334 synchronize_irq(v3d->irq[i]); 335 } 336 337 /* Clear any pending interrupts we might have left. */ 338 for (core = 0; core < v3d->cores; core++) 339 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 340 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 341 342 cancel_work_sync(&v3d->overflow_mem_work); 343 } 344 345 /** Reinitializes interrupt registers when a GPU reset is performed. */ 346 void v3d_irq_reset(struct v3d_dev *v3d) 347 { 348 v3d_irq_enable(v3d); 349 } 350