1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2014-2018 Broadcom */ 3 4 /** 5 * DOC: Interrupt management for the V3D engine 6 * 7 * When we take a bin, render, TFU done, or CSD done interrupt, we 8 * need to signal the fence for that job so that the scheduler can 9 * queue up the next one and unblock any waiters. 10 * 11 * When we take the binner out of memory interrupt, we need to 12 * allocate some new memory and pass it to the binner so that the 13 * current job can make progress. 14 */ 15 16 #include <linux/platform_device.h> 17 #include <linux/sched/clock.h> 18 19 #include <drm/drm_print.h> 20 21 #include "v3d_drv.h" 22 #include "v3d_regs.h" 23 #include "v3d_trace.h" 24 25 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM | \ 26 V3D_INT_FLDONE | \ 27 V3D_INT_FRDONE | \ 28 V3D_INT_CSDDONE(ver) | \ 29 (ver < 71 ? V3D_INT_GMPV : 0))) 30 31 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV | \ 32 V3D_HUB_INT_MMU_PTI | \ 33 V3D_HUB_INT_MMU_CAP | \ 34 V3D_HUB_INT_TFUC | \ 35 (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0))) 36 37 static irqreturn_t 38 v3d_hub_irq(int irq, void *arg); 39 40 static void 41 v3d_overflow_mem_work(struct work_struct *work) 42 { 43 struct v3d_dev *v3d = 44 container_of(work, struct v3d_dev, overflow_mem_work); 45 struct drm_device *dev = &v3d->drm; 46 struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024); 47 struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; 48 struct v3d_bin_job *bin_job; 49 struct drm_gem_object *obj; 50 unsigned long irqflags; 51 52 if (IS_ERR(bo)) { 53 DRM_ERROR("Couldn't allocate binner overflow mem\n"); 54 return; 55 } 56 obj = &bo->base.base; 57 58 /* We lost a race, and our work task came in after the bin job 59 * completed and exited. This can happen because the HW 60 * signals OOM before it's fully OOM, so the binner might just 61 * barely complete. 62 * 63 * If we lose the race and our work task comes in after a new 64 * bin job got scheduled, that's fine. We'll just give them 65 * some binner pool anyway. 66 */ 67 spin_lock_irqsave(&queue->queue_lock, irqflags); 68 bin_job = (struct v3d_bin_job *)queue->active_job; 69 70 if (!bin_job) { 71 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 72 goto out; 73 } 74 75 drm_gem_object_get(obj); 76 list_add_tail(&bo->unref_head, &bin_job->render->unref_list); 77 spin_unlock_irqrestore(&queue->queue_lock, irqflags); 78 79 v3d_mmu_flush_all(v3d); 80 81 V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT); 82 V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size); 83 84 out: 85 drm_gem_object_put(obj); 86 } 87 88 static void 89 v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q, 90 void (*trace_irq)(struct drm_device *, uint64_t)) 91 { 92 struct v3d_queue_state *queue = &v3d->queue[q]; 93 struct v3d_fence *fence = to_v3d_fence(queue->active_job->irq_fence); 94 95 v3d_job_update_stats(queue->active_job, q); 96 trace_irq(&v3d->drm, fence->seqno); 97 98 queue->active_job = NULL; 99 dma_fence_signal(&fence->base); 100 } 101 102 static irqreturn_t 103 v3d_irq(int irq, void *arg) 104 { 105 struct v3d_dev *v3d = arg; 106 u32 intsts; 107 irqreturn_t status = IRQ_NONE; 108 109 intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS); 110 111 /* Acknowledge the interrupts we're handling here. */ 112 V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts); 113 114 if (intsts & V3D_INT_OUTOMEM) { 115 /* Note that the OOM status is edge signaled, so the 116 * interrupt won't happen again until the we actually 117 * add more memory. Also, as of V3D 4.1, FLDONE won't 118 * be reported until any OOM state has been cleared. 119 */ 120 schedule_work(&v3d->overflow_mem_work); 121 status = IRQ_HANDLED; 122 } 123 124 if (intsts & V3D_INT_FLDONE) { 125 v3d_irq_signal_fence(v3d, V3D_BIN, trace_v3d_bcl_irq); 126 status = IRQ_HANDLED; 127 } 128 129 if (intsts & V3D_INT_FRDONE) { 130 v3d_irq_signal_fence(v3d, V3D_RENDER, trace_v3d_rcl_irq); 131 status = IRQ_HANDLED; 132 } 133 134 if (intsts & V3D_INT_CSDDONE(v3d->ver)) { 135 v3d_irq_signal_fence(v3d, V3D_CSD, trace_v3d_csd_irq); 136 status = IRQ_HANDLED; 137 } 138 139 /* We shouldn't be triggering these if we have GMP in 140 * always-allowed mode. 141 */ 142 if (v3d->ver < V3D_GEN_71 && (intsts & V3D_INT_GMPV)) 143 dev_err(v3d->drm.dev, "GMP violation\n"); 144 145 /* V3D 4.2 wires the hub and core IRQs together, so if we & 146 * didn't see the common one then check hub for MMU IRQs. 147 */ 148 if (v3d->single_irq_line && status == IRQ_NONE) 149 return v3d_hub_irq(irq, arg); 150 151 return status; 152 } 153 154 static irqreturn_t 155 v3d_hub_irq(int irq, void *arg) 156 { 157 struct v3d_dev *v3d = arg; 158 u32 intsts; 159 irqreturn_t status = IRQ_NONE; 160 161 intsts = V3D_READ(V3D_HUB_INT_STS); 162 163 /* Acknowledge the interrupts we're handling here. */ 164 V3D_WRITE(V3D_HUB_INT_CLR, intsts); 165 166 if (intsts & V3D_HUB_INT_TFUC) { 167 v3d_irq_signal_fence(v3d, V3D_TFU, trace_v3d_tfu_irq); 168 status = IRQ_HANDLED; 169 } 170 171 if (intsts & (V3D_HUB_INT_MMU_WRV | 172 V3D_HUB_INT_MMU_PTI | 173 V3D_HUB_INT_MMU_CAP)) { 174 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID); 175 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) << 176 (v3d->va_width - 32)); 177 static const struct { 178 u32 begin; 179 u32 end; 180 const char *client; 181 } v3d41_axi_ids[] = { 182 {0x00, 0x20, "L2T"}, 183 {0x20, 0x21, "PTB"}, 184 {0x40, 0x41, "PSE"}, 185 {0x60, 0x80, "TLB"}, 186 {0x80, 0x88, "CLE"}, 187 {0xA0, 0xA1, "TFU"}, 188 {0xC0, 0xE0, "MMU"}, 189 {0xE0, 0xE1, "GMP"}, 190 }, v3d71_axi_ids[] = { 191 {0x00, 0x30, "L2T"}, 192 {0x30, 0x38, "CLE"}, 193 {0x38, 0x39, "PTB"}, 194 {0x39, 0x3A, "PSE"}, 195 {0x3A, 0x3B, "CSD"}, 196 {0x40, 0x60, "TLB"}, 197 {0x60, 0x70, "MMU"}, 198 {0x7C, 0x7E, "TFU"}, 199 {0x7F, 0x80, "GMP"}, 200 }; 201 const char *client = "?"; 202 203 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL)); 204 205 if (v3d->ver >= V3D_GEN_71) { 206 size_t i; 207 208 axi_id = axi_id & 0x7F; 209 for (i = 0; i < ARRAY_SIZE(v3d71_axi_ids); i++) { 210 if (axi_id >= v3d71_axi_ids[i].begin && 211 axi_id < v3d71_axi_ids[i].end) { 212 client = v3d71_axi_ids[i].client; 213 break; 214 } 215 } 216 } else if (v3d->ver >= V3D_GEN_41) { 217 size_t i; 218 219 axi_id = axi_id & 0xFF; 220 for (i = 0; i < ARRAY_SIZE(v3d41_axi_ids); i++) { 221 if (axi_id >= v3d41_axi_ids[i].begin && 222 axi_id < v3d41_axi_ids[i].end) { 223 client = v3d41_axi_ids[i].client; 224 break; 225 } 226 } 227 } 228 229 dev_err(v3d->drm.dev, "MMU error from client %s (0x%x) at 0x%llx%s%s%s\n", 230 client, axi_id, (long long)vio_addr, 231 ((intsts & V3D_HUB_INT_MMU_WRV) ? 232 ", write violation" : ""), 233 ((intsts & V3D_HUB_INT_MMU_PTI) ? 234 ", pte invalid" : ""), 235 ((intsts & V3D_HUB_INT_MMU_CAP) ? 236 ", cap exceeded" : "")); 237 status = IRQ_HANDLED; 238 } 239 240 if (v3d->ver >= V3D_GEN_71 && (intsts & V3D_V7_HUB_INT_GMPV)) { 241 dev_err(v3d->drm.dev, "GMP Violation\n"); 242 status = IRQ_HANDLED; 243 } 244 245 return status; 246 } 247 248 int 249 v3d_irq_init(struct v3d_dev *v3d) 250 { 251 int irq, ret, core; 252 253 INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work); 254 255 /* Clear any pending interrupts someone might have left around 256 * for us. 257 */ 258 for (core = 0; core < v3d->cores; core++) 259 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 260 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 261 262 irq = platform_get_irq_optional(v3d_to_pdev(v3d), 1); 263 if (irq == -EPROBE_DEFER) 264 return irq; 265 if (irq > 0) { 266 v3d->irq[V3D_CORE_IRQ] = irq; 267 268 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 269 v3d_irq, IRQF_SHARED, 270 "v3d_core0", v3d); 271 if (ret) 272 goto fail; 273 274 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 275 if (irq < 0) 276 return irq; 277 v3d->irq[V3D_HUB_IRQ] = irq; 278 279 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_HUB_IRQ], 280 v3d_hub_irq, IRQF_SHARED, 281 "v3d_hub", v3d); 282 if (ret) 283 goto fail; 284 } else { 285 v3d->single_irq_line = true; 286 287 irq = platform_get_irq(v3d_to_pdev(v3d), 0); 288 if (irq < 0) 289 return irq; 290 v3d->irq[V3D_CORE_IRQ] = irq; 291 292 ret = devm_request_irq(v3d->drm.dev, v3d->irq[V3D_CORE_IRQ], 293 v3d_irq, IRQF_SHARED, 294 "v3d", v3d); 295 if (ret) 296 goto fail; 297 } 298 299 v3d_irq_enable(v3d); 300 return 0; 301 302 fail: 303 if (ret != -EPROBE_DEFER) 304 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret); 305 return ret; 306 } 307 308 void 309 v3d_irq_enable(struct v3d_dev *v3d) 310 { 311 int core; 312 313 /* Enable our set of interrupts, masking out any others. */ 314 for (core = 0; core < v3d->cores; core++) { 315 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver)); 316 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver)); 317 } 318 319 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver)); 320 V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver)); 321 } 322 323 void 324 v3d_irq_disable(struct v3d_dev *v3d) 325 { 326 int core; 327 328 /* Disable all interrupts. */ 329 for (core = 0; core < v3d->cores; core++) 330 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0); 331 V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0); 332 333 /* Finish any interrupt handler still in flight. */ 334 for (int i = 0; i < V3D_MAX_IRQS; i++) { 335 if (v3d->irq[i]) 336 synchronize_irq(v3d->irq[i]); 337 } 338 339 /* Clear any pending interrupts we might have left. */ 340 for (core = 0; core < v3d->cores; core++) 341 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver)); 342 V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver)); 343 344 cancel_work_sync(&v3d->overflow_mem_work); 345 } 346 347 /** Reinitializes interrupt registers when a GPU reset is performed. */ 348 void v3d_irq_reset(struct v3d_dev *v3d) 349 { 350 v3d_irq_enable(v3d); 351 } 352