1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/interrupt.h> 5 #include <linux/iopoll.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 9 #include <drm/lima_drm.h> 10 11 #include "lima_device.h" 12 #include "lima_gp.h" 13 #include "lima_regs.h" 14 #include "lima_gem.h" 15 #include "lima_vm.h" 16 17 #define gp_write(reg, data) writel(data, ip->iomem + reg) 18 #define gp_read(reg) readl(ip->iomem + reg) 19 20 static irqreturn_t lima_gp_irq_handler(int irq, void *data) 21 { 22 struct lima_ip *ip = data; 23 struct lima_device *dev = ip->dev; 24 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 25 struct lima_sched_task *task = pipe->current_task; 26 u32 state = gp_read(LIMA_GP_INT_STAT); 27 u32 status = gp_read(LIMA_GP_STATUS); 28 bool done = false; 29 30 /* for shared irq case */ 31 if (!state) 32 return IRQ_NONE; 33 34 if (state & LIMA_GP_IRQ_MASK_ERROR) { 35 if ((state & LIMA_GP_IRQ_MASK_ERROR) == 36 LIMA_GP_IRQ_PLBU_OUT_OF_MEM) { 37 dev_dbg(dev->dev, "%s out of heap irq status=%x\n", 38 lima_ip_name(ip), status); 39 } else { 40 dev_err(dev->dev, "%s error irq state=%x status=%x\n", 41 lima_ip_name(ip), state, status); 42 if (task) 43 task->recoverable = false; 44 } 45 46 /* mask all interrupts before hard reset */ 47 gp_write(LIMA_GP_INT_MASK, 0); 48 49 pipe->error = true; 50 done = true; 51 } else { 52 bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST | 53 LIMA_GP_IRQ_PLBU_END_CMD_LST); 54 bool active = status & (LIMA_GP_STATUS_VS_ACTIVE | 55 LIMA_GP_STATUS_PLBU_ACTIVE); 56 done = valid && !active; 57 pipe->error = false; 58 } 59 60 gp_write(LIMA_GP_INT_CLEAR, state); 61 62 if (done) 63 lima_sched_pipe_task_done(pipe); 64 65 return IRQ_HANDLED; 66 } 67 68 static void lima_gp_soft_reset_async(struct lima_ip *ip) 69 { 70 if (ip->data.async_reset) 71 return; 72 73 gp_write(LIMA_GP_INT_MASK, 0); 74 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED); 75 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET); 76 ip->data.async_reset = true; 77 } 78 79 static int lima_gp_soft_reset_async_wait(struct lima_ip *ip) 80 { 81 struct lima_device *dev = ip->dev; 82 int err; 83 u32 v; 84 85 if (!ip->data.async_reset) 86 return 0; 87 88 err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v, 89 v & LIMA_GP_IRQ_RESET_COMPLETED, 90 0, 100); 91 if (err) { 92 dev_err(dev->dev, "%s soft reset time out\n", 93 lima_ip_name(ip)); 94 return err; 95 } 96 97 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); 98 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); 99 100 ip->data.async_reset = false; 101 return 0; 102 } 103 104 static int lima_gp_task_validate(struct lima_sched_pipe *pipe, 105 struct lima_sched_task *task) 106 { 107 struct drm_lima_gp_frame *frame = task->frame; 108 u32 *f = frame->frame; 109 (void)pipe; 110 111 if (f[LIMA_GP_VSCL_START_ADDR >> 2] > 112 f[LIMA_GP_VSCL_END_ADDR >> 2] || 113 f[LIMA_GP_PLBUCL_START_ADDR >> 2] > 114 f[LIMA_GP_PLBUCL_END_ADDR >> 2] || 115 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] > 116 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]) 117 return -EINVAL; 118 119 if (f[LIMA_GP_VSCL_START_ADDR >> 2] == 120 f[LIMA_GP_VSCL_END_ADDR >> 2] && 121 f[LIMA_GP_PLBUCL_START_ADDR >> 2] == 122 f[LIMA_GP_PLBUCL_END_ADDR >> 2]) 123 return -EINVAL; 124 125 return 0; 126 } 127 128 static void lima_gp_task_run(struct lima_sched_pipe *pipe, 129 struct lima_sched_task *task) 130 { 131 struct lima_ip *ip = pipe->processor[0]; 132 struct drm_lima_gp_frame *frame = task->frame; 133 u32 *f = frame->frame; 134 u32 cmd = 0; 135 int i; 136 137 /* update real heap buffer size for GP */ 138 for (i = 0; i < task->num_bos; i++) { 139 struct lima_bo *bo = task->bos[i]; 140 141 if (bo->heap_size && 142 lima_vm_get_va(task->vm, bo) == 143 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) { 144 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = 145 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + 146 bo->heap_size; 147 task->recoverable = true; 148 task->heap = bo; 149 break; 150 } 151 } 152 153 if (f[LIMA_GP_VSCL_START_ADDR >> 2] != 154 f[LIMA_GP_VSCL_END_ADDR >> 2]) 155 cmd |= LIMA_GP_CMD_START_VS; 156 if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] != 157 f[LIMA_GP_PLBUCL_END_ADDR >> 2]) 158 cmd |= LIMA_GP_CMD_START_PLBU; 159 160 /* before any hw ops, wait last success task async soft reset */ 161 lima_gp_soft_reset_async_wait(ip); 162 163 for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++) 164 writel(f[i], ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4); 165 166 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); 167 gp_write(LIMA_GP_CMD, cmd); 168 } 169 170 static int lima_gp_bus_stop_poll(struct lima_ip *ip) 171 { 172 return !!(gp_read(LIMA_GP_STATUS) & LIMA_GP_STATUS_BUS_STOPPED); 173 } 174 175 static int lima_gp_hard_reset_poll(struct lima_ip *ip) 176 { 177 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000); 178 return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000; 179 } 180 181 static int lima_gp_hard_reset(struct lima_ip *ip) 182 { 183 struct lima_device *dev = ip->dev; 184 int ret; 185 186 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000); 187 gp_write(LIMA_GP_INT_MASK, 0); 188 189 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_STOP_BUS); 190 ret = lima_poll_timeout(ip, lima_gp_bus_stop_poll, 10, 100); 191 if (ret) { 192 dev_err(dev->dev, "%s bus stop timeout\n", lima_ip_name(ip)); 193 return ret; 194 } 195 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET); 196 ret = lima_poll_timeout(ip, lima_gp_hard_reset_poll, 10, 100); 197 if (ret) { 198 dev_err(dev->dev, "%s hard reset timeout\n", lima_ip_name(ip)); 199 return ret; 200 } 201 202 gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0); 203 gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); 204 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); 205 206 /* 207 * if there was an async soft reset queued, 208 * don't wait for it in the next job 209 */ 210 ip->data.async_reset = false; 211 212 return 0; 213 } 214 215 static void lima_gp_task_fini(struct lima_sched_pipe *pipe) 216 { 217 lima_gp_soft_reset_async(pipe->processor[0]); 218 } 219 220 static void lima_gp_task_error(struct lima_sched_pipe *pipe) 221 { 222 struct lima_ip *ip = pipe->processor[0]; 223 224 dev_err(ip->dev->dev, "%s task error int_state=%x status=%x\n", 225 lima_ip_name(ip), gp_read(LIMA_GP_INT_STAT), 226 gp_read(LIMA_GP_STATUS)); 227 228 lima_gp_hard_reset(ip); 229 } 230 231 static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe) 232 { 233 lima_sched_pipe_task_done(pipe); 234 } 235 236 static void lima_gp_task_mask_irq(struct lima_sched_pipe *pipe) 237 { 238 struct lima_ip *ip = pipe->processor[0]; 239 240 gp_write(LIMA_GP_INT_MASK, 0); 241 } 242 243 static int lima_gp_task_recover(struct lima_sched_pipe *pipe) 244 { 245 struct lima_ip *ip = pipe->processor[0]; 246 struct lima_sched_task *task = pipe->current_task; 247 struct drm_lima_gp_frame *frame = task->frame; 248 u32 *f = frame->frame; 249 size_t fail_size = 250 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] - 251 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]; 252 253 if (fail_size == task->heap->heap_size) { 254 int ret; 255 256 ret = lima_heap_alloc(task->heap, task->vm); 257 if (ret < 0) 258 return ret; 259 } 260 261 gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); 262 /* Resume from where we stopped, i.e. new start is old end */ 263 gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR, 264 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); 265 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = 266 f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size; 267 gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR, 268 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); 269 gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); 270 return 0; 271 } 272 273 static void lima_gp_print_version(struct lima_ip *ip) 274 { 275 u32 version, major, minor; 276 char *name; 277 278 version = gp_read(LIMA_GP_VERSION); 279 major = (version >> 8) & 0xFF; 280 minor = version & 0xFF; 281 switch (version >> 16) { 282 case 0xA07: 283 name = "mali200"; 284 break; 285 case 0xC07: 286 name = "mali300"; 287 break; 288 case 0xB07: 289 name = "mali400"; 290 break; 291 case 0xD07: 292 name = "mali450"; 293 break; 294 default: 295 name = "unknown"; 296 break; 297 } 298 dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n", 299 lima_ip_name(ip), name, major, minor); 300 } 301 302 static struct kmem_cache *lima_gp_task_slab; 303 static int lima_gp_task_slab_refcnt; 304 305 static int lima_gp_hw_init(struct lima_ip *ip) 306 { 307 ip->data.async_reset = false; 308 lima_gp_soft_reset_async(ip); 309 return lima_gp_soft_reset_async_wait(ip); 310 } 311 312 int lima_gp_resume(struct lima_ip *ip) 313 { 314 return lima_gp_hw_init(ip); 315 } 316 317 void lima_gp_suspend(struct lima_ip *ip) 318 { 319 320 } 321 322 int lima_gp_init(struct lima_ip *ip) 323 { 324 struct lima_device *dev = ip->dev; 325 int err; 326 327 lima_gp_print_version(ip); 328 329 err = lima_gp_hw_init(ip); 330 if (err) 331 return err; 332 333 err = devm_request_irq(dev->dev, ip->irq, lima_gp_irq_handler, 334 IRQF_SHARED, lima_ip_name(ip), ip); 335 if (err) { 336 dev_err(dev->dev, "%s fail to request irq\n", 337 lima_ip_name(ip)); 338 return err; 339 } 340 341 dev->gp_version = gp_read(LIMA_GP_VERSION); 342 343 return 0; 344 } 345 346 void lima_gp_fini(struct lima_ip *ip) 347 { 348 struct lima_device *dev = ip->dev; 349 350 devm_free_irq(dev->dev, ip->irq, ip); 351 } 352 353 int lima_gp_pipe_init(struct lima_device *dev) 354 { 355 int frame_size = sizeof(struct drm_lima_gp_frame); 356 struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; 357 358 if (!lima_gp_task_slab) { 359 lima_gp_task_slab = kmem_cache_create_usercopy( 360 "lima_gp_task", sizeof(struct lima_sched_task) + frame_size, 361 0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task), 362 frame_size, NULL); 363 if (!lima_gp_task_slab) 364 return -ENOMEM; 365 } 366 lima_gp_task_slab_refcnt++; 367 368 pipe->frame_size = frame_size; 369 pipe->task_slab = lima_gp_task_slab; 370 371 pipe->task_validate = lima_gp_task_validate; 372 pipe->task_run = lima_gp_task_run; 373 pipe->task_fini = lima_gp_task_fini; 374 pipe->task_error = lima_gp_task_error; 375 pipe->task_mmu_error = lima_gp_task_mmu_error; 376 pipe->task_recover = lima_gp_task_recover; 377 pipe->task_mask_irq = lima_gp_task_mask_irq; 378 379 return 0; 380 } 381 382 void lima_gp_pipe_fini(struct lima_device *dev) 383 { 384 if (!--lima_gp_task_slab_refcnt) { 385 kmem_cache_destroy(lima_gp_task_slab); 386 lima_gp_task_slab = NULL; 387 } 388 } 389