1 // SPDX-License-Identifier: GPL-2.0 2 // 3 // Copyright (c) 2018 MediaTek Inc. 4 5 #include <linux/bitops.h> 6 #include <linux/clk.h> 7 #include <linux/clk-provider.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/errno.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/iopoll.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/sizes.h> 18 #include <linux/mailbox_controller.h> 19 #include <linux/mailbox/mtk-cmdq-mailbox.h> 20 #include <linux/of.h> 21 22 #define CMDQ_MBOX_AUTOSUSPEND_DELAY_MS 100 23 24 #define CMDQ_OP_CODE_MASK (0xff << CMDQ_OP_CODE_SHIFT) 25 #define CMDQ_NUM_CMD(t) (t->cmd_buf_size / CMDQ_INST_SIZE) 26 27 #define CMDQ_CURR_IRQ_STATUS 0x10 28 #define CMDQ_SYNC_TOKEN_UPDATE 0x68 29 #define CMDQ_THR_SLOT_CYCLES 0x30 30 #define CMDQ_THR_BASE 0x100 31 #define CMDQ_THR_SIZE 0x80 32 #define CMDQ_THR_WARM_RESET 0x00 33 #define CMDQ_THR_ENABLE_TASK 0x04 34 #define CMDQ_THR_SUSPEND_TASK 0x08 35 #define CMDQ_THR_CURR_STATUS 0x0c 36 #define CMDQ_THR_IRQ_STATUS 0x10 37 #define CMDQ_THR_IRQ_ENABLE 0x14 38 #define CMDQ_THR_CURR_ADDR 0x20 39 #define CMDQ_THR_END_ADDR 0x24 40 #define CMDQ_THR_WAIT_TOKEN 0x30 41 #define CMDQ_THR_PRIORITY 0x40 42 43 #define GCE_GCTL_VALUE 0x48 44 #define GCE_CTRL_BY_SW GENMASK(2, 0) 45 #define GCE_DDR_EN GENMASK(18, 16) 46 47 #define GCE_VM_ID_MAP(n) (0x5018 + (n) / 10 * 4) 48 #define GCE_VM_ID_MAP_THR_FLD_SHIFT(n) ((n) % 10 * 3) 49 #define GCE_VM_ID_MAP_HOST_VM GENMASK(2, 0) 50 #define GCE_VM_CPR_GSIZE 0x50c4 51 #define GCE_VM_CPR_GSIZE_FLD_SHIFT(vm_id) ((vm_id) * 4) 52 #define GCE_VM_CPR_GSIZE_MAX GENMASK(3, 0) 53 54 #define CMDQ_THR_ACTIVE_SLOT_CYCLES 0x3200 55 #define CMDQ_THR_ENABLED 0x1 56 #define CMDQ_THR_DISABLED 0x0 57 #define CMDQ_THR_SUSPEND 0x1 58 #define CMDQ_THR_RESUME 0x0 59 #define CMDQ_THR_STATUS_SUSPENDED BIT(1) 60 #define CMDQ_THR_DO_WARM_RESET BIT(0) 61 #define CMDQ_THR_IRQ_DONE 0x1 62 #define CMDQ_THR_IRQ_ERROR 0x12 63 #define CMDQ_THR_IRQ_EN (CMDQ_THR_IRQ_ERROR | CMDQ_THR_IRQ_DONE) 64 #define CMDQ_THR_IS_WAITING BIT(31) 65 66 #define CMDQ_JUMP_BY_OFFSET 0x10000000 67 #define CMDQ_JUMP_BY_PA 0x10000001 68 69 struct cmdq_thread { 70 struct mbox_chan *chan; 71 void __iomem *base; 72 struct list_head task_busy_list; 73 u32 priority; 74 }; 75 76 struct cmdq_task { 77 struct cmdq *cmdq; 78 struct list_head list_entry; 79 dma_addr_t pa_base; 80 struct cmdq_thread *thread; 81 struct cmdq_pkt *pkt; /* the packet sent from mailbox client */ 82 }; 83 84 struct cmdq { 85 struct mbox_controller mbox; 86 void __iomem *base; 87 int irq; 88 u32 irq_mask; 89 const struct gce_plat *pdata; 90 struct cmdq_thread *thread; 91 struct clk_bulk_data *clocks; 92 bool suspended; 93 }; 94 95 struct gce_plat { 96 u32 thread_nr; 97 u8 shift; 98 dma_addr_t mminfra_offset; 99 bool control_by_sw; 100 bool sw_ddr_en; 101 bool gce_vm; 102 u32 gce_num; 103 }; 104 105 static inline u32 cmdq_convert_gce_addr(dma_addr_t addr, const struct gce_plat *pdata) 106 { 107 /* Convert DMA addr (PA or IOVA) to GCE readable addr */ 108 return (addr + pdata->mminfra_offset) >> pdata->shift; 109 } 110 111 static inline dma_addr_t cmdq_revert_gce_addr(u32 addr, const struct gce_plat *pdata) 112 { 113 /* Revert GCE readable addr to DMA addr (PA or IOVA) */ 114 return ((dma_addr_t)addr << pdata->shift) - pdata->mminfra_offset; 115 } 116 117 void cmdq_get_mbox_priv(struct mbox_chan *chan, struct cmdq_mbox_priv *priv) 118 { 119 struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox); 120 121 priv->shift_pa = cmdq->pdata->shift; 122 priv->mminfra_offset = cmdq->pdata->mminfra_offset; 123 } 124 EXPORT_SYMBOL(cmdq_get_mbox_priv); 125 126 u8 cmdq_get_shift_pa(struct mbox_chan *chan) 127 { 128 struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox); 129 130 return cmdq->pdata->shift; 131 } 132 EXPORT_SYMBOL(cmdq_get_shift_pa); 133 134 static void cmdq_vm_init(struct cmdq *cmdq) 135 { 136 int i; 137 u32 vm_cpr_gsize = 0, vm_id_map = 0; 138 u32 *vm_map = NULL; 139 140 if (!cmdq->pdata->gce_vm) 141 return; 142 143 vm_map = kcalloc(cmdq->pdata->thread_nr, sizeof(*vm_map), GFP_KERNEL); 144 if (!vm_map) 145 return; 146 147 /* only configure the max CPR SRAM size to host vm (vm_id = 0) currently */ 148 vm_cpr_gsize = GCE_VM_CPR_GSIZE_MAX << GCE_VM_CPR_GSIZE_FLD_SHIFT(0); 149 150 /* set all thread mapping to host vm currently */ 151 for (i = 0; i < cmdq->pdata->thread_nr; i++) 152 vm_map[i] = GCE_VM_ID_MAP_HOST_VM << GCE_VM_ID_MAP_THR_FLD_SHIFT(i); 153 154 /* set the amount of CPR SRAM to allocate to each VM */ 155 writel(vm_cpr_gsize, cmdq->base + GCE_VM_CPR_GSIZE); 156 157 /* config CPR_GSIZE before setting VM_ID_MAP to avoid data leakage */ 158 for (i = 0; i < cmdq->pdata->thread_nr; i++) { 159 vm_id_map |= vm_map[i]; 160 /* config every 10 threads, e.g., thread id=0~9, 10~19, ..., into one register */ 161 if ((i + 1) % 10 == 0) { 162 writel(vm_id_map, cmdq->base + GCE_VM_ID_MAP(i)); 163 vm_id_map = 0; 164 } 165 } 166 /* config remaining threads settings */ 167 if (cmdq->pdata->thread_nr % 10 != 0) 168 writel(vm_id_map, cmdq->base + GCE_VM_ID_MAP(cmdq->pdata->thread_nr - 1)); 169 170 kfree(vm_map); 171 } 172 173 static void cmdq_gctl_value_toggle(struct cmdq *cmdq, bool ddr_enable) 174 { 175 u32 val = cmdq->pdata->control_by_sw ? GCE_CTRL_BY_SW : 0; 176 177 if (!cmdq->pdata->control_by_sw && !cmdq->pdata->sw_ddr_en) 178 return; 179 180 if (cmdq->pdata->sw_ddr_en && ddr_enable) 181 val |= GCE_DDR_EN; 182 183 writel(val, cmdq->base + GCE_GCTL_VALUE); 184 } 185 186 static int cmdq_thread_suspend(struct cmdq *cmdq, struct cmdq_thread *thread) 187 { 188 u32 status; 189 190 writel(CMDQ_THR_SUSPEND, thread->base + CMDQ_THR_SUSPEND_TASK); 191 192 /* If already disabled, treat as suspended successful. */ 193 if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED)) 194 return 0; 195 196 if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_CURR_STATUS, 197 status, status & CMDQ_THR_STATUS_SUSPENDED, 0, 10)) { 198 dev_err(cmdq->mbox.dev, "suspend GCE thread 0x%x failed\n", 199 (u32)(thread->base - cmdq->base)); 200 return -EFAULT; 201 } 202 203 return 0; 204 } 205 206 static void cmdq_thread_resume(struct cmdq_thread *thread) 207 { 208 writel(CMDQ_THR_RESUME, thread->base + CMDQ_THR_SUSPEND_TASK); 209 } 210 211 static void cmdq_init(struct cmdq *cmdq) 212 { 213 int i; 214 215 WARN_ON(clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks)); 216 217 cmdq_vm_init(cmdq); 218 cmdq_gctl_value_toggle(cmdq, true); 219 220 writel(CMDQ_THR_ACTIVE_SLOT_CYCLES, cmdq->base + CMDQ_THR_SLOT_CYCLES); 221 for (i = 0; i <= CMDQ_MAX_EVENT; i++) 222 writel(i, cmdq->base + CMDQ_SYNC_TOKEN_UPDATE); 223 clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks); 224 } 225 226 static int cmdq_thread_reset(struct cmdq *cmdq, struct cmdq_thread *thread) 227 { 228 u32 warm_reset; 229 230 writel(CMDQ_THR_DO_WARM_RESET, thread->base + CMDQ_THR_WARM_RESET); 231 if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_WARM_RESET, 232 warm_reset, !(warm_reset & CMDQ_THR_DO_WARM_RESET), 233 0, 10)) { 234 dev_err(cmdq->mbox.dev, "reset GCE thread 0x%x failed\n", 235 (u32)(thread->base - cmdq->base)); 236 return -EFAULT; 237 } 238 239 return 0; 240 } 241 242 static void cmdq_thread_disable(struct cmdq *cmdq, struct cmdq_thread *thread) 243 { 244 cmdq_thread_reset(cmdq, thread); 245 writel(CMDQ_THR_DISABLED, thread->base + CMDQ_THR_ENABLE_TASK); 246 } 247 248 /* notify GCE to re-fetch commands by setting GCE thread PC */ 249 static void cmdq_thread_invalidate_fetched_data(struct cmdq_thread *thread) 250 { 251 writel(readl(thread->base + CMDQ_THR_CURR_ADDR), 252 thread->base + CMDQ_THR_CURR_ADDR); 253 } 254 255 static void cmdq_task_insert_into_thread(struct cmdq_task *task) 256 { 257 struct device *dev = task->cmdq->mbox.dev; 258 struct cmdq_thread *thread = task->thread; 259 struct cmdq_task *prev_task = list_last_entry( 260 &thread->task_busy_list, typeof(*task), list_entry); 261 u64 *prev_task_base = prev_task->pkt->va_base; 262 u32 gce_addr = cmdq_convert_gce_addr(task->pa_base, task->cmdq->pdata); 263 264 /* let previous task jump to this task */ 265 dma_sync_single_for_cpu(dev, prev_task->pa_base, 266 prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE); 267 prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] = (u64)CMDQ_JUMP_BY_PA << 32 | gce_addr; 268 dma_sync_single_for_device(dev, prev_task->pa_base, 269 prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE); 270 271 cmdq_thread_invalidate_fetched_data(thread); 272 } 273 274 static bool cmdq_thread_is_in_wfe(struct cmdq_thread *thread) 275 { 276 return readl(thread->base + CMDQ_THR_WAIT_TOKEN) & CMDQ_THR_IS_WAITING; 277 } 278 279 static void cmdq_task_exec_done(struct cmdq_task *task, int sta) 280 { 281 struct cmdq_cb_data data; 282 283 data.sta = sta; 284 data.pkt = task->pkt; 285 mbox_chan_received_data(task->thread->chan, &data); 286 287 list_del(&task->list_entry); 288 } 289 290 static void cmdq_task_handle_error(struct cmdq_task *task) 291 { 292 struct cmdq_thread *thread = task->thread; 293 struct cmdq_task *next_task; 294 struct cmdq *cmdq = task->cmdq; 295 296 dev_err(cmdq->mbox.dev, "task 0x%p error\n", task); 297 WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0); 298 next_task = list_first_entry_or_null(&thread->task_busy_list, 299 struct cmdq_task, list_entry); 300 if (next_task) 301 writel(next_task->pa_base >> cmdq->pdata->shift, 302 thread->base + CMDQ_THR_CURR_ADDR); 303 cmdq_thread_resume(thread); 304 } 305 306 static void cmdq_thread_irq_handler(struct cmdq *cmdq, 307 struct cmdq_thread *thread) 308 { 309 struct cmdq_task *task, *tmp, *curr_task = NULL; 310 u32 irq_flag, gce_addr; 311 dma_addr_t curr_pa, task_end_pa; 312 bool err; 313 314 irq_flag = readl(thread->base + CMDQ_THR_IRQ_STATUS); 315 writel(~irq_flag, thread->base + CMDQ_THR_IRQ_STATUS); 316 317 /* 318 * When ISR call this function, another CPU core could run 319 * "release task" right before we acquire the spin lock, and thus 320 * reset / disable this GCE thread, so we need to check the enable 321 * bit of this GCE thread. 322 */ 323 if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED)) 324 return; 325 326 if (irq_flag & CMDQ_THR_IRQ_ERROR) 327 err = true; 328 else if (irq_flag & CMDQ_THR_IRQ_DONE) 329 err = false; 330 else 331 return; 332 333 gce_addr = readl(thread->base + CMDQ_THR_CURR_ADDR); 334 curr_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); 335 336 list_for_each_entry_safe(task, tmp, &thread->task_busy_list, 337 list_entry) { 338 task_end_pa = task->pa_base + task->pkt->cmd_buf_size; 339 if (curr_pa >= task->pa_base && curr_pa < task_end_pa) 340 curr_task = task; 341 342 if (!curr_task || curr_pa == task_end_pa - CMDQ_INST_SIZE) { 343 cmdq_task_exec_done(task, 0); 344 kfree(task); 345 } else if (err) { 346 cmdq_task_exec_done(task, -ENOEXEC); 347 cmdq_task_handle_error(curr_task); 348 kfree(task); 349 } 350 351 if (curr_task) 352 break; 353 } 354 355 if (list_empty(&thread->task_busy_list)) 356 cmdq_thread_disable(cmdq, thread); 357 } 358 359 static irqreturn_t cmdq_irq_handler(int irq, void *dev) 360 { 361 struct cmdq *cmdq = dev; 362 unsigned long irq_status, flags = 0L; 363 int bit; 364 365 irq_status = readl(cmdq->base + CMDQ_CURR_IRQ_STATUS) & cmdq->irq_mask; 366 if (!(irq_status ^ cmdq->irq_mask)) 367 return IRQ_NONE; 368 369 for_each_clear_bit(bit, &irq_status, cmdq->pdata->thread_nr) { 370 struct cmdq_thread *thread = &cmdq->thread[bit]; 371 372 spin_lock_irqsave(&thread->chan->lock, flags); 373 cmdq_thread_irq_handler(cmdq, thread); 374 spin_unlock_irqrestore(&thread->chan->lock, flags); 375 } 376 377 pm_runtime_mark_last_busy(cmdq->mbox.dev); 378 379 return IRQ_HANDLED; 380 } 381 382 static int cmdq_runtime_resume(struct device *dev) 383 { 384 struct cmdq *cmdq = dev_get_drvdata(dev); 385 int ret; 386 387 ret = clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks); 388 if (ret) 389 return ret; 390 391 cmdq_gctl_value_toggle(cmdq, true); 392 return 0; 393 } 394 395 static int cmdq_runtime_suspend(struct device *dev) 396 { 397 struct cmdq *cmdq = dev_get_drvdata(dev); 398 399 cmdq_gctl_value_toggle(cmdq, false); 400 clk_bulk_disable(cmdq->pdata->gce_num, cmdq->clocks); 401 return 0; 402 } 403 404 static int cmdq_suspend(struct device *dev) 405 { 406 struct cmdq *cmdq = dev_get_drvdata(dev); 407 struct cmdq_thread *thread; 408 int i; 409 bool task_running = false; 410 411 cmdq->suspended = true; 412 413 for (i = 0; i < cmdq->pdata->thread_nr; i++) { 414 thread = &cmdq->thread[i]; 415 if (!list_empty(&thread->task_busy_list)) { 416 task_running = true; 417 break; 418 } 419 } 420 421 if (task_running) 422 dev_warn(dev, "exist running task(s) in suspend\n"); 423 424 return pm_runtime_force_suspend(dev); 425 } 426 427 static int cmdq_resume(struct device *dev) 428 { 429 struct cmdq *cmdq = dev_get_drvdata(dev); 430 431 WARN_ON(pm_runtime_force_resume(dev)); 432 cmdq->suspended = false; 433 434 return 0; 435 } 436 437 static void cmdq_remove(struct platform_device *pdev) 438 { 439 struct cmdq *cmdq = platform_get_drvdata(pdev); 440 441 if (!IS_ENABLED(CONFIG_PM)) 442 cmdq_runtime_suspend(&pdev->dev); 443 444 clk_bulk_unprepare(cmdq->pdata->gce_num, cmdq->clocks); 445 } 446 447 static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) 448 { 449 struct cmdq_pkt *pkt = (struct cmdq_pkt *)data; 450 struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv; 451 struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev); 452 struct cmdq_task *task; 453 u32 gce_addr; 454 dma_addr_t curr_pa, end_pa; 455 456 /* Client should not flush new tasks if suspended. */ 457 WARN_ON(cmdq->suspended); 458 459 task = kzalloc_obj(*task, GFP_ATOMIC); 460 if (!task) 461 return -ENOMEM; 462 463 task->cmdq = cmdq; 464 INIT_LIST_HEAD(&task->list_entry); 465 task->pa_base = pkt->pa_base; 466 task->thread = thread; 467 task->pkt = pkt; 468 469 if (list_empty(&thread->task_busy_list)) { 470 /* 471 * The thread reset will clear thread related register to 0, 472 * including pc, end, priority, irq, suspend and enable. Thus 473 * set CMDQ_THR_ENABLED to CMDQ_THR_ENABLE_TASK will enable 474 * thread and make it running. 475 */ 476 WARN_ON(cmdq_thread_reset(cmdq, thread) < 0); 477 478 gce_addr = cmdq_convert_gce_addr(task->pa_base, cmdq->pdata); 479 writel(gce_addr, thread->base + CMDQ_THR_CURR_ADDR); 480 gce_addr = cmdq_convert_gce_addr(task->pa_base + pkt->cmd_buf_size, cmdq->pdata); 481 writel(gce_addr, thread->base + CMDQ_THR_END_ADDR); 482 483 writel(thread->priority, thread->base + CMDQ_THR_PRIORITY); 484 writel(CMDQ_THR_IRQ_EN, thread->base + CMDQ_THR_IRQ_ENABLE); 485 writel(CMDQ_THR_ENABLED, thread->base + CMDQ_THR_ENABLE_TASK); 486 } else { 487 WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0); 488 gce_addr = readl(thread->base + CMDQ_THR_CURR_ADDR); 489 curr_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); 490 gce_addr = readl(thread->base + CMDQ_THR_END_ADDR); 491 end_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); 492 /* check boundary */ 493 if (curr_pa == end_pa - CMDQ_INST_SIZE || 494 curr_pa == end_pa) { 495 /* set to this task directly */ 496 gce_addr = cmdq_convert_gce_addr(task->pa_base, cmdq->pdata); 497 writel(gce_addr, thread->base + CMDQ_THR_CURR_ADDR); 498 } else { 499 cmdq_task_insert_into_thread(task); 500 smp_mb(); /* modify jump before enable thread */ 501 } 502 gce_addr = cmdq_convert_gce_addr(task->pa_base + pkt->cmd_buf_size, cmdq->pdata); 503 writel(gce_addr, thread->base + CMDQ_THR_END_ADDR); 504 cmdq_thread_resume(thread); 505 } 506 list_move_tail(&task->list_entry, &thread->task_busy_list); 507 508 return 0; 509 } 510 511 static int cmdq_mbox_startup(struct mbox_chan *chan) 512 { 513 return 0; 514 } 515 516 static void cmdq_mbox_shutdown(struct mbox_chan *chan) 517 { 518 struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv; 519 struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev); 520 struct cmdq_task *task, *tmp; 521 unsigned long flags; 522 523 WARN_ON(pm_runtime_get_sync(cmdq->mbox.dev) < 0); 524 525 spin_lock_irqsave(&thread->chan->lock, flags); 526 if (list_empty(&thread->task_busy_list)) 527 goto done; 528 529 WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0); 530 531 /* make sure executed tasks have success callback */ 532 cmdq_thread_irq_handler(cmdq, thread); 533 if (list_empty(&thread->task_busy_list)) 534 goto done; 535 536 list_for_each_entry_safe(task, tmp, &thread->task_busy_list, 537 list_entry) { 538 cmdq_task_exec_done(task, -ECONNABORTED); 539 kfree(task); 540 } 541 542 cmdq_thread_disable(cmdq, thread); 543 544 done: 545 /* 546 * The thread->task_busy_list empty means thread already disable. The 547 * cmdq_mbox_send_data() always reset thread which clear disable and 548 * suspend statue when first pkt send to channel, so there is no need 549 * to do any operation here, only unlock and leave. 550 */ 551 spin_unlock_irqrestore(&thread->chan->lock, flags); 552 553 pm_runtime_mark_last_busy(cmdq->mbox.dev); 554 pm_runtime_put_autosuspend(cmdq->mbox.dev); 555 } 556 557 static int cmdq_mbox_flush(struct mbox_chan *chan, unsigned long timeout) 558 { 559 struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv; 560 struct cmdq_cb_data data; 561 struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev); 562 struct cmdq_task *task, *tmp; 563 unsigned long flags; 564 u32 enable; 565 int ret; 566 567 ret = pm_runtime_get_sync(cmdq->mbox.dev); 568 if (ret < 0) { 569 pm_runtime_put_noidle(cmdq->mbox.dev); 570 return ret; 571 } 572 573 spin_lock_irqsave(&thread->chan->lock, flags); 574 if (list_empty(&thread->task_busy_list)) 575 goto out; 576 577 WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0); 578 if (!cmdq_thread_is_in_wfe(thread)) 579 goto wait; 580 581 list_for_each_entry_safe(task, tmp, &thread->task_busy_list, 582 list_entry) { 583 data.sta = -ECONNABORTED; 584 data.pkt = task->pkt; 585 mbox_chan_received_data(task->thread->chan, &data); 586 list_del(&task->list_entry); 587 kfree(task); 588 } 589 590 cmdq_thread_resume(thread); 591 cmdq_thread_disable(cmdq, thread); 592 593 out: 594 spin_unlock_irqrestore(&thread->chan->lock, flags); 595 pm_runtime_mark_last_busy(cmdq->mbox.dev); 596 pm_runtime_put_autosuspend(cmdq->mbox.dev); 597 598 return 0; 599 600 wait: 601 cmdq_thread_resume(thread); 602 spin_unlock_irqrestore(&thread->chan->lock, flags); 603 if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_ENABLE_TASK, 604 enable, enable == 0, 1, timeout)) { 605 dev_err(cmdq->mbox.dev, "Fail to wait GCE thread 0x%x done\n", 606 (u32)(thread->base - cmdq->base)); 607 608 pm_runtime_mark_last_busy(cmdq->mbox.dev); 609 pm_runtime_put_autosuspend(cmdq->mbox.dev); 610 611 return -EFAULT; 612 } 613 pm_runtime_mark_last_busy(cmdq->mbox.dev); 614 pm_runtime_put_autosuspend(cmdq->mbox.dev); 615 return 0; 616 } 617 618 static const struct mbox_chan_ops cmdq_mbox_chan_ops = { 619 .send_data = cmdq_mbox_send_data, 620 .startup = cmdq_mbox_startup, 621 .shutdown = cmdq_mbox_shutdown, 622 .flush = cmdq_mbox_flush, 623 }; 624 625 static struct mbox_chan *cmdq_xlate(struct mbox_controller *mbox, 626 const struct of_phandle_args *sp) 627 { 628 int ind = sp->args[0]; 629 struct cmdq_thread *thread; 630 631 if (ind >= mbox->num_chans) 632 return ERR_PTR(-EINVAL); 633 634 thread = (struct cmdq_thread *)mbox->chans[ind].con_priv; 635 thread->priority = sp->args[1]; 636 thread->chan = &mbox->chans[ind]; 637 638 return &mbox->chans[ind]; 639 } 640 641 static int cmdq_get_clocks(struct device *dev, struct cmdq *cmdq) 642 { 643 static const char * const gce_name = "gce"; 644 struct device_node *parent = dev->of_node->parent; 645 struct clk_bulk_data *clks; 646 647 cmdq->clocks = devm_kcalloc(dev, cmdq->pdata->gce_num, 648 sizeof(*cmdq->clocks), GFP_KERNEL); 649 if (!cmdq->clocks) 650 return -ENOMEM; 651 652 if (cmdq->pdata->gce_num == 1) { 653 clks = &cmdq->clocks[0]; 654 655 clks->id = gce_name; 656 clks->clk = devm_clk_get(dev, NULL); 657 if (IS_ERR(clks->clk)) 658 return dev_err_probe(dev, PTR_ERR(clks->clk), 659 "failed to get gce clock\n"); 660 661 return 0; 662 } 663 664 /* 665 * If there is more than one GCE, get the clocks for the others too, 666 * as the clock of the main GCE must be enabled for additional IPs 667 * to be reachable. 668 */ 669 for_each_child_of_node_scoped(parent, node) { 670 int alias_id = of_alias_get_id(node, gce_name); 671 672 if (alias_id < 0 || alias_id >= cmdq->pdata->gce_num) 673 continue; 674 675 clks = &cmdq->clocks[alias_id]; 676 677 clks->id = devm_kasprintf(dev, GFP_KERNEL, "gce%d", alias_id); 678 if (!clks->id) 679 return -ENOMEM; 680 681 clks->clk = of_clk_get(node, 0); 682 if (IS_ERR(clks->clk)) 683 return dev_err_probe(dev, PTR_ERR(clks->clk), 684 "failed to get gce%d clock\n", alias_id); 685 } 686 687 return 0; 688 } 689 690 static int cmdq_probe(struct platform_device *pdev) 691 { 692 struct device *dev = &pdev->dev; 693 struct cmdq *cmdq; 694 int err, i; 695 696 cmdq = devm_kzalloc(dev, sizeof(*cmdq), GFP_KERNEL); 697 if (!cmdq) 698 return -ENOMEM; 699 700 cmdq->base = devm_platform_ioremap_resource(pdev, 0); 701 if (IS_ERR(cmdq->base)) 702 return PTR_ERR(cmdq->base); 703 704 cmdq->irq = platform_get_irq(pdev, 0); 705 if (cmdq->irq < 0) 706 return cmdq->irq; 707 708 cmdq->pdata = device_get_match_data(dev); 709 if (!cmdq->pdata) { 710 dev_err(dev, "failed to get match data\n"); 711 return -EINVAL; 712 } 713 714 cmdq->irq_mask = GENMASK(cmdq->pdata->thread_nr - 1, 0); 715 716 dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n", 717 dev, cmdq->base, cmdq->irq); 718 719 err = cmdq_get_clocks(dev, cmdq); 720 if (err) 721 return err; 722 723 dma_set_coherent_mask(dev, 724 DMA_BIT_MASK(sizeof(u32) * BITS_PER_BYTE + cmdq->pdata->shift)); 725 726 cmdq->mbox.dev = dev; 727 cmdq->mbox.chans = devm_kcalloc(dev, cmdq->pdata->thread_nr, 728 sizeof(*cmdq->mbox.chans), GFP_KERNEL); 729 if (!cmdq->mbox.chans) 730 return -ENOMEM; 731 732 cmdq->mbox.num_chans = cmdq->pdata->thread_nr; 733 cmdq->mbox.ops = &cmdq_mbox_chan_ops; 734 cmdq->mbox.of_xlate = cmdq_xlate; 735 736 /* make use of MBOX_TXDONE_BY_ACK */ 737 cmdq->mbox.txdone_irq = false; 738 cmdq->mbox.txdone_poll = false; 739 740 cmdq->thread = devm_kcalloc(dev, cmdq->pdata->thread_nr, 741 sizeof(*cmdq->thread), GFP_KERNEL); 742 if (!cmdq->thread) 743 return -ENOMEM; 744 745 for (i = 0; i < cmdq->pdata->thread_nr; i++) { 746 cmdq->thread[i].base = cmdq->base + CMDQ_THR_BASE + 747 CMDQ_THR_SIZE * i; 748 INIT_LIST_HEAD(&cmdq->thread[i].task_busy_list); 749 cmdq->mbox.chans[i].con_priv = (void *)&cmdq->thread[i]; 750 } 751 752 platform_set_drvdata(pdev, cmdq); 753 754 WARN_ON(clk_bulk_prepare(cmdq->pdata->gce_num, cmdq->clocks)); 755 756 cmdq_init(cmdq); 757 758 err = devm_request_irq(dev, cmdq->irq, cmdq_irq_handler, IRQF_SHARED, 759 "mtk_cmdq", cmdq); 760 if (err < 0) { 761 dev_err(dev, "failed to register ISR (%d)\n", err); 762 return err; 763 } 764 765 /* If Runtime PM is not available enable the clocks now. */ 766 if (!IS_ENABLED(CONFIG_PM)) { 767 err = cmdq_runtime_resume(dev); 768 if (err) 769 return err; 770 } 771 772 err = devm_pm_runtime_enable(dev); 773 if (err) 774 return err; 775 776 pm_runtime_set_autosuspend_delay(dev, CMDQ_MBOX_AUTOSUSPEND_DELAY_MS); 777 pm_runtime_use_autosuspend(dev); 778 779 err = devm_mbox_controller_register(dev, &cmdq->mbox); 780 if (err < 0) { 781 dev_err(dev, "failed to register mailbox: %d\n", err); 782 return err; 783 } 784 785 return 0; 786 } 787 788 static const struct dev_pm_ops cmdq_pm_ops = { 789 .suspend = cmdq_suspend, 790 .resume = cmdq_resume, 791 SET_RUNTIME_PM_OPS(cmdq_runtime_suspend, 792 cmdq_runtime_resume, NULL) 793 }; 794 795 static const struct gce_plat gce_plat_mt6779 = { 796 .thread_nr = 24, 797 .shift = 3, 798 .control_by_sw = false, 799 .gce_num = 1 800 }; 801 802 static const struct gce_plat gce_plat_mt8173 = { 803 .thread_nr = 16, 804 .shift = 0, 805 .control_by_sw = false, 806 .gce_num = 1 807 }; 808 809 static const struct gce_plat gce_plat_mt8183 = { 810 .thread_nr = 24, 811 .shift = 0, 812 .control_by_sw = false, 813 .gce_num = 1 814 }; 815 816 static const struct gce_plat gce_plat_mt8186 = { 817 .thread_nr = 24, 818 .shift = 3, 819 .control_by_sw = true, 820 .sw_ddr_en = true, 821 .gce_num = 1 822 }; 823 824 static const struct gce_plat gce_plat_mt8188 = { 825 .thread_nr = 32, 826 .shift = 3, 827 .control_by_sw = true, 828 .gce_num = 2 829 }; 830 831 static const struct gce_plat gce_plat_mt8192 = { 832 .thread_nr = 24, 833 .shift = 3, 834 .control_by_sw = true, 835 .gce_num = 1 836 }; 837 838 static const struct gce_plat gce_plat_mt8195 = { 839 .thread_nr = 24, 840 .shift = 3, 841 .control_by_sw = true, 842 .gce_num = 2 843 }; 844 845 static const struct gce_plat gce_plat_mt8196 = { 846 .thread_nr = 32, 847 .shift = 3, 848 .mminfra_offset = SZ_2G, 849 .control_by_sw = true, 850 .sw_ddr_en = true, 851 .gce_vm = true, 852 .gce_num = 2 853 }; 854 855 static const struct of_device_id cmdq_of_ids[] = { 856 {.compatible = "mediatek,mt6779-gce", .data = (void *)&gce_plat_mt6779}, 857 {.compatible = "mediatek,mt8173-gce", .data = (void *)&gce_plat_mt8173}, 858 {.compatible = "mediatek,mt8183-gce", .data = (void *)&gce_plat_mt8183}, 859 {.compatible = "mediatek,mt8186-gce", .data = (void *)&gce_plat_mt8186}, 860 {.compatible = "mediatek,mt8188-gce", .data = (void *)&gce_plat_mt8188}, 861 {.compatible = "mediatek,mt8192-gce", .data = (void *)&gce_plat_mt8192}, 862 {.compatible = "mediatek,mt8195-gce", .data = (void *)&gce_plat_mt8195}, 863 {.compatible = "mediatek,mt8196-gce", .data = (void *)&gce_plat_mt8196}, 864 {} 865 }; 866 MODULE_DEVICE_TABLE(of, cmdq_of_ids); 867 868 static struct platform_driver cmdq_drv = { 869 .probe = cmdq_probe, 870 .remove = cmdq_remove, 871 .driver = { 872 .name = "mtk_cmdq", 873 .pm = &cmdq_pm_ops, 874 .of_match_table = cmdq_of_ids, 875 } 876 }; 877 878 static int __init cmdq_drv_init(void) 879 { 880 return platform_driver_register(&cmdq_drv); 881 } 882 883 static void __exit cmdq_drv_exit(void) 884 { 885 platform_driver_unregister(&cmdq_drv); 886 } 887 888 subsys_initcall(cmdq_drv_init); 889 module_exit(cmdq_drv_exit); 890 891 MODULE_DESCRIPTION("Mediatek Command Queue(CMDQ) Mailbox driver"); 892 MODULE_LICENSE("GPL v2"); 893