1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD AE4DMA driver 4 * 5 * Copyright (c) 2024, Advanced Micro Devices, Inc. 6 * All Rights Reserved. 7 * 8 * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> 9 */ 10 11 #include "ae4dma.h" 12 13 static unsigned int max_hw_q = 1; 14 module_param(max_hw_q, uint, 0444); 15 MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)"); 16 17 static void ae4_pending_work(struct work_struct *work) 18 { 19 struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work); 20 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; 21 struct pt_cmd *cmd; 22 u32 cridx; 23 24 for (;;) { 25 wait_event_interruptible(ae4cmd_q->q_w, 26 ((atomic64_read(&ae4cmd_q->done_cnt)) < 27 atomic64_read(&ae4cmd_q->intr_cnt))); 28 29 atomic64_inc(&ae4cmd_q->done_cnt); 30 31 mutex_lock(&ae4cmd_q->cmd_lock); 32 cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); 33 while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) { 34 cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry); 35 list_del(&cmd->entry); 36 37 ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx); 38 cmd->pt_cmd_callback(cmd->data, cmd->ret); 39 40 ae4cmd_q->q_cmd_count--; 41 ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN; 42 43 complete_all(&ae4cmd_q->cmp); 44 } 45 mutex_unlock(&ae4cmd_q->cmd_lock); 46 } 47 } 48 49 static irqreturn_t ae4_core_irq_handler(int irq, void *data) 50 { 51 struct ae4_cmd_queue *ae4cmd_q = data; 52 struct pt_cmd_queue *cmd_q; 53 struct pt_device *pt; 54 u32 status; 55 56 cmd_q = &ae4cmd_q->cmd_q; 57 pt = cmd_q->pt; 58 59 pt->total_interrupts++; 60 atomic64_inc(&ae4cmd_q->intr_cnt); 61 62 status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF); 63 if (status & BIT(0)) { 64 status &= GENMASK(31, 1); 65 writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF); 66 } 67 68 wake_up(&ae4cmd_q->q_w); 69 70 return IRQ_HANDLED; 71 } 72 73 void ae4_destroy_work(struct ae4_device *ae4) 74 { 75 struct ae4_cmd_queue *ae4cmd_q; 76 int i; 77 78 for (i = 0; i < ae4->cmd_q_count; i++) { 79 ae4cmd_q = &ae4->ae4cmd_q[i]; 80 81 if (!ae4cmd_q->pws) 82 break; 83 84 cancel_delayed_work_sync(&ae4cmd_q->p_work); 85 destroy_workqueue(ae4cmd_q->pws); 86 } 87 } 88 89 int ae4_core_init(struct ae4_device *ae4) 90 { 91 struct pt_device *pt = &ae4->pt; 92 struct ae4_cmd_queue *ae4cmd_q; 93 struct device *dev = pt->dev; 94 struct pt_cmd_queue *cmd_q; 95 int i, ret = 0; 96 97 writel(max_hw_q, pt->io_regs); 98 99 for (i = 0; i < max_hw_q; i++) { 100 ae4cmd_q = &ae4->ae4cmd_q[i]; 101 ae4cmd_q->id = ae4->cmd_q_count; 102 ae4->cmd_q_count++; 103 104 cmd_q = &ae4cmd_q->cmd_q; 105 cmd_q->pt = pt; 106 107 cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); 108 109 ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0, 110 dev_name(pt->dev), ae4cmd_q); 111 if (ret) 112 return ret; 113 114 cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc)); 115 116 cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma, 117 GFP_KERNEL); 118 if (!cmd_q->qbase) 119 return -ENOMEM; 120 } 121 122 for (i = 0; i < ae4->cmd_q_count; i++) { 123 ae4cmd_q = &ae4->ae4cmd_q[i]; 124 125 cmd_q = &ae4cmd_q->cmd_q; 126 127 cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); 128 129 /* Update the device registers with queue information. */ 130 writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF); 131 132 cmd_q->qdma_tail = cmd_q->qbase_dma; 133 writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF); 134 writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF); 135 136 INIT_LIST_HEAD(&ae4cmd_q->cmd); 137 init_waitqueue_head(&ae4cmd_q->q_w); 138 139 ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id); 140 if (!ae4cmd_q->pws) { 141 ae4_destroy_work(ae4); 142 return -ENOMEM; 143 } 144 INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work); 145 queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100)); 146 147 init_completion(&ae4cmd_q->cmp); 148 } 149 150 ret = pt_dmaengine_register(pt); 151 if (ret) 152 ae4_destroy_work(ae4); 153 else 154 ptdma_debugfs_setup(pt); 155 156 return ret; 157 } 158