// SPDX-License-Identifier: GPL-2.0-or-later /* * DMA driver for AMD Queue-based DMA Subsystem * * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. */ #include #include #include #include #include #include #include #include #include #include "qdma.h" #define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") #define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) /* MMIO regmap config for all QDMA registers */ static const struct regmap_config qdma_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, }; static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) { return container_of(chan, struct qdma_queue, vchan.chan); } static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) { return container_of(vdesc, struct qdma_mm_vdesc, vdesc); } static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) { u32 idx; idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; qdev->qintr_ring_idx %= qdev->qintr_ring_num; return idx; } static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, enum qdma_reg_fields field) { const struct qdma_reg_field *f = &qdev->rfields[field]; u16 low_pos, hi_pos, low_bit, hi_bit; u64 value = 0, mask; low_pos = f->lsb / BITS_PER_TYPE(*data); hi_pos = f->msb / BITS_PER_TYPE(*data); if (low_pos == hi_pos) { low_bit = f->lsb % BITS_PER_TYPE(*data); hi_bit = f->msb % BITS_PER_TYPE(*data); mask = GENMASK(hi_bit, low_bit); value = (data[low_pos] & mask) >> low_bit; } else if (hi_pos == low_pos + 1) { low_bit = f->lsb % BITS_PER_TYPE(*data); hi_bit = low_bit + (f->msb - f->lsb); value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | data[low_pos]; mask = GENMASK_ULL(hi_bit, low_bit); value = (value & mask) >> low_bit; } else { hi_bit = f->msb % BITS_PER_TYPE(*data); mask = GENMASK(hi_bit, 0); value = data[hi_pos] & mask; low_bit = f->msb - f->lsb - hi_bit; value <<= low_bit; low_bit -= 32; value |= (u64)data[hi_pos - 1] << low_bit; mask = GENMASK(31, 32 - low_bit); value |= (data[hi_pos - 2] & mask) >> low_bit; } return value; } static void qdma_set_field(const struct qdma_device *qdev, u32 *data, enum qdma_reg_fields field, u64 value) { const struct qdma_reg_field *f = &qdev->rfields[field]; u16 low_pos, hi_pos, low_bit; low_pos = f->lsb / BITS_PER_TYPE(*data); hi_pos = f->msb / BITS_PER_TYPE(*data); low_bit = f->lsb % BITS_PER_TYPE(*data); data[low_pos++] |= value << low_bit; if (low_pos <= hi_pos) data[low_pos++] |= (u32)(value >> (32 - low_bit)); if (low_pos <= hi_pos) data[low_pos] |= (u32)(value >> (64 - low_bit)); } static inline int qdma_reg_write(const struct qdma_device *qdev, const u32 *data, enum qdma_regs reg) { const struct qdma_reg *r = &qdev->roffs[reg]; int ret; if (r->count > 1) ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); else ret = regmap_write(qdev->regmap, r->off, *data); return ret; } static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, enum qdma_regs reg) { const struct qdma_reg *r = &qdev->roffs[reg]; int ret; if (r->count > 1) ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); else ret = regmap_read(qdev->regmap, r->off, data); return ret; } static int qdma_context_cmd_execute(const struct qdma_device *qdev, enum qdma_ctxt_type type, enum qdma_ctxt_cmd cmd, u16 index) { u32 value = 0; int ret; qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); if (ret) return ret; ret = regmap_read_poll_timeout(qdev->regmap, QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), value, !qdma_get_field(qdev, &value, QDMA_REGF_CMD_BUSY), QDMA_POLL_INTRVL_US, QDMA_POLL_TIMEOUT_US); if (ret) { qdma_err(qdev, "Context command execution timed out"); return ret; } return 0; } static int qdma_context_write_data(const struct qdma_device *qdev, const u32 *data) { u32 mask[QDMA_CTXT_REGMAP_LEN]; int ret; memset(mask, ~0, sizeof(mask)); ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); if (ret) return ret; ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); if (ret) return ret; return 0; } static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, const struct qdma_ctxt_sw_desc *ctxt, u32 *data) { memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); } static void qdma_prep_intr_context(const struct qdma_device *qdev, const struct qdma_ctxt_intr *ctxt, u32 *data) { memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); } static void qdma_prep_fmap_context(const struct qdma_device *qdev, const struct qdma_ctxt_fmap *ctxt, u32 *data) { memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); } /* * Program the indirect context register space * * Once the queue is enabled, context is dynamically updated by hardware. Any * modification of the context through this API when the queue is enabled can * result in unexpected behavior. Reading the context when the queue is enabled * is not recommended as it can result in reduced performance. */ static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) { int ret; mutex_lock(&qdev->ctxt_lock); if (cmd == QDMA_CTXT_WRITE) { ret = qdma_context_write_data(qdev, ctxt); if (ret) goto failed; } ret = qdma_context_cmd_execute(qdev, type, cmd, index); if (ret) goto failed; if (cmd == QDMA_CTXT_READ) { ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); if (ret) goto failed; } failed: mutex_unlock(&qdev->ctxt_lock); return ret; } static int qdma_check_queue_status(struct qdma_device *qdev, enum dma_transfer_direction dir, u16 qid) { u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; enum qdma_ctxt_type type; int ret; if (dir == DMA_MEM_TO_DEV) type = QDMA_CTXT_DESC_SW_H2C; else type = QDMA_CTXT_DESC_SW_C2H; ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); if (ret) return ret; status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); if (status) { qdma_err(qdev, "queue %d already in use", qid); return -EBUSY; } return 0; } static int qdma_clear_queue_context(const struct qdma_queue *queue) { enum qdma_ctxt_type h2c_types[] = { QDMA_CTXT_DESC_SW_H2C, QDMA_CTXT_DESC_HW_H2C, QDMA_CTXT_DESC_CR_H2C, QDMA_CTXT_PFTCH, }; enum qdma_ctxt_type c2h_types[] = { QDMA_CTXT_DESC_SW_C2H, QDMA_CTXT_DESC_HW_C2H, QDMA_CTXT_DESC_CR_C2H, QDMA_CTXT_PFTCH, }; struct qdma_device *qdev = queue->qdev; enum qdma_ctxt_type *type; int ret, num, i; if (queue->dir == DMA_MEM_TO_DEV) { type = h2c_types; num = ARRAY_SIZE(h2c_types); } else { type = c2h_types; num = ARRAY_SIZE(c2h_types); } for (i = 0; i < num; i++) { ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, queue->qid, NULL); if (ret) { qdma_err(qdev, "Failed to clear ctxt %d", type[i]); return ret; } } return 0; } static int qdma_setup_fmap_context(struct qdma_device *qdev) { u32 ctxt[QDMA_CTXT_REGMAP_LEN]; struct qdma_ctxt_fmap fmap; int ret; ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, qdev->fid, NULL); if (ret) { qdma_err(qdev, "Failed clearing context"); return ret; } fmap.qbase = 0; fmap.qmax = qdev->chan_num * 2; qdma_prep_fmap_context(qdev, &fmap, ctxt); ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, qdev->fid, ctxt); if (ret) qdma_err(qdev, "Failed setup fmap, ret %d", ret); return ret; } static int qdma_setup_queue_context(struct qdma_device *qdev, const struct qdma_ctxt_sw_desc *sw_desc, enum dma_transfer_direction dir, u16 qid) { u32 ctxt[QDMA_CTXT_REGMAP_LEN]; enum qdma_ctxt_type type; int ret; if (dir == DMA_MEM_TO_DEV) type = QDMA_CTXT_DESC_SW_H2C; else type = QDMA_CTXT_DESC_SW_C2H; qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); /* Setup SW descriptor context */ ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); if (ret) qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); return ret; } /* * Enable or disable memory-mapped DMA engines * 1: enable, 0: disable */ static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) { int ret; ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); return ret; } static int qdma_get_hw_info(struct qdma_device *qdev) { struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); u32 value = 0; int ret; ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); if (ret) return ret; value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; if (pdata->max_mm_channels * 2 > value) { qdma_err(qdev, "not enough hw queues %d", value); return -EINVAL; } qdev->chan_num = pdata->max_mm_channels; ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); if (ret) return ret; qdma_info(qdev, "max channel %d, function id %d", qdev->chan_num, qdev->fid); return 0; } static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) { struct qdma_device *qdev = queue->qdev; return regmap_write(qdev->regmap, queue->pidx_reg, pidx | QDMA_QUEUE_ARM_BIT); } static inline int qdma_update_cidx(const struct qdma_queue *queue, u16 ridx, u16 cidx) { struct qdma_device *qdev = queue->qdev; return regmap_write(qdev->regmap, queue->cidx_reg, ((u32)ridx << 16) | cidx); } /** * qdma_free_vdesc - Free descriptor * @vdesc: Virtual DMA descriptor */ static void qdma_free_vdesc(struct virt_dma_desc *vdesc) { struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); kfree(vd); } static int qdma_alloc_queues(struct qdma_device *qdev, enum dma_transfer_direction dir) { struct qdma_queue *q, **queues; u32 i, pidx_base; int ret; if (dir == DMA_MEM_TO_DEV) { queues = &qdev->h2c_queues; pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); } else { queues = &qdev->c2h_queues; pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); } *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), GFP_KERNEL); if (!*queues) return -ENOMEM; for (i = 0; i < qdev->chan_num; i++) { ret = qdma_check_queue_status(qdev, dir, i); if (ret) return ret; q = &(*queues)[i]; q->ring_size = QDMA_DEFAULT_RING_SIZE; q->idx_mask = q->ring_size - 2; q->qdev = qdev; q->dir = dir; q->qid = i; q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + i * QDMA_DMAP_REG_STRIDE; q->vchan.desc_free = qdma_free_vdesc; vchan_init(&q->vchan, &qdev->dma_dev); } return 0; } static int qdma_device_verify(struct qdma_device *qdev) { u32 value; int ret; ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); if (ret) return ret; value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); if (value != QDMA_IDENTIFIER) { qdma_err(qdev, "Invalid identifier"); return -ENODEV; } qdev->rfields = qdma_regfs_default; qdev->roffs = qdma_regos_default; return 0; } static int qdma_device_setup(struct qdma_device *qdev) { struct device *dev = &qdev->pdev->dev; u32 ring_sz = QDMA_DEFAULT_RING_SIZE; int ret = 0; while (dev && get_dma_ops(dev)) dev = dev->parent; if (!dev) { qdma_err(qdev, "dma device not found"); return -EINVAL; } set_dma_ops(&qdev->pdev->dev, get_dma_ops(dev)); ret = qdma_setup_fmap_context(qdev); if (ret) { qdma_err(qdev, "Failed setup fmap context"); return ret; } /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); if (ret) { qdma_err(qdev, "Failed to setup ring %d of size %ld", QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); return ret; } /* Enable memory-mapped DMA engine in both directions */ ret = qdma_sgdma_control(qdev, 1); if (ret) { qdma_err(qdev, "Failed to SGDMA with error %d", ret); return ret; } ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); if (ret) { qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); return ret; } ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); if (ret) { qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); return ret; } return 0; } /** * qdma_free_queue_resources() - Free queue resources * @chan: DMA channel */ static void qdma_free_queue_resources(struct dma_chan *chan) { struct qdma_queue *queue = to_qdma_queue(chan); struct qdma_device *qdev = queue->qdev; struct device *dev = qdev->dma_dev.dev; qdma_clear_queue_context(queue); vchan_free_chan_resources(&queue->vchan); dma_free_coherent(dev, queue->ring_size * QDMA_MM_DESC_SIZE, queue->desc_base, queue->dma_desc_base); } /** * qdma_alloc_queue_resources() - Allocate queue resources * @chan: DMA channel */ static int qdma_alloc_queue_resources(struct dma_chan *chan) { struct qdma_queue *queue = to_qdma_queue(chan); struct qdma_device *qdev = queue->qdev; struct qdma_ctxt_sw_desc desc; size_t size; int ret; ret = qdma_clear_queue_context(queue); if (ret) return ret; size = queue->ring_size * QDMA_MM_DESC_SIZE; queue->desc_base = dma_alloc_coherent(qdev->dma_dev.dev, size, &queue->dma_desc_base, GFP_KERNEL); if (!queue->desc_base) { qdma_err(qdev, "Failed to allocate descriptor ring"); return -ENOMEM; } /* Setup SW descriptor queue context for DMA memory map */ desc.vec = qdma_get_intr_ring_idx(qdev); desc.desc_base = queue->dma_desc_base; ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); if (ret) { qdma_err(qdev, "Failed to setup SW desc ctxt for %s", chan->name); dma_free_coherent(qdev->dma_dev.dev, size, queue->desc_base, queue->dma_desc_base); return ret; } queue->pidx = 0; queue->cidx = 0; return 0; } static bool qdma_filter_fn(struct dma_chan *chan, void *param) { struct qdma_queue *queue = to_qdma_queue(chan); struct qdma_queue_info *info = param; return info->dir == queue->dir; } static int qdma_xfer_start(struct qdma_queue *queue) { struct qdma_device *qdev = queue->qdev; int ret; if (!vchan_next_desc(&queue->vchan)) return 0; qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); if (ret) { qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", queue->pidx, CHAN_STR(queue), queue->qid); } return ret; } static void qdma_issue_pending(struct dma_chan *chan) { struct qdma_queue *queue = to_qdma_queue(chan); unsigned long flags; spin_lock_irqsave(&queue->vchan.lock, flags); if (vchan_issue_pending(&queue->vchan)) { if (queue->submitted_vdesc) { queue->issued_vdesc = queue->submitted_vdesc; queue->submitted_vdesc = NULL; } qdma_xfer_start(queue); } spin_unlock_irqrestore(&queue->vchan.lock, flags); } static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) { struct qdma_mm_desc *desc; if (((q->pidx + 1) & q->idx_mask) == q->cidx) return NULL; desc = q->desc_base + q->pidx; q->pidx = (q->pidx + 1) & q->idx_mask; return desc; } static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) { struct qdma_mm_desc *desc; struct scatterlist *sg; u64 addr, *src, *dst; u32 rest, len; int ret = 0; u32 i; if (!vdesc->sg_len) return 0; if (q->dir == DMA_MEM_TO_DEV) { dst = &vdesc->dev_addr; src = &addr; } else { dst = &addr; src = &vdesc->dev_addr; } for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { addr = sg_dma_address(sg) + vdesc->sg_off; rest = sg_dma_len(sg) - vdesc->sg_off; while (rest) { len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); desc = qdma_get_desc(q); if (!desc) { ret = -EBUSY; goto out; } desc->src_addr = cpu_to_le64(*src); desc->dst_addr = cpu_to_le64(*dst); desc->len = cpu_to_le32(len); vdesc->dev_addr += len; vdesc->sg_off += len; vdesc->pending_descs++; addr += len; rest -= len; } vdesc->sg_off = 0; } out: vdesc->sg_len -= i; vdesc->pidx = q->pidx; return ret; } static void qdma_fill_pending_vdesc(struct qdma_queue *q) { struct virt_dma_chan *vc = &q->vchan; struct qdma_mm_vdesc *vdesc = NULL; struct virt_dma_desc *vd; int ret; if (!list_empty(&vc->desc_issued)) { vd = &q->issued_vdesc->vdesc; list_for_each_entry_from(vd, &vc->desc_issued, node) { vdesc = to_qdma_vdesc(vd); ret = qdma_hw_enqueue(q, vdesc); if (ret) { q->issued_vdesc = vdesc; return; } } q->issued_vdesc = vdesc; } if (list_empty(&vc->desc_submitted)) return; if (q->submitted_vdesc) vd = &q->submitted_vdesc->vdesc; else vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); list_for_each_entry_from(vd, &vc->desc_submitted, node) { vdesc = to_qdma_vdesc(vd); ret = qdma_hw_enqueue(q, vdesc); if (ret) break; } q->submitted_vdesc = vdesc; } static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) { struct virt_dma_chan *vc = to_virt_chan(tx->chan); struct qdma_queue *q = to_qdma_queue(&vc->chan); struct virt_dma_desc *vd; unsigned long flags; dma_cookie_t cookie; vd = container_of(tx, struct virt_dma_desc, tx); spin_lock_irqsave(&vc->lock, flags); cookie = dma_cookie_assign(tx); list_move_tail(&vd->node, &vc->desc_submitted); qdma_fill_pending_vdesc(q); spin_unlock_irqrestore(&vc->lock, flags); return cookie; } static struct dma_async_tx_descriptor * qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { struct qdma_queue *q = to_qdma_queue(chan); struct dma_async_tx_descriptor *tx; struct qdma_mm_vdesc *vdesc; vdesc = kzalloc(sizeof(*vdesc), GFP_NOWAIT); if (!vdesc) return NULL; vdesc->sgl = sgl; vdesc->sg_len = sg_len; if (dir == DMA_MEM_TO_DEV) vdesc->dev_addr = q->cfg.dst_addr; else vdesc->dev_addr = q->cfg.src_addr; tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); tx->tx_submit = qdma_tx_submit; return tx; } static int qdma_device_config(struct dma_chan *chan, struct dma_slave_config *cfg) { struct qdma_queue *q = to_qdma_queue(chan); memcpy(&q->cfg, cfg, sizeof(*cfg)); return 0; } static int qdma_arm_err_intr(const struct qdma_device *qdev) { u32 value = 0; qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); } static irqreturn_t qdma_error_isr(int irq, void *data) { struct qdma_device *qdev = data; u32 err_stat = 0; int ret; ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); if (ret) { qdma_err(qdev, "read error state failed, ret %d", ret); goto out; } qdma_err(qdev, "global error %d", err_stat); ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); if (ret) qdma_err(qdev, "clear error state failed, ret %d", ret); out: qdma_arm_err_intr(qdev); return IRQ_HANDLED; } static irqreturn_t qdma_queue_isr(int irq, void *data) { struct qdma_intr_ring *intr = data; struct qdma_queue *q = NULL; struct qdma_device *qdev; u32 index, comp_desc; u64 intr_ent; u8 color; int ret; u16 qid; qdev = intr->qdev; index = intr->cidx; while (1) { struct virt_dma_desc *vd; struct qdma_mm_vdesc *vdesc; unsigned long flags; u32 cidx; intr_ent = le64_to_cpu(intr->base[index]); color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); if (color != intr->color) break; qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) q = qdev->c2h_queues; else q = qdev->h2c_queues; q += qid; cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); spin_lock_irqsave(&q->vchan.lock, flags); comp_desc = (cidx - q->cidx) & q->idx_mask; vd = vchan_next_desc(&q->vchan); if (!vd) goto skip; vdesc = to_qdma_vdesc(vd); while (comp_desc > vdesc->pending_descs) { list_del(&vd->node); vchan_cookie_complete(vd); comp_desc -= vdesc->pending_descs; vd = vchan_next_desc(&q->vchan); vdesc = to_qdma_vdesc(vd); } vdesc->pending_descs -= comp_desc; if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { list_del(&vd->node); vchan_cookie_complete(vd); } q->cidx = cidx; qdma_fill_pending_vdesc(q); qdma_xfer_start(q); skip: spin_unlock_irqrestore(&q->vchan.lock, flags); /* * Wrap the index value and flip the expected color value if * interrupt aggregation PIDX has wrapped around. */ index++; index &= QDMA_INTR_RING_IDX_MASK; if (!index) intr->color = !intr->color; } /* * Update the software interrupt aggregation ring CIDX if a valid entry * was found. */ if (q) { qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); /* * Record the last read index of status descriptor from the * interrupt aggregation ring. */ intr->cidx = index; ret = qdma_update_cidx(q, intr->ridx, index); if (ret) { qdma_err(qdev, "Failed to update IRQ CIDX"); return IRQ_NONE; } } return IRQ_HANDLED; } static int qdma_init_error_irq(struct qdma_device *qdev) { struct device *dev = &qdev->pdev->dev; int ret; u32 vec; vec = qdev->queue_irq_start - 1; ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, IRQF_ONESHOT, "amd-qdma-error", qdev); if (ret) { qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); return ret; } ret = qdma_arm_err_intr(qdev); if (ret) qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); return ret; } static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) { u32 ctxt[QDMA_CTXT_REGMAP_LEN]; struct device *dev = &qdev->pdev->dev; struct qdma_intr_ring *ring; struct qdma_ctxt_intr intr_ctxt; u32 vector; int ret, i; qdev->qintr_ring_num = qdev->queue_irq_num; qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, sizeof(*qdev->qintr_rings), GFP_KERNEL); if (!qdev->qintr_rings) return -ENOMEM; vector = qdev->queue_irq_start; for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { ring = &qdev->qintr_rings[i]; ring->qdev = qdev; ring->msix_id = qdev->err_irq_idx + i + 1; ring->ridx = i; ring->color = 1; ring->base = dmam_alloc_coherent(dev, QDMA_INTR_RING_SIZE, &ring->dev_base, GFP_KERNEL); if (!ring->base) { qdma_err(qdev, "Failed to alloc intr ring %d", i); return -ENOMEM; } intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; intr_ctxt.vec = ring->msix_id; intr_ctxt.valid = true; intr_ctxt.color = true; ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, QDMA_CTXT_CLEAR, ring->ridx, NULL); if (ret) { qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); return ret; } qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, QDMA_CTXT_WRITE, ring->ridx, ctxt); if (ret) { qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); return ret; } ret = devm_request_threaded_irq(dev, vector, NULL, qdma_queue_isr, IRQF_ONESHOT, "amd-qdma-queue", ring); if (ret) { qdma_err(qdev, "Failed to request irq %d", vector); return ret; } } return 0; } static int qdma_intr_init(struct qdma_device *qdev) { int ret; ret = qdma_init_error_irq(qdev); if (ret) { qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); return ret; } ret = qdmam_alloc_qintr_rings(qdev); if (ret) { qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); return ret; } return 0; } static void amd_qdma_remove(struct platform_device *pdev) { struct qdma_device *qdev = platform_get_drvdata(pdev); qdma_sgdma_control(qdev, 0); dma_async_device_unregister(&qdev->dma_dev); mutex_destroy(&qdev->ctxt_lock); } static int amd_qdma_probe(struct platform_device *pdev) { struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); struct qdma_device *qdev; struct resource *res; void __iomem *regs; int ret; qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); if (!qdev) return -ENOMEM; platform_set_drvdata(pdev, qdev); qdev->pdev = pdev; mutex_init(&qdev->ctxt_lock); res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); if (!res) { qdma_err(qdev, "Failed to get IRQ resource"); ret = -ENODEV; goto failed; } qdev->err_irq_idx = pdata->irq_index; qdev->queue_irq_start = res->start + 1; qdev->queue_irq_num = resource_size(res) - 1; regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(regs)) { ret = PTR_ERR(regs); qdma_err(qdev, "Failed to map IO resource, err %d", ret); goto failed; } qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, &qdma_regmap_config); if (IS_ERR(qdev->regmap)) { ret = PTR_ERR(qdev->regmap); qdma_err(qdev, "Regmap init failed, err %d", ret); goto failed; } ret = qdma_device_verify(qdev); if (ret) goto failed; ret = qdma_get_hw_info(qdev); if (ret) goto failed; INIT_LIST_HEAD(&qdev->dma_dev.channels); ret = qdma_device_setup(qdev); if (ret) goto failed; ret = qdma_intr_init(qdev); if (ret) { qdma_err(qdev, "Failed to initialize IRQs %d", ret); goto failed_disable_engine; } dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); qdev->dma_dev.dev = &pdev->dev; qdev->dma_dev.filter.map = pdata->device_map; qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; qdev->dma_dev.filter.fn = qdma_filter_fn; qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; qdev->dma_dev.device_config = qdma_device_config; qdev->dma_dev.device_issue_pending = qdma_issue_pending; qdev->dma_dev.device_tx_status = dma_cookie_status; qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); ret = dma_async_device_register(&qdev->dma_dev); if (ret) { qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); goto failed_disable_engine; } return 0; failed_disable_engine: qdma_sgdma_control(qdev, 0); failed: mutex_destroy(&qdev->ctxt_lock); qdma_err(qdev, "Failed to probe AMD QDMA driver"); return ret; } static struct platform_driver amd_qdma_driver = { .driver = { .name = "amd-qdma", }, .probe = amd_qdma_probe, .remove_new = amd_qdma_remove, }; module_platform_driver(amd_qdma_driver); MODULE_DESCRIPTION("AMD QDMA driver"); MODULE_AUTHOR("XRT Team "); MODULE_LICENSE("GPL");