1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * DMA driver for AMD Queue-based DMA Subsystem
4 *
5 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
6 */
7 #include <linux/bitfield.h>
8 #include <linux/bitops.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/module.h>
12 #include <linux/platform_device.h>
13 #include <linux/platform_data/amd_qdma.h>
14 #include <linux/regmap.h>
15
16 #include "qdma.h"
17
18 #define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H")
19 #define QDMA_REG_OFF(d, r) ((d)->roffs[r].off)
20
21 /* MMIO regmap config for all QDMA registers */
22 static const struct regmap_config qdma_regmap_config = {
23 .reg_bits = 32,
24 .val_bits = 32,
25 .reg_stride = 4,
26 };
27
to_qdma_queue(struct dma_chan * chan)28 static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan)
29 {
30 return container_of(chan, struct qdma_queue, vchan.chan);
31 }
32
to_qdma_vdesc(struct virt_dma_desc * vdesc)33 static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc)
34 {
35 return container_of(vdesc, struct qdma_mm_vdesc, vdesc);
36 }
37
qdma_get_intr_ring_idx(struct qdma_device * qdev)38 static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev)
39 {
40 u32 idx;
41
42 idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx;
43 qdev->qintr_ring_idx %= qdev->qintr_ring_num;
44
45 return idx;
46 }
47
qdma_get_field(const struct qdma_device * qdev,const u32 * data,enum qdma_reg_fields field)48 static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data,
49 enum qdma_reg_fields field)
50 {
51 const struct qdma_reg_field *f = &qdev->rfields[field];
52 u16 low_pos, hi_pos, low_bit, hi_bit;
53 u64 value = 0, mask;
54
55 low_pos = f->lsb / BITS_PER_TYPE(*data);
56 hi_pos = f->msb / BITS_PER_TYPE(*data);
57
58 if (low_pos == hi_pos) {
59 low_bit = f->lsb % BITS_PER_TYPE(*data);
60 hi_bit = f->msb % BITS_PER_TYPE(*data);
61 mask = GENMASK(hi_bit, low_bit);
62 value = (data[low_pos] & mask) >> low_bit;
63 } else if (hi_pos == low_pos + 1) {
64 low_bit = f->lsb % BITS_PER_TYPE(*data);
65 hi_bit = low_bit + (f->msb - f->lsb);
66 value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) |
67 data[low_pos];
68 mask = GENMASK_ULL(hi_bit, low_bit);
69 value = (value & mask) >> low_bit;
70 } else {
71 hi_bit = f->msb % BITS_PER_TYPE(*data);
72 mask = GENMASK(hi_bit, 0);
73 value = data[hi_pos] & mask;
74 low_bit = f->msb - f->lsb - hi_bit;
75 value <<= low_bit;
76 low_bit -= 32;
77 value |= (u64)data[hi_pos - 1] << low_bit;
78 mask = GENMASK(31, 32 - low_bit);
79 value |= (data[hi_pos - 2] & mask) >> low_bit;
80 }
81
82 return value;
83 }
84
qdma_set_field(const struct qdma_device * qdev,u32 * data,enum qdma_reg_fields field,u64 value)85 static void qdma_set_field(const struct qdma_device *qdev, u32 *data,
86 enum qdma_reg_fields field, u64 value)
87 {
88 const struct qdma_reg_field *f = &qdev->rfields[field];
89 u16 low_pos, hi_pos, low_bit;
90
91 low_pos = f->lsb / BITS_PER_TYPE(*data);
92 hi_pos = f->msb / BITS_PER_TYPE(*data);
93 low_bit = f->lsb % BITS_PER_TYPE(*data);
94
95 data[low_pos++] |= value << low_bit;
96 if (low_pos <= hi_pos)
97 data[low_pos++] |= (u32)(value >> (32 - low_bit));
98 if (low_pos <= hi_pos)
99 data[low_pos] |= (u32)(value >> (64 - low_bit));
100 }
101
qdma_reg_write(const struct qdma_device * qdev,const u32 * data,enum qdma_regs reg)102 static inline int qdma_reg_write(const struct qdma_device *qdev,
103 const u32 *data, enum qdma_regs reg)
104 {
105 const struct qdma_reg *r = &qdev->roffs[reg];
106 int ret;
107
108 if (r->count > 1)
109 ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count);
110 else
111 ret = regmap_write(qdev->regmap, r->off, *data);
112
113 return ret;
114 }
115
qdma_reg_read(const struct qdma_device * qdev,u32 * data,enum qdma_regs reg)116 static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data,
117 enum qdma_regs reg)
118 {
119 const struct qdma_reg *r = &qdev->roffs[reg];
120 int ret;
121
122 if (r->count > 1)
123 ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count);
124 else
125 ret = regmap_read(qdev->regmap, r->off, data);
126
127 return ret;
128 }
129
qdma_context_cmd_execute(const struct qdma_device * qdev,enum qdma_ctxt_type type,enum qdma_ctxt_cmd cmd,u16 index)130 static int qdma_context_cmd_execute(const struct qdma_device *qdev,
131 enum qdma_ctxt_type type,
132 enum qdma_ctxt_cmd cmd, u16 index)
133 {
134 u32 value = 0;
135 int ret;
136
137 qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index);
138 qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd);
139 qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type);
140
141 ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD);
142 if (ret)
143 return ret;
144
145 ret = regmap_read_poll_timeout(qdev->regmap,
146 QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD),
147 value,
148 !qdma_get_field(qdev, &value,
149 QDMA_REGF_CMD_BUSY),
150 QDMA_POLL_INTRVL_US,
151 QDMA_POLL_TIMEOUT_US);
152 if (ret) {
153 qdma_err(qdev, "Context command execution timed out");
154 return ret;
155 }
156
157 return 0;
158 }
159
qdma_context_write_data(const struct qdma_device * qdev,const u32 * data)160 static int qdma_context_write_data(const struct qdma_device *qdev,
161 const u32 *data)
162 {
163 u32 mask[QDMA_CTXT_REGMAP_LEN];
164 int ret;
165
166 memset(mask, ~0, sizeof(mask));
167
168 ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK);
169 if (ret)
170 return ret;
171
172 ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA);
173 if (ret)
174 return ret;
175
176 return 0;
177 }
178
qdma_prep_sw_desc_context(const struct qdma_device * qdev,const struct qdma_ctxt_sw_desc * ctxt,u32 * data)179 static void qdma_prep_sw_desc_context(const struct qdma_device *qdev,
180 const struct qdma_ctxt_sw_desc *ctxt,
181 u32 *data)
182 {
183 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
184 qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base);
185 qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec);
186 qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid);
187
188 qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B);
189 qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID);
190 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM);
191 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1);
192 qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1);
193 qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1);
194 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1);
195 qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1);
196 qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1);
197 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1);
198 qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1);
199 }
200
qdma_prep_intr_context(const struct qdma_device * qdev,const struct qdma_ctxt_intr * ctxt,u32 * data)201 static void qdma_prep_intr_context(const struct qdma_device *qdev,
202 const struct qdma_ctxt_intr *ctxt,
203 u32 *data)
204 {
205 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
206 qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base);
207 qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec);
208 qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size);
209 qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid);
210 qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color);
211 qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid);
212 }
213
qdma_prep_fmap_context(const struct qdma_device * qdev,const struct qdma_ctxt_fmap * ctxt,u32 * data)214 static void qdma_prep_fmap_context(const struct qdma_device *qdev,
215 const struct qdma_ctxt_fmap *ctxt,
216 u32 *data)
217 {
218 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
219 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase);
220 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax);
221 }
222
223 /*
224 * Program the indirect context register space
225 *
226 * Once the queue is enabled, context is dynamically updated by hardware. Any
227 * modification of the context through this API when the queue is enabled can
228 * result in unexpected behavior. Reading the context when the queue is enabled
229 * is not recommended as it can result in reduced performance.
230 */
qdma_prog_context(struct qdma_device * qdev,enum qdma_ctxt_type type,enum qdma_ctxt_cmd cmd,u16 index,u32 * ctxt)231 static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type,
232 enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt)
233 {
234 int ret;
235
236 mutex_lock(&qdev->ctxt_lock);
237 if (cmd == QDMA_CTXT_WRITE) {
238 ret = qdma_context_write_data(qdev, ctxt);
239 if (ret)
240 goto failed;
241 }
242
243 ret = qdma_context_cmd_execute(qdev, type, cmd, index);
244 if (ret)
245 goto failed;
246
247 if (cmd == QDMA_CTXT_READ) {
248 ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA);
249 if (ret)
250 goto failed;
251 }
252
253 failed:
254 mutex_unlock(&qdev->ctxt_lock);
255
256 return ret;
257 }
258
qdma_check_queue_status(struct qdma_device * qdev,enum dma_transfer_direction dir,u16 qid)259 static int qdma_check_queue_status(struct qdma_device *qdev,
260 enum dma_transfer_direction dir, u16 qid)
261 {
262 u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0};
263 enum qdma_ctxt_type type;
264 int ret;
265
266 if (dir == DMA_MEM_TO_DEV)
267 type = QDMA_CTXT_DESC_SW_H2C;
268 else
269 type = QDMA_CTXT_DESC_SW_C2H;
270 ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data);
271 if (ret)
272 return ret;
273
274 status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE);
275 if (status) {
276 qdma_err(qdev, "queue %d already in use", qid);
277 return -EBUSY;
278 }
279
280 return 0;
281 }
282
qdma_clear_queue_context(const struct qdma_queue * queue)283 static int qdma_clear_queue_context(const struct qdma_queue *queue)
284 {
285 static const enum qdma_ctxt_type h2c_types[] = {
286 QDMA_CTXT_DESC_SW_H2C,
287 QDMA_CTXT_DESC_HW_H2C,
288 QDMA_CTXT_DESC_CR_H2C,
289 QDMA_CTXT_PFTCH,
290 };
291 static const enum qdma_ctxt_type c2h_types[] = {
292 QDMA_CTXT_DESC_SW_C2H,
293 QDMA_CTXT_DESC_HW_C2H,
294 QDMA_CTXT_DESC_CR_C2H,
295 QDMA_CTXT_PFTCH,
296 };
297 struct qdma_device *qdev = queue->qdev;
298 const enum qdma_ctxt_type *type;
299 int ret, num, i;
300
301 if (queue->dir == DMA_MEM_TO_DEV) {
302 type = h2c_types;
303 num = ARRAY_SIZE(h2c_types);
304 } else {
305 type = c2h_types;
306 num = ARRAY_SIZE(c2h_types);
307 }
308 for (i = 0; i < num; i++) {
309 ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR,
310 queue->qid, NULL);
311 if (ret) {
312 qdma_err(qdev, "Failed to clear ctxt %d", type[i]);
313 return ret;
314 }
315 }
316
317 return 0;
318 }
319
qdma_setup_fmap_context(struct qdma_device * qdev)320 static int qdma_setup_fmap_context(struct qdma_device *qdev)
321 {
322 u32 ctxt[QDMA_CTXT_REGMAP_LEN];
323 struct qdma_ctxt_fmap fmap;
324 int ret;
325
326 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR,
327 qdev->fid, NULL);
328 if (ret) {
329 qdma_err(qdev, "Failed clearing context");
330 return ret;
331 }
332
333 fmap.qbase = 0;
334 fmap.qmax = qdev->chan_num * 2;
335 qdma_prep_fmap_context(qdev, &fmap, ctxt);
336 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE,
337 qdev->fid, ctxt);
338 if (ret)
339 qdma_err(qdev, "Failed setup fmap, ret %d", ret);
340
341 return ret;
342 }
343
qdma_setup_queue_context(struct qdma_device * qdev,const struct qdma_ctxt_sw_desc * sw_desc,enum dma_transfer_direction dir,u16 qid)344 static int qdma_setup_queue_context(struct qdma_device *qdev,
345 const struct qdma_ctxt_sw_desc *sw_desc,
346 enum dma_transfer_direction dir, u16 qid)
347 {
348 u32 ctxt[QDMA_CTXT_REGMAP_LEN];
349 enum qdma_ctxt_type type;
350 int ret;
351
352 if (dir == DMA_MEM_TO_DEV)
353 type = QDMA_CTXT_DESC_SW_H2C;
354 else
355 type = QDMA_CTXT_DESC_SW_C2H;
356
357 qdma_prep_sw_desc_context(qdev, sw_desc, ctxt);
358 /* Setup SW descriptor context */
359 ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt);
360 if (ret)
361 qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid);
362
363 return ret;
364 }
365
366 /*
367 * Enable or disable memory-mapped DMA engines
368 * 1: enable, 0: disable
369 */
qdma_sgdma_control(struct qdma_device * qdev,u32 ctrl)370 static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl)
371 {
372 int ret;
373
374 ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL);
375 ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL);
376
377 return ret;
378 }
379
qdma_get_hw_info(struct qdma_device * qdev)380 static int qdma_get_hw_info(struct qdma_device *qdev)
381 {
382 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev);
383 u32 value = 0;
384 int ret;
385
386 ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT);
387 if (ret)
388 return ret;
389
390 value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1;
391 if (pdata->max_mm_channels * 2 > value) {
392 qdma_err(qdev, "not enough hw queues %d", value);
393 return -EINVAL;
394 }
395 qdev->chan_num = pdata->max_mm_channels;
396
397 ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID);
398 if (ret)
399 return ret;
400
401 qdma_info(qdev, "max channel %d, function id %d",
402 qdev->chan_num, qdev->fid);
403
404 return 0;
405 }
406
qdma_update_pidx(const struct qdma_queue * queue,u16 pidx)407 static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx)
408 {
409 struct qdma_device *qdev = queue->qdev;
410
411 return regmap_write(qdev->regmap, queue->pidx_reg,
412 pidx | QDMA_QUEUE_ARM_BIT);
413 }
414
qdma_update_cidx(const struct qdma_queue * queue,u16 ridx,u16 cidx)415 static inline int qdma_update_cidx(const struct qdma_queue *queue,
416 u16 ridx, u16 cidx)
417 {
418 struct qdma_device *qdev = queue->qdev;
419
420 return regmap_write(qdev->regmap, queue->cidx_reg,
421 ((u32)ridx << 16) | cidx);
422 }
423
424 /**
425 * qdma_free_vdesc - Free descriptor
426 * @vdesc: Virtual DMA descriptor
427 */
qdma_free_vdesc(struct virt_dma_desc * vdesc)428 static void qdma_free_vdesc(struct virt_dma_desc *vdesc)
429 {
430 struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc);
431
432 kfree(vd);
433 }
434
qdma_alloc_queues(struct qdma_device * qdev,enum dma_transfer_direction dir)435 static int qdma_alloc_queues(struct qdma_device *qdev,
436 enum dma_transfer_direction dir)
437 {
438 struct qdma_queue *q, **queues;
439 u32 i, pidx_base;
440 int ret;
441
442 if (dir == DMA_MEM_TO_DEV) {
443 queues = &qdev->h2c_queues;
444 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX);
445 } else {
446 queues = &qdev->c2h_queues;
447 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX);
448 }
449
450 *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q),
451 GFP_KERNEL);
452 if (!*queues)
453 return -ENOMEM;
454
455 for (i = 0; i < qdev->chan_num; i++) {
456 ret = qdma_check_queue_status(qdev, dir, i);
457 if (ret)
458 return ret;
459
460 q = &(*queues)[i];
461 q->ring_size = QDMA_DEFAULT_RING_SIZE;
462 q->idx_mask = q->ring_size - 2;
463 q->qdev = qdev;
464 q->dir = dir;
465 q->qid = i;
466 q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE;
467 q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) +
468 i * QDMA_DMAP_REG_STRIDE;
469 q->vchan.desc_free = qdma_free_vdesc;
470 vchan_init(&q->vchan, &qdev->dma_dev);
471 }
472
473 return 0;
474 }
475
qdma_device_verify(struct qdma_device * qdev)476 static int qdma_device_verify(struct qdma_device *qdev)
477 {
478 u32 value;
479 int ret;
480
481 ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value);
482 if (ret)
483 return ret;
484
485 value = FIELD_GET(QDMA_IDENTIFIER_MASK, value);
486 if (value != QDMA_IDENTIFIER) {
487 qdma_err(qdev, "Invalid identifier");
488 return -ENODEV;
489 }
490 qdev->rfields = qdma_regfs_default;
491 qdev->roffs = qdma_regos_default;
492
493 return 0;
494 }
495
qdma_device_setup(struct qdma_device * qdev)496 static int qdma_device_setup(struct qdma_device *qdev)
497 {
498 u32 ring_sz = QDMA_DEFAULT_RING_SIZE;
499 int ret = 0;
500
501 ret = qdma_setup_fmap_context(qdev);
502 if (ret) {
503 qdma_err(qdev, "Failed setup fmap context");
504 return ret;
505 }
506
507 /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */
508 ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE);
509 if (ret) {
510 qdma_err(qdev, "Failed to setup ring %d of size %ld",
511 QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE);
512 return ret;
513 }
514
515 /* Enable memory-mapped DMA engine in both directions */
516 ret = qdma_sgdma_control(qdev, 1);
517 if (ret) {
518 qdma_err(qdev, "Failed to SGDMA with error %d", ret);
519 return ret;
520 }
521
522 ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV);
523 if (ret) {
524 qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret);
525 return ret;
526 }
527
528 ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM);
529 if (ret) {
530 qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret);
531 return ret;
532 }
533
534 return 0;
535 }
536
537 /**
538 * qdma_free_queue_resources() - Free queue resources
539 * @chan: DMA channel
540 */
qdma_free_queue_resources(struct dma_chan * chan)541 static void qdma_free_queue_resources(struct dma_chan *chan)
542 {
543 struct qdma_queue *queue = to_qdma_queue(chan);
544 struct qdma_device *qdev = queue->qdev;
545 struct qdma_platdata *pdata;
546
547 qdma_clear_queue_context(queue);
548 vchan_free_chan_resources(&queue->vchan);
549 pdata = dev_get_platdata(&qdev->pdev->dev);
550 dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE,
551 queue->desc_base, queue->dma_desc_base);
552 }
553
554 /**
555 * qdma_alloc_queue_resources() - Allocate queue resources
556 * @chan: DMA channel
557 */
qdma_alloc_queue_resources(struct dma_chan * chan)558 static int qdma_alloc_queue_resources(struct dma_chan *chan)
559 {
560 struct qdma_queue *queue = to_qdma_queue(chan);
561 struct qdma_device *qdev = queue->qdev;
562 struct qdma_ctxt_sw_desc desc;
563 struct qdma_platdata *pdata;
564 size_t size;
565 int ret;
566
567 ret = qdma_clear_queue_context(queue);
568 if (ret)
569 return ret;
570
571 pdata = dev_get_platdata(&qdev->pdev->dev);
572 size = queue->ring_size * QDMA_MM_DESC_SIZE;
573 queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size,
574 &queue->dma_desc_base,
575 GFP_KERNEL);
576 if (!queue->desc_base) {
577 qdma_err(qdev, "Failed to allocate descriptor ring");
578 return -ENOMEM;
579 }
580
581 /* Setup SW descriptor queue context for DMA memory map */
582 desc.vec = qdma_get_intr_ring_idx(qdev);
583 desc.desc_base = queue->dma_desc_base;
584 ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid);
585 if (ret) {
586 qdma_err(qdev, "Failed to setup SW desc ctxt for %s",
587 chan->name);
588 dma_free_coherent(pdata->dma_dev, size, queue->desc_base,
589 queue->dma_desc_base);
590 return ret;
591 }
592
593 queue->pidx = 0;
594 queue->cidx = 0;
595
596 return 0;
597 }
598
qdma_filter_fn(struct dma_chan * chan,void * param)599 static bool qdma_filter_fn(struct dma_chan *chan, void *param)
600 {
601 struct qdma_queue *queue = to_qdma_queue(chan);
602 struct qdma_queue_info *info = param;
603
604 return info->dir == queue->dir;
605 }
606
qdma_xfer_start(struct qdma_queue * queue)607 static int qdma_xfer_start(struct qdma_queue *queue)
608 {
609 struct qdma_device *qdev = queue->qdev;
610 int ret;
611
612 if (!vchan_next_desc(&queue->vchan))
613 return 0;
614
615 qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d",
616 queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid);
617
618 ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx);
619 if (ret) {
620 qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d",
621 queue->pidx, CHAN_STR(queue), queue->qid);
622 }
623
624 return ret;
625 }
626
qdma_issue_pending(struct dma_chan * chan)627 static void qdma_issue_pending(struct dma_chan *chan)
628 {
629 struct qdma_queue *queue = to_qdma_queue(chan);
630 unsigned long flags;
631
632 spin_lock_irqsave(&queue->vchan.lock, flags);
633 if (vchan_issue_pending(&queue->vchan)) {
634 if (queue->submitted_vdesc) {
635 queue->issued_vdesc = queue->submitted_vdesc;
636 queue->submitted_vdesc = NULL;
637 }
638 qdma_xfer_start(queue);
639 }
640
641 spin_unlock_irqrestore(&queue->vchan.lock, flags);
642 }
643
qdma_get_desc(struct qdma_queue * q)644 static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q)
645 {
646 struct qdma_mm_desc *desc;
647
648 if (((q->pidx + 1) & q->idx_mask) == q->cidx)
649 return NULL;
650
651 desc = q->desc_base + q->pidx;
652 q->pidx = (q->pidx + 1) & q->idx_mask;
653
654 return desc;
655 }
656
qdma_hw_enqueue(struct qdma_queue * q,struct qdma_mm_vdesc * vdesc)657 static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc)
658 {
659 struct qdma_mm_desc *desc;
660 struct scatterlist *sg;
661 u64 addr, *src, *dst;
662 u32 rest, len;
663 int ret = 0;
664 u32 i;
665
666 if (!vdesc->sg_len)
667 return 0;
668
669 if (q->dir == DMA_MEM_TO_DEV) {
670 dst = &vdesc->dev_addr;
671 src = &addr;
672 } else {
673 dst = &addr;
674 src = &vdesc->dev_addr;
675 }
676
677 for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) {
678 addr = sg_dma_address(sg) + vdesc->sg_off;
679 rest = sg_dma_len(sg) - vdesc->sg_off;
680 while (rest) {
681 len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN);
682 desc = qdma_get_desc(q);
683 if (!desc) {
684 ret = -EBUSY;
685 goto out;
686 }
687
688 desc->src_addr = cpu_to_le64(*src);
689 desc->dst_addr = cpu_to_le64(*dst);
690 desc->len = cpu_to_le32(len);
691
692 vdesc->dev_addr += len;
693 vdesc->sg_off += len;
694 vdesc->pending_descs++;
695 addr += len;
696 rest -= len;
697 }
698 vdesc->sg_off = 0;
699 }
700 out:
701 vdesc->sg_len -= i;
702 vdesc->pidx = q->pidx;
703 return ret;
704 }
705
qdma_fill_pending_vdesc(struct qdma_queue * q)706 static void qdma_fill_pending_vdesc(struct qdma_queue *q)
707 {
708 struct virt_dma_chan *vc = &q->vchan;
709 struct qdma_mm_vdesc *vdesc = NULL;
710 struct virt_dma_desc *vd;
711 int ret;
712
713 if (!list_empty(&vc->desc_issued)) {
714 vd = &q->issued_vdesc->vdesc;
715 list_for_each_entry_from(vd, &vc->desc_issued, node) {
716 vdesc = to_qdma_vdesc(vd);
717 ret = qdma_hw_enqueue(q, vdesc);
718 if (ret) {
719 q->issued_vdesc = vdesc;
720 return;
721 }
722 }
723 q->issued_vdesc = vdesc;
724 }
725
726 if (list_empty(&vc->desc_submitted))
727 return;
728
729 if (q->submitted_vdesc)
730 vd = &q->submitted_vdesc->vdesc;
731 else
732 vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node);
733
734 list_for_each_entry_from(vd, &vc->desc_submitted, node) {
735 vdesc = to_qdma_vdesc(vd);
736 ret = qdma_hw_enqueue(q, vdesc);
737 if (ret)
738 break;
739 }
740 q->submitted_vdesc = vdesc;
741 }
742
qdma_tx_submit(struct dma_async_tx_descriptor * tx)743 static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx)
744 {
745 struct virt_dma_chan *vc = to_virt_chan(tx->chan);
746 struct qdma_queue *q = to_qdma_queue(&vc->chan);
747 struct virt_dma_desc *vd;
748 unsigned long flags;
749 dma_cookie_t cookie;
750
751 vd = container_of(tx, struct virt_dma_desc, tx);
752 spin_lock_irqsave(&vc->lock, flags);
753 cookie = dma_cookie_assign(tx);
754
755 list_move_tail(&vd->node, &vc->desc_submitted);
756 qdma_fill_pending_vdesc(q);
757 spin_unlock_irqrestore(&vc->lock, flags);
758
759 return cookie;
760 }
761
762 static struct dma_async_tx_descriptor *
qdma_prep_device_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)763 qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
764 unsigned int sg_len, enum dma_transfer_direction dir,
765 unsigned long flags, void *context)
766 {
767 struct qdma_queue *q = to_qdma_queue(chan);
768 struct dma_async_tx_descriptor *tx;
769 struct qdma_mm_vdesc *vdesc;
770
771 vdesc = kzalloc_obj(*vdesc, GFP_NOWAIT);
772 if (!vdesc)
773 return NULL;
774 vdesc->sgl = sgl;
775 vdesc->sg_len = sg_len;
776 if (dir == DMA_MEM_TO_DEV)
777 vdesc->dev_addr = q->cfg.dst_addr;
778 else
779 vdesc->dev_addr = q->cfg.src_addr;
780
781 tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags);
782 tx->tx_submit = qdma_tx_submit;
783
784 return tx;
785 }
786
qdma_device_config(struct dma_chan * chan,struct dma_slave_config * cfg)787 static int qdma_device_config(struct dma_chan *chan,
788 struct dma_slave_config *cfg)
789 {
790 struct qdma_queue *q = to_qdma_queue(chan);
791
792 memcpy(&q->cfg, cfg, sizeof(*cfg));
793
794 return 0;
795 }
796
qdma_arm_err_intr(const struct qdma_device * qdev)797 static int qdma_arm_err_intr(const struct qdma_device *qdev)
798 {
799 u32 value = 0;
800
801 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid);
802 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx);
803 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1);
804
805 return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT);
806 }
807
qdma_error_isr(int irq,void * data)808 static irqreturn_t qdma_error_isr(int irq, void *data)
809 {
810 struct qdma_device *qdev = data;
811 u32 err_stat = 0;
812 int ret;
813
814 ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT);
815 if (ret) {
816 qdma_err(qdev, "read error state failed, ret %d", ret);
817 goto out;
818 }
819
820 qdma_err(qdev, "global error %d", err_stat);
821 ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT);
822 if (ret)
823 qdma_err(qdev, "clear error state failed, ret %d", ret);
824
825 out:
826 qdma_arm_err_intr(qdev);
827 return IRQ_HANDLED;
828 }
829
qdma_queue_isr(int irq,void * data)830 static irqreturn_t qdma_queue_isr(int irq, void *data)
831 {
832 struct qdma_intr_ring *intr = data;
833 struct qdma_queue *q = NULL;
834 struct qdma_device *qdev;
835 u32 index, comp_desc;
836 u64 intr_ent;
837 u8 color;
838 int ret;
839 u16 qid;
840
841 qdev = intr->qdev;
842 index = intr->cidx;
843 while (1) {
844 struct virt_dma_desc *vd;
845 struct qdma_mm_vdesc *vdesc;
846 unsigned long flags;
847 u32 cidx;
848
849 intr_ent = le64_to_cpu(intr->base[index]);
850 color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent);
851 if (color != intr->color)
852 break;
853
854 qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent);
855 if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent))
856 q = qdev->c2h_queues;
857 else
858 q = qdev->h2c_queues;
859 q += qid;
860
861 cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent);
862
863 spin_lock_irqsave(&q->vchan.lock, flags);
864 comp_desc = (cidx - q->cidx) & q->idx_mask;
865
866 vd = vchan_next_desc(&q->vchan);
867 if (!vd)
868 goto skip;
869
870 vdesc = to_qdma_vdesc(vd);
871 while (comp_desc > vdesc->pending_descs) {
872 list_del(&vd->node);
873 vchan_cookie_complete(vd);
874 comp_desc -= vdesc->pending_descs;
875 vd = vchan_next_desc(&q->vchan);
876 vdesc = to_qdma_vdesc(vd);
877 }
878 vdesc->pending_descs -= comp_desc;
879 if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) {
880 list_del(&vd->node);
881 vchan_cookie_complete(vd);
882 }
883 q->cidx = cidx;
884
885 qdma_fill_pending_vdesc(q);
886 qdma_xfer_start(q);
887
888 skip:
889 spin_unlock_irqrestore(&q->vchan.lock, flags);
890
891 /*
892 * Wrap the index value and flip the expected color value if
893 * interrupt aggregation PIDX has wrapped around.
894 */
895 index++;
896 index &= QDMA_INTR_RING_IDX_MASK;
897 if (!index)
898 intr->color = !intr->color;
899 }
900
901 /*
902 * Update the software interrupt aggregation ring CIDX if a valid entry
903 * was found.
904 */
905 if (q) {
906 qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index);
907
908 /*
909 * Record the last read index of status descriptor from the
910 * interrupt aggregation ring.
911 */
912 intr->cidx = index;
913
914 ret = qdma_update_cidx(q, intr->ridx, index);
915 if (ret) {
916 qdma_err(qdev, "Failed to update IRQ CIDX");
917 return IRQ_NONE;
918 }
919 }
920
921 return IRQ_HANDLED;
922 }
923
qdma_init_error_irq(struct qdma_device * qdev)924 static int qdma_init_error_irq(struct qdma_device *qdev)
925 {
926 struct device *dev = &qdev->pdev->dev;
927 int ret;
928 u32 vec;
929
930 vec = qdev->queue_irq_start - 1;
931
932 ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr,
933 IRQF_ONESHOT, "amd-qdma-error", qdev);
934 if (ret) {
935 qdma_err(qdev, "Failed to request error IRQ vector: %d", vec);
936 return ret;
937 }
938
939 ret = qdma_arm_err_intr(qdev);
940 if (ret)
941 qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret);
942
943 return ret;
944 }
945
qdmam_alloc_qintr_rings(struct qdma_device * qdev)946 static int qdmam_alloc_qintr_rings(struct qdma_device *qdev)
947 {
948 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev);
949 struct device *dev = &qdev->pdev->dev;
950 u32 ctxt[QDMA_CTXT_REGMAP_LEN];
951 struct qdma_intr_ring *ring;
952 struct qdma_ctxt_intr intr_ctxt;
953 u32 vector;
954 int ret, i;
955
956 qdev->qintr_ring_num = qdev->queue_irq_num;
957 qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num,
958 sizeof(*qdev->qintr_rings),
959 GFP_KERNEL);
960 if (!qdev->qintr_rings)
961 return -ENOMEM;
962
963 vector = qdev->queue_irq_start;
964 for (i = 0; i < qdev->qintr_ring_num; i++, vector++) {
965 ring = &qdev->qintr_rings[i];
966 ring->qdev = qdev;
967 ring->msix_id = qdev->err_irq_idx + i + 1;
968 ring->ridx = i;
969 ring->color = 1;
970 ring->base = dmam_alloc_coherent(pdata->dma_dev,
971 QDMA_INTR_RING_SIZE,
972 &ring->dev_base, GFP_KERNEL);
973 if (!ring->base) {
974 qdma_err(qdev, "Failed to alloc intr ring %d", i);
975 return -ENOMEM;
976 }
977 intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base);
978 intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096;
979 intr_ctxt.vec = ring->msix_id;
980 intr_ctxt.valid = true;
981 intr_ctxt.color = true;
982 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL,
983 QDMA_CTXT_CLEAR, ring->ridx, NULL);
984 if (ret) {
985 qdma_err(qdev, "Failed clear intr ctx, ret %d", ret);
986 return ret;
987 }
988
989 qdma_prep_intr_context(qdev, &intr_ctxt, ctxt);
990 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL,
991 QDMA_CTXT_WRITE, ring->ridx, ctxt);
992 if (ret) {
993 qdma_err(qdev, "Failed setup intr ctx, ret %d", ret);
994 return ret;
995 }
996
997 ret = devm_request_threaded_irq(dev, vector, NULL,
998 qdma_queue_isr, IRQF_ONESHOT,
999 "amd-qdma-queue", ring);
1000 if (ret) {
1001 qdma_err(qdev, "Failed to request irq %d", vector);
1002 return ret;
1003 }
1004 }
1005
1006 return 0;
1007 }
1008
qdma_intr_init(struct qdma_device * qdev)1009 static int qdma_intr_init(struct qdma_device *qdev)
1010 {
1011 int ret;
1012
1013 ret = qdma_init_error_irq(qdev);
1014 if (ret) {
1015 qdma_err(qdev, "Failed to init error IRQs, ret %d", ret);
1016 return ret;
1017 }
1018
1019 ret = qdmam_alloc_qintr_rings(qdev);
1020 if (ret) {
1021 qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret);
1022 return ret;
1023 }
1024
1025 return 0;
1026 }
1027
amd_qdma_remove(struct platform_device * pdev)1028 static void amd_qdma_remove(struct platform_device *pdev)
1029 {
1030 struct qdma_device *qdev = platform_get_drvdata(pdev);
1031
1032 qdma_sgdma_control(qdev, 0);
1033 dma_async_device_unregister(&qdev->dma_dev);
1034
1035 mutex_destroy(&qdev->ctxt_lock);
1036 }
1037
amd_qdma_probe(struct platform_device * pdev)1038 static int amd_qdma_probe(struct platform_device *pdev)
1039 {
1040 struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev);
1041 struct qdma_device *qdev;
1042 struct resource *res;
1043 void __iomem *regs;
1044 int ret;
1045
1046 qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
1047 if (!qdev)
1048 return -ENOMEM;
1049
1050 platform_set_drvdata(pdev, qdev);
1051 qdev->pdev = pdev;
1052 mutex_init(&qdev->ctxt_lock);
1053
1054 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1055 if (!res) {
1056 qdma_err(qdev, "Failed to get IRQ resource");
1057 ret = -ENODEV;
1058 goto failed;
1059 }
1060 qdev->err_irq_idx = pdata->irq_index;
1061 qdev->queue_irq_start = res->start + 1;
1062 qdev->queue_irq_num = resource_size(res) - 1;
1063
1064 regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
1065 if (IS_ERR(regs)) {
1066 ret = PTR_ERR(regs);
1067 qdma_err(qdev, "Failed to map IO resource, err %d", ret);
1068 goto failed;
1069 }
1070
1071 qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs,
1072 &qdma_regmap_config);
1073 if (IS_ERR(qdev->regmap)) {
1074 ret = PTR_ERR(qdev->regmap);
1075 qdma_err(qdev, "Regmap init failed, err %d", ret);
1076 goto failed;
1077 }
1078
1079 ret = qdma_device_verify(qdev);
1080 if (ret)
1081 goto failed;
1082
1083 ret = qdma_get_hw_info(qdev);
1084 if (ret)
1085 goto failed;
1086
1087 INIT_LIST_HEAD(&qdev->dma_dev.channels);
1088
1089 ret = qdma_device_setup(qdev);
1090 if (ret)
1091 goto failed;
1092
1093 ret = qdma_intr_init(qdev);
1094 if (ret) {
1095 qdma_err(qdev, "Failed to initialize IRQs %d", ret);
1096 goto failed_disable_engine;
1097 }
1098
1099 dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask);
1100 dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask);
1101
1102 qdev->dma_dev.dev = &pdev->dev;
1103 qdev->dma_dev.filter.map = pdata->device_map;
1104 qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2;
1105 qdev->dma_dev.filter.fn = qdma_filter_fn;
1106 qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources;
1107 qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources;
1108 qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg;
1109 qdev->dma_dev.device_config = qdma_device_config;
1110 qdev->dma_dev.device_issue_pending = qdma_issue_pending;
1111 qdev->dma_dev.device_tx_status = dma_cookie_status;
1112 qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1113
1114 ret = dma_async_device_register(&qdev->dma_dev);
1115 if (ret) {
1116 qdma_err(qdev, "Failed to register AMD QDMA: %d", ret);
1117 goto failed_disable_engine;
1118 }
1119
1120 return 0;
1121
1122 failed_disable_engine:
1123 qdma_sgdma_control(qdev, 0);
1124 failed:
1125 mutex_destroy(&qdev->ctxt_lock);
1126 qdma_err(qdev, "Failed to probe AMD QDMA driver");
1127 return ret;
1128 }
1129
1130 static struct platform_driver amd_qdma_driver = {
1131 .driver = {
1132 .name = "amd-qdma",
1133 },
1134 .probe = amd_qdma_probe,
1135 .remove = amd_qdma_remove,
1136 };
1137
1138 module_platform_driver(amd_qdma_driver);
1139
1140 MODULE_DESCRIPTION("AMD QDMA driver");
1141 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
1142 MODULE_LICENSE("GPL");
1143