1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * DMA driver for AMD Queue-based DMA Subsystem 4 * 5 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 6 */ 7 #include <linux/bitfield.h> 8 #include <linux/bitops.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/module.h> 12 #include <linux/mod_devicetable.h> 13 #include <linux/platform_device.h> 14 #include <linux/platform_data/amd_qdma.h> 15 #include <linux/regmap.h> 16 17 #include "qdma.h" 18 19 #define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") 20 #define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) 21 22 /* MMIO regmap config for all QDMA registers */ 23 static const struct regmap_config qdma_regmap_config = { 24 .reg_bits = 32, 25 .val_bits = 32, 26 .reg_stride = 4, 27 }; 28 29 static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) 30 { 31 return container_of(chan, struct qdma_queue, vchan.chan); 32 } 33 34 static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) 35 { 36 return container_of(vdesc, struct qdma_mm_vdesc, vdesc); 37 } 38 39 static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) 40 { 41 u32 idx; 42 43 idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; 44 qdev->qintr_ring_idx %= qdev->qintr_ring_num; 45 46 return idx; 47 } 48 49 static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, 50 enum qdma_reg_fields field) 51 { 52 const struct qdma_reg_field *f = &qdev->rfields[field]; 53 u16 low_pos, hi_pos, low_bit, hi_bit; 54 u64 value = 0, mask; 55 56 low_pos = f->lsb / BITS_PER_TYPE(*data); 57 hi_pos = f->msb / BITS_PER_TYPE(*data); 58 59 if (low_pos == hi_pos) { 60 low_bit = f->lsb % BITS_PER_TYPE(*data); 61 hi_bit = f->msb % BITS_PER_TYPE(*data); 62 mask = GENMASK(hi_bit, low_bit); 63 value = (data[low_pos] & mask) >> low_bit; 64 } else if (hi_pos == low_pos + 1) { 65 low_bit = f->lsb % BITS_PER_TYPE(*data); 66 hi_bit = low_bit + (f->msb - f->lsb); 67 value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | 68 data[low_pos]; 69 mask = GENMASK_ULL(hi_bit, low_bit); 70 value = (value & mask) >> low_bit; 71 } else { 72 hi_bit = f->msb % BITS_PER_TYPE(*data); 73 mask = GENMASK(hi_bit, 0); 74 value = data[hi_pos] & mask; 75 low_bit = f->msb - f->lsb - hi_bit; 76 value <<= low_bit; 77 low_bit -= 32; 78 value |= (u64)data[hi_pos - 1] << low_bit; 79 mask = GENMASK(31, 32 - low_bit); 80 value |= (data[hi_pos - 2] & mask) >> low_bit; 81 } 82 83 return value; 84 } 85 86 static void qdma_set_field(const struct qdma_device *qdev, u32 *data, 87 enum qdma_reg_fields field, u64 value) 88 { 89 const struct qdma_reg_field *f = &qdev->rfields[field]; 90 u16 low_pos, hi_pos, low_bit; 91 92 low_pos = f->lsb / BITS_PER_TYPE(*data); 93 hi_pos = f->msb / BITS_PER_TYPE(*data); 94 low_bit = f->lsb % BITS_PER_TYPE(*data); 95 96 data[low_pos++] |= value << low_bit; 97 if (low_pos <= hi_pos) 98 data[low_pos++] |= (u32)(value >> (32 - low_bit)); 99 if (low_pos <= hi_pos) 100 data[low_pos] |= (u32)(value >> (64 - low_bit)); 101 } 102 103 static inline int qdma_reg_write(const struct qdma_device *qdev, 104 const u32 *data, enum qdma_regs reg) 105 { 106 const struct qdma_reg *r = &qdev->roffs[reg]; 107 int ret; 108 109 if (r->count > 1) 110 ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); 111 else 112 ret = regmap_write(qdev->regmap, r->off, *data); 113 114 return ret; 115 } 116 117 static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, 118 enum qdma_regs reg) 119 { 120 const struct qdma_reg *r = &qdev->roffs[reg]; 121 int ret; 122 123 if (r->count > 1) 124 ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); 125 else 126 ret = regmap_read(qdev->regmap, r->off, data); 127 128 return ret; 129 } 130 131 static int qdma_context_cmd_execute(const struct qdma_device *qdev, 132 enum qdma_ctxt_type type, 133 enum qdma_ctxt_cmd cmd, u16 index) 134 { 135 u32 value = 0; 136 int ret; 137 138 qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); 139 qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); 140 qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); 141 142 ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); 143 if (ret) 144 return ret; 145 146 ret = regmap_read_poll_timeout(qdev->regmap, 147 QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), 148 value, 149 !qdma_get_field(qdev, &value, 150 QDMA_REGF_CMD_BUSY), 151 QDMA_POLL_INTRVL_US, 152 QDMA_POLL_TIMEOUT_US); 153 if (ret) { 154 qdma_err(qdev, "Context command execution timed out"); 155 return ret; 156 } 157 158 return 0; 159 } 160 161 static int qdma_context_write_data(const struct qdma_device *qdev, 162 const u32 *data) 163 { 164 u32 mask[QDMA_CTXT_REGMAP_LEN]; 165 int ret; 166 167 memset(mask, ~0, sizeof(mask)); 168 169 ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); 170 if (ret) 171 return ret; 172 173 ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); 174 if (ret) 175 return ret; 176 177 return 0; 178 } 179 180 static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, 181 const struct qdma_ctxt_sw_desc *ctxt, 182 u32 *data) 183 { 184 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 185 qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); 186 qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); 187 qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); 188 189 qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); 190 qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); 191 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); 192 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); 193 qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); 194 qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); 195 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); 196 qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); 197 qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); 198 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); 199 qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); 200 } 201 202 static void qdma_prep_intr_context(const struct qdma_device *qdev, 203 const struct qdma_ctxt_intr *ctxt, 204 u32 *data) 205 { 206 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 207 qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); 208 qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); 209 qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); 210 qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); 211 qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); 212 qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); 213 } 214 215 static void qdma_prep_fmap_context(const struct qdma_device *qdev, 216 const struct qdma_ctxt_fmap *ctxt, 217 u32 *data) 218 { 219 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 220 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); 221 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); 222 } 223 224 /* 225 * Program the indirect context register space 226 * 227 * Once the queue is enabled, context is dynamically updated by hardware. Any 228 * modification of the context through this API when the queue is enabled can 229 * result in unexpected behavior. Reading the context when the queue is enabled 230 * is not recommended as it can result in reduced performance. 231 */ 232 static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, 233 enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) 234 { 235 int ret; 236 237 mutex_lock(&qdev->ctxt_lock); 238 if (cmd == QDMA_CTXT_WRITE) { 239 ret = qdma_context_write_data(qdev, ctxt); 240 if (ret) 241 goto failed; 242 } 243 244 ret = qdma_context_cmd_execute(qdev, type, cmd, index); 245 if (ret) 246 goto failed; 247 248 if (cmd == QDMA_CTXT_READ) { 249 ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); 250 if (ret) 251 goto failed; 252 } 253 254 failed: 255 mutex_unlock(&qdev->ctxt_lock); 256 257 return ret; 258 } 259 260 static int qdma_check_queue_status(struct qdma_device *qdev, 261 enum dma_transfer_direction dir, u16 qid) 262 { 263 u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; 264 enum qdma_ctxt_type type; 265 int ret; 266 267 if (dir == DMA_MEM_TO_DEV) 268 type = QDMA_CTXT_DESC_SW_H2C; 269 else 270 type = QDMA_CTXT_DESC_SW_C2H; 271 ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); 272 if (ret) 273 return ret; 274 275 status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); 276 if (status) { 277 qdma_err(qdev, "queue %d already in use", qid); 278 return -EBUSY; 279 } 280 281 return 0; 282 } 283 284 static int qdma_clear_queue_context(const struct qdma_queue *queue) 285 { 286 static const enum qdma_ctxt_type h2c_types[] = { 287 QDMA_CTXT_DESC_SW_H2C, 288 QDMA_CTXT_DESC_HW_H2C, 289 QDMA_CTXT_DESC_CR_H2C, 290 QDMA_CTXT_PFTCH, 291 }; 292 static const enum qdma_ctxt_type c2h_types[] = { 293 QDMA_CTXT_DESC_SW_C2H, 294 QDMA_CTXT_DESC_HW_C2H, 295 QDMA_CTXT_DESC_CR_C2H, 296 QDMA_CTXT_PFTCH, 297 }; 298 struct qdma_device *qdev = queue->qdev; 299 const enum qdma_ctxt_type *type; 300 int ret, num, i; 301 302 if (queue->dir == DMA_MEM_TO_DEV) { 303 type = h2c_types; 304 num = ARRAY_SIZE(h2c_types); 305 } else { 306 type = c2h_types; 307 num = ARRAY_SIZE(c2h_types); 308 } 309 for (i = 0; i < num; i++) { 310 ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, 311 queue->qid, NULL); 312 if (ret) { 313 qdma_err(qdev, "Failed to clear ctxt %d", type[i]); 314 return ret; 315 } 316 } 317 318 return 0; 319 } 320 321 static int qdma_setup_fmap_context(struct qdma_device *qdev) 322 { 323 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 324 struct qdma_ctxt_fmap fmap; 325 int ret; 326 327 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, 328 qdev->fid, NULL); 329 if (ret) { 330 qdma_err(qdev, "Failed clearing context"); 331 return ret; 332 } 333 334 fmap.qbase = 0; 335 fmap.qmax = qdev->chan_num * 2; 336 qdma_prep_fmap_context(qdev, &fmap, ctxt); 337 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, 338 qdev->fid, ctxt); 339 if (ret) 340 qdma_err(qdev, "Failed setup fmap, ret %d", ret); 341 342 return ret; 343 } 344 345 static int qdma_setup_queue_context(struct qdma_device *qdev, 346 const struct qdma_ctxt_sw_desc *sw_desc, 347 enum dma_transfer_direction dir, u16 qid) 348 { 349 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 350 enum qdma_ctxt_type type; 351 int ret; 352 353 if (dir == DMA_MEM_TO_DEV) 354 type = QDMA_CTXT_DESC_SW_H2C; 355 else 356 type = QDMA_CTXT_DESC_SW_C2H; 357 358 qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); 359 /* Setup SW descriptor context */ 360 ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); 361 if (ret) 362 qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); 363 364 return ret; 365 } 366 367 /* 368 * Enable or disable memory-mapped DMA engines 369 * 1: enable, 0: disable 370 */ 371 static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) 372 { 373 int ret; 374 375 ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); 376 ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); 377 378 return ret; 379 } 380 381 static int qdma_get_hw_info(struct qdma_device *qdev) 382 { 383 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); 384 u32 value = 0; 385 int ret; 386 387 ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); 388 if (ret) 389 return ret; 390 391 value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; 392 if (pdata->max_mm_channels * 2 > value) { 393 qdma_err(qdev, "not enough hw queues %d", value); 394 return -EINVAL; 395 } 396 qdev->chan_num = pdata->max_mm_channels; 397 398 ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); 399 if (ret) 400 return ret; 401 402 qdma_info(qdev, "max channel %d, function id %d", 403 qdev->chan_num, qdev->fid); 404 405 return 0; 406 } 407 408 static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) 409 { 410 struct qdma_device *qdev = queue->qdev; 411 412 return regmap_write(qdev->regmap, queue->pidx_reg, 413 pidx | QDMA_QUEUE_ARM_BIT); 414 } 415 416 static inline int qdma_update_cidx(const struct qdma_queue *queue, 417 u16 ridx, u16 cidx) 418 { 419 struct qdma_device *qdev = queue->qdev; 420 421 return regmap_write(qdev->regmap, queue->cidx_reg, 422 ((u32)ridx << 16) | cidx); 423 } 424 425 /** 426 * qdma_free_vdesc - Free descriptor 427 * @vdesc: Virtual DMA descriptor 428 */ 429 static void qdma_free_vdesc(struct virt_dma_desc *vdesc) 430 { 431 struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); 432 433 kfree(vd); 434 } 435 436 static int qdma_alloc_queues(struct qdma_device *qdev, 437 enum dma_transfer_direction dir) 438 { 439 struct qdma_queue *q, **queues; 440 u32 i, pidx_base; 441 int ret; 442 443 if (dir == DMA_MEM_TO_DEV) { 444 queues = &qdev->h2c_queues; 445 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); 446 } else { 447 queues = &qdev->c2h_queues; 448 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); 449 } 450 451 *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), 452 GFP_KERNEL); 453 if (!*queues) 454 return -ENOMEM; 455 456 for (i = 0; i < qdev->chan_num; i++) { 457 ret = qdma_check_queue_status(qdev, dir, i); 458 if (ret) 459 return ret; 460 461 q = &(*queues)[i]; 462 q->ring_size = QDMA_DEFAULT_RING_SIZE; 463 q->idx_mask = q->ring_size - 2; 464 q->qdev = qdev; 465 q->dir = dir; 466 q->qid = i; 467 q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; 468 q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + 469 i * QDMA_DMAP_REG_STRIDE; 470 q->vchan.desc_free = qdma_free_vdesc; 471 vchan_init(&q->vchan, &qdev->dma_dev); 472 } 473 474 return 0; 475 } 476 477 static int qdma_device_verify(struct qdma_device *qdev) 478 { 479 u32 value; 480 int ret; 481 482 ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); 483 if (ret) 484 return ret; 485 486 value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); 487 if (value != QDMA_IDENTIFIER) { 488 qdma_err(qdev, "Invalid identifier"); 489 return -ENODEV; 490 } 491 qdev->rfields = qdma_regfs_default; 492 qdev->roffs = qdma_regos_default; 493 494 return 0; 495 } 496 497 static int qdma_device_setup(struct qdma_device *qdev) 498 { 499 u32 ring_sz = QDMA_DEFAULT_RING_SIZE; 500 int ret = 0; 501 502 ret = qdma_setup_fmap_context(qdev); 503 if (ret) { 504 qdma_err(qdev, "Failed setup fmap context"); 505 return ret; 506 } 507 508 /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ 509 ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); 510 if (ret) { 511 qdma_err(qdev, "Failed to setup ring %d of size %ld", 512 QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); 513 return ret; 514 } 515 516 /* Enable memory-mapped DMA engine in both directions */ 517 ret = qdma_sgdma_control(qdev, 1); 518 if (ret) { 519 qdma_err(qdev, "Failed to SGDMA with error %d", ret); 520 return ret; 521 } 522 523 ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); 524 if (ret) { 525 qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); 526 return ret; 527 } 528 529 ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); 530 if (ret) { 531 qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); 532 return ret; 533 } 534 535 return 0; 536 } 537 538 /** 539 * qdma_free_queue_resources() - Free queue resources 540 * @chan: DMA channel 541 */ 542 static void qdma_free_queue_resources(struct dma_chan *chan) 543 { 544 struct qdma_queue *queue = to_qdma_queue(chan); 545 struct qdma_device *qdev = queue->qdev; 546 struct qdma_platdata *pdata; 547 548 qdma_clear_queue_context(queue); 549 vchan_free_chan_resources(&queue->vchan); 550 pdata = dev_get_platdata(&qdev->pdev->dev); 551 dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, 552 queue->desc_base, queue->dma_desc_base); 553 } 554 555 /** 556 * qdma_alloc_queue_resources() - Allocate queue resources 557 * @chan: DMA channel 558 */ 559 static int qdma_alloc_queue_resources(struct dma_chan *chan) 560 { 561 struct qdma_queue *queue = to_qdma_queue(chan); 562 struct qdma_device *qdev = queue->qdev; 563 struct qdma_ctxt_sw_desc desc; 564 struct qdma_platdata *pdata; 565 size_t size; 566 int ret; 567 568 ret = qdma_clear_queue_context(queue); 569 if (ret) 570 return ret; 571 572 pdata = dev_get_platdata(&qdev->pdev->dev); 573 size = queue->ring_size * QDMA_MM_DESC_SIZE; 574 queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, 575 &queue->dma_desc_base, 576 GFP_KERNEL); 577 if (!queue->desc_base) { 578 qdma_err(qdev, "Failed to allocate descriptor ring"); 579 return -ENOMEM; 580 } 581 582 /* Setup SW descriptor queue context for DMA memory map */ 583 desc.vec = qdma_get_intr_ring_idx(qdev); 584 desc.desc_base = queue->dma_desc_base; 585 ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); 586 if (ret) { 587 qdma_err(qdev, "Failed to setup SW desc ctxt for %s", 588 chan->name); 589 dma_free_coherent(pdata->dma_dev, size, queue->desc_base, 590 queue->dma_desc_base); 591 return ret; 592 } 593 594 queue->pidx = 0; 595 queue->cidx = 0; 596 597 return 0; 598 } 599 600 static bool qdma_filter_fn(struct dma_chan *chan, void *param) 601 { 602 struct qdma_queue *queue = to_qdma_queue(chan); 603 struct qdma_queue_info *info = param; 604 605 return info->dir == queue->dir; 606 } 607 608 static int qdma_xfer_start(struct qdma_queue *queue) 609 { 610 struct qdma_device *qdev = queue->qdev; 611 int ret; 612 613 if (!vchan_next_desc(&queue->vchan)) 614 return 0; 615 616 qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", 617 queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); 618 619 ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); 620 if (ret) { 621 qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", 622 queue->pidx, CHAN_STR(queue), queue->qid); 623 } 624 625 return ret; 626 } 627 628 static void qdma_issue_pending(struct dma_chan *chan) 629 { 630 struct qdma_queue *queue = to_qdma_queue(chan); 631 unsigned long flags; 632 633 spin_lock_irqsave(&queue->vchan.lock, flags); 634 if (vchan_issue_pending(&queue->vchan)) { 635 if (queue->submitted_vdesc) { 636 queue->issued_vdesc = queue->submitted_vdesc; 637 queue->submitted_vdesc = NULL; 638 } 639 qdma_xfer_start(queue); 640 } 641 642 spin_unlock_irqrestore(&queue->vchan.lock, flags); 643 } 644 645 static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) 646 { 647 struct qdma_mm_desc *desc; 648 649 if (((q->pidx + 1) & q->idx_mask) == q->cidx) 650 return NULL; 651 652 desc = q->desc_base + q->pidx; 653 q->pidx = (q->pidx + 1) & q->idx_mask; 654 655 return desc; 656 } 657 658 static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) 659 { 660 struct qdma_mm_desc *desc; 661 struct scatterlist *sg; 662 u64 addr, *src, *dst; 663 u32 rest, len; 664 int ret = 0; 665 u32 i; 666 667 if (!vdesc->sg_len) 668 return 0; 669 670 if (q->dir == DMA_MEM_TO_DEV) { 671 dst = &vdesc->dev_addr; 672 src = &addr; 673 } else { 674 dst = &addr; 675 src = &vdesc->dev_addr; 676 } 677 678 for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { 679 addr = sg_dma_address(sg) + vdesc->sg_off; 680 rest = sg_dma_len(sg) - vdesc->sg_off; 681 while (rest) { 682 len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); 683 desc = qdma_get_desc(q); 684 if (!desc) { 685 ret = -EBUSY; 686 goto out; 687 } 688 689 desc->src_addr = cpu_to_le64(*src); 690 desc->dst_addr = cpu_to_le64(*dst); 691 desc->len = cpu_to_le32(len); 692 693 vdesc->dev_addr += len; 694 vdesc->sg_off += len; 695 vdesc->pending_descs++; 696 addr += len; 697 rest -= len; 698 } 699 vdesc->sg_off = 0; 700 } 701 out: 702 vdesc->sg_len -= i; 703 vdesc->pidx = q->pidx; 704 return ret; 705 } 706 707 static void qdma_fill_pending_vdesc(struct qdma_queue *q) 708 { 709 struct virt_dma_chan *vc = &q->vchan; 710 struct qdma_mm_vdesc *vdesc = NULL; 711 struct virt_dma_desc *vd; 712 int ret; 713 714 if (!list_empty(&vc->desc_issued)) { 715 vd = &q->issued_vdesc->vdesc; 716 list_for_each_entry_from(vd, &vc->desc_issued, node) { 717 vdesc = to_qdma_vdesc(vd); 718 ret = qdma_hw_enqueue(q, vdesc); 719 if (ret) { 720 q->issued_vdesc = vdesc; 721 return; 722 } 723 } 724 q->issued_vdesc = vdesc; 725 } 726 727 if (list_empty(&vc->desc_submitted)) 728 return; 729 730 if (q->submitted_vdesc) 731 vd = &q->submitted_vdesc->vdesc; 732 else 733 vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); 734 735 list_for_each_entry_from(vd, &vc->desc_submitted, node) { 736 vdesc = to_qdma_vdesc(vd); 737 ret = qdma_hw_enqueue(q, vdesc); 738 if (ret) 739 break; 740 } 741 q->submitted_vdesc = vdesc; 742 } 743 744 static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) 745 { 746 struct virt_dma_chan *vc = to_virt_chan(tx->chan); 747 struct qdma_queue *q = to_qdma_queue(&vc->chan); 748 struct virt_dma_desc *vd; 749 unsigned long flags; 750 dma_cookie_t cookie; 751 752 vd = container_of(tx, struct virt_dma_desc, tx); 753 spin_lock_irqsave(&vc->lock, flags); 754 cookie = dma_cookie_assign(tx); 755 756 list_move_tail(&vd->node, &vc->desc_submitted); 757 qdma_fill_pending_vdesc(q); 758 spin_unlock_irqrestore(&vc->lock, flags); 759 760 return cookie; 761 } 762 763 static struct dma_async_tx_descriptor * 764 qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, 765 unsigned int sg_len, enum dma_transfer_direction dir, 766 unsigned long flags, void *context) 767 { 768 struct qdma_queue *q = to_qdma_queue(chan); 769 struct dma_async_tx_descriptor *tx; 770 struct qdma_mm_vdesc *vdesc; 771 772 vdesc = kzalloc(sizeof(*vdesc), GFP_NOWAIT); 773 if (!vdesc) 774 return NULL; 775 vdesc->sgl = sgl; 776 vdesc->sg_len = sg_len; 777 if (dir == DMA_MEM_TO_DEV) 778 vdesc->dev_addr = q->cfg.dst_addr; 779 else 780 vdesc->dev_addr = q->cfg.src_addr; 781 782 tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); 783 tx->tx_submit = qdma_tx_submit; 784 785 return tx; 786 } 787 788 static int qdma_device_config(struct dma_chan *chan, 789 struct dma_slave_config *cfg) 790 { 791 struct qdma_queue *q = to_qdma_queue(chan); 792 793 memcpy(&q->cfg, cfg, sizeof(*cfg)); 794 795 return 0; 796 } 797 798 static int qdma_arm_err_intr(const struct qdma_device *qdev) 799 { 800 u32 value = 0; 801 802 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); 803 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); 804 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); 805 806 return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); 807 } 808 809 static irqreturn_t qdma_error_isr(int irq, void *data) 810 { 811 struct qdma_device *qdev = data; 812 u32 err_stat = 0; 813 int ret; 814 815 ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); 816 if (ret) { 817 qdma_err(qdev, "read error state failed, ret %d", ret); 818 goto out; 819 } 820 821 qdma_err(qdev, "global error %d", err_stat); 822 ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); 823 if (ret) 824 qdma_err(qdev, "clear error state failed, ret %d", ret); 825 826 out: 827 qdma_arm_err_intr(qdev); 828 return IRQ_HANDLED; 829 } 830 831 static irqreturn_t qdma_queue_isr(int irq, void *data) 832 { 833 struct qdma_intr_ring *intr = data; 834 struct qdma_queue *q = NULL; 835 struct qdma_device *qdev; 836 u32 index, comp_desc; 837 u64 intr_ent; 838 u8 color; 839 int ret; 840 u16 qid; 841 842 qdev = intr->qdev; 843 index = intr->cidx; 844 while (1) { 845 struct virt_dma_desc *vd; 846 struct qdma_mm_vdesc *vdesc; 847 unsigned long flags; 848 u32 cidx; 849 850 intr_ent = le64_to_cpu(intr->base[index]); 851 color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); 852 if (color != intr->color) 853 break; 854 855 qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); 856 if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) 857 q = qdev->c2h_queues; 858 else 859 q = qdev->h2c_queues; 860 q += qid; 861 862 cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); 863 864 spin_lock_irqsave(&q->vchan.lock, flags); 865 comp_desc = (cidx - q->cidx) & q->idx_mask; 866 867 vd = vchan_next_desc(&q->vchan); 868 if (!vd) 869 goto skip; 870 871 vdesc = to_qdma_vdesc(vd); 872 while (comp_desc > vdesc->pending_descs) { 873 list_del(&vd->node); 874 vchan_cookie_complete(vd); 875 comp_desc -= vdesc->pending_descs; 876 vd = vchan_next_desc(&q->vchan); 877 vdesc = to_qdma_vdesc(vd); 878 } 879 vdesc->pending_descs -= comp_desc; 880 if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { 881 list_del(&vd->node); 882 vchan_cookie_complete(vd); 883 } 884 q->cidx = cidx; 885 886 qdma_fill_pending_vdesc(q); 887 qdma_xfer_start(q); 888 889 skip: 890 spin_unlock_irqrestore(&q->vchan.lock, flags); 891 892 /* 893 * Wrap the index value and flip the expected color value if 894 * interrupt aggregation PIDX has wrapped around. 895 */ 896 index++; 897 index &= QDMA_INTR_RING_IDX_MASK; 898 if (!index) 899 intr->color = !intr->color; 900 } 901 902 /* 903 * Update the software interrupt aggregation ring CIDX if a valid entry 904 * was found. 905 */ 906 if (q) { 907 qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); 908 909 /* 910 * Record the last read index of status descriptor from the 911 * interrupt aggregation ring. 912 */ 913 intr->cidx = index; 914 915 ret = qdma_update_cidx(q, intr->ridx, index); 916 if (ret) { 917 qdma_err(qdev, "Failed to update IRQ CIDX"); 918 return IRQ_NONE; 919 } 920 } 921 922 return IRQ_HANDLED; 923 } 924 925 static int qdma_init_error_irq(struct qdma_device *qdev) 926 { 927 struct device *dev = &qdev->pdev->dev; 928 int ret; 929 u32 vec; 930 931 vec = qdev->queue_irq_start - 1; 932 933 ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, 934 IRQF_ONESHOT, "amd-qdma-error", qdev); 935 if (ret) { 936 qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); 937 return ret; 938 } 939 940 ret = qdma_arm_err_intr(qdev); 941 if (ret) 942 qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); 943 944 return ret; 945 } 946 947 static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) 948 { 949 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); 950 struct device *dev = &qdev->pdev->dev; 951 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 952 struct qdma_intr_ring *ring; 953 struct qdma_ctxt_intr intr_ctxt; 954 u32 vector; 955 int ret, i; 956 957 qdev->qintr_ring_num = qdev->queue_irq_num; 958 qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, 959 sizeof(*qdev->qintr_rings), 960 GFP_KERNEL); 961 if (!qdev->qintr_rings) 962 return -ENOMEM; 963 964 vector = qdev->queue_irq_start; 965 for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { 966 ring = &qdev->qintr_rings[i]; 967 ring->qdev = qdev; 968 ring->msix_id = qdev->err_irq_idx + i + 1; 969 ring->ridx = i; 970 ring->color = 1; 971 ring->base = dmam_alloc_coherent(pdata->dma_dev, 972 QDMA_INTR_RING_SIZE, 973 &ring->dev_base, GFP_KERNEL); 974 if (!ring->base) { 975 qdma_err(qdev, "Failed to alloc intr ring %d", i); 976 return -ENOMEM; 977 } 978 intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); 979 intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; 980 intr_ctxt.vec = ring->msix_id; 981 intr_ctxt.valid = true; 982 intr_ctxt.color = true; 983 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, 984 QDMA_CTXT_CLEAR, ring->ridx, NULL); 985 if (ret) { 986 qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); 987 return ret; 988 } 989 990 qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); 991 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, 992 QDMA_CTXT_WRITE, ring->ridx, ctxt); 993 if (ret) { 994 qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); 995 return ret; 996 } 997 998 ret = devm_request_threaded_irq(dev, vector, NULL, 999 qdma_queue_isr, IRQF_ONESHOT, 1000 "amd-qdma-queue", ring); 1001 if (ret) { 1002 qdma_err(qdev, "Failed to request irq %d", vector); 1003 return ret; 1004 } 1005 } 1006 1007 return 0; 1008 } 1009 1010 static int qdma_intr_init(struct qdma_device *qdev) 1011 { 1012 int ret; 1013 1014 ret = qdma_init_error_irq(qdev); 1015 if (ret) { 1016 qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); 1017 return ret; 1018 } 1019 1020 ret = qdmam_alloc_qintr_rings(qdev); 1021 if (ret) { 1022 qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); 1023 return ret; 1024 } 1025 1026 return 0; 1027 } 1028 1029 static void amd_qdma_remove(struct platform_device *pdev) 1030 { 1031 struct qdma_device *qdev = platform_get_drvdata(pdev); 1032 1033 qdma_sgdma_control(qdev, 0); 1034 dma_async_device_unregister(&qdev->dma_dev); 1035 1036 mutex_destroy(&qdev->ctxt_lock); 1037 } 1038 1039 static int amd_qdma_probe(struct platform_device *pdev) 1040 { 1041 struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); 1042 struct qdma_device *qdev; 1043 struct resource *res; 1044 void __iomem *regs; 1045 int ret; 1046 1047 qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); 1048 if (!qdev) 1049 return -ENOMEM; 1050 1051 platform_set_drvdata(pdev, qdev); 1052 qdev->pdev = pdev; 1053 mutex_init(&qdev->ctxt_lock); 1054 1055 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1056 if (!res) { 1057 qdma_err(qdev, "Failed to get IRQ resource"); 1058 ret = -ENODEV; 1059 goto failed; 1060 } 1061 qdev->err_irq_idx = pdata->irq_index; 1062 qdev->queue_irq_start = res->start + 1; 1063 qdev->queue_irq_num = resource_size(res) - 1; 1064 1065 regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); 1066 if (IS_ERR(regs)) { 1067 ret = PTR_ERR(regs); 1068 qdma_err(qdev, "Failed to map IO resource, err %d", ret); 1069 goto failed; 1070 } 1071 1072 qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, 1073 &qdma_regmap_config); 1074 if (IS_ERR(qdev->regmap)) { 1075 ret = PTR_ERR(qdev->regmap); 1076 qdma_err(qdev, "Regmap init failed, err %d", ret); 1077 goto failed; 1078 } 1079 1080 ret = qdma_device_verify(qdev); 1081 if (ret) 1082 goto failed; 1083 1084 ret = qdma_get_hw_info(qdev); 1085 if (ret) 1086 goto failed; 1087 1088 INIT_LIST_HEAD(&qdev->dma_dev.channels); 1089 1090 ret = qdma_device_setup(qdev); 1091 if (ret) 1092 goto failed; 1093 1094 ret = qdma_intr_init(qdev); 1095 if (ret) { 1096 qdma_err(qdev, "Failed to initialize IRQs %d", ret); 1097 goto failed_disable_engine; 1098 } 1099 1100 dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); 1101 dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); 1102 1103 qdev->dma_dev.dev = &pdev->dev; 1104 qdev->dma_dev.filter.map = pdata->device_map; 1105 qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; 1106 qdev->dma_dev.filter.fn = qdma_filter_fn; 1107 qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; 1108 qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; 1109 qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; 1110 qdev->dma_dev.device_config = qdma_device_config; 1111 qdev->dma_dev.device_issue_pending = qdma_issue_pending; 1112 qdev->dma_dev.device_tx_status = dma_cookie_status; 1113 qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1114 1115 ret = dma_async_device_register(&qdev->dma_dev); 1116 if (ret) { 1117 qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); 1118 goto failed_disable_engine; 1119 } 1120 1121 return 0; 1122 1123 failed_disable_engine: 1124 qdma_sgdma_control(qdev, 0); 1125 failed: 1126 mutex_destroy(&qdev->ctxt_lock); 1127 qdma_err(qdev, "Failed to probe AMD QDMA driver"); 1128 return ret; 1129 } 1130 1131 static struct platform_driver amd_qdma_driver = { 1132 .driver = { 1133 .name = "amd-qdma", 1134 }, 1135 .probe = amd_qdma_probe, 1136 .remove = amd_qdma_remove, 1137 }; 1138 1139 module_platform_driver(amd_qdma_driver); 1140 1141 MODULE_DESCRIPTION("AMD QDMA driver"); 1142 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 1143 MODULE_LICENSE("GPL"); 1144