1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * DMA driver for AMD Queue-based DMA Subsystem 4 * 5 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 6 */ 7 #include <linux/bitfield.h> 8 #include <linux/bitops.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/module.h> 12 #include <linux/platform_device.h> 13 #include <linux/platform_data/amd_qdma.h> 14 #include <linux/regmap.h> 15 16 #include "qdma.h" 17 18 #define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") 19 #define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) 20 21 /* MMIO regmap config for all QDMA registers */ 22 static const struct regmap_config qdma_regmap_config = { 23 .reg_bits = 32, 24 .val_bits = 32, 25 .reg_stride = 4, 26 }; 27 28 static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) 29 { 30 return container_of(chan, struct qdma_queue, vchan.chan); 31 } 32 33 static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) 34 { 35 return container_of(vdesc, struct qdma_mm_vdesc, vdesc); 36 } 37 38 static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) 39 { 40 u32 idx; 41 42 idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; 43 qdev->qintr_ring_idx %= qdev->qintr_ring_num; 44 45 return idx; 46 } 47 48 static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, 49 enum qdma_reg_fields field) 50 { 51 const struct qdma_reg_field *f = &qdev->rfields[field]; 52 u16 low_pos, hi_pos, low_bit, hi_bit; 53 u64 value = 0, mask; 54 55 low_pos = f->lsb / BITS_PER_TYPE(*data); 56 hi_pos = f->msb / BITS_PER_TYPE(*data); 57 58 if (low_pos == hi_pos) { 59 low_bit = f->lsb % BITS_PER_TYPE(*data); 60 hi_bit = f->msb % BITS_PER_TYPE(*data); 61 mask = GENMASK(hi_bit, low_bit); 62 value = (data[low_pos] & mask) >> low_bit; 63 } else if (hi_pos == low_pos + 1) { 64 low_bit = f->lsb % BITS_PER_TYPE(*data); 65 hi_bit = low_bit + (f->msb - f->lsb); 66 value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | 67 data[low_pos]; 68 mask = GENMASK_ULL(hi_bit, low_bit); 69 value = (value & mask) >> low_bit; 70 } else { 71 hi_bit = f->msb % BITS_PER_TYPE(*data); 72 mask = GENMASK(hi_bit, 0); 73 value = data[hi_pos] & mask; 74 low_bit = f->msb - f->lsb - hi_bit; 75 value <<= low_bit; 76 low_bit -= 32; 77 value |= (u64)data[hi_pos - 1] << low_bit; 78 mask = GENMASK(31, 32 - low_bit); 79 value |= (data[hi_pos - 2] & mask) >> low_bit; 80 } 81 82 return value; 83 } 84 85 static void qdma_set_field(const struct qdma_device *qdev, u32 *data, 86 enum qdma_reg_fields field, u64 value) 87 { 88 const struct qdma_reg_field *f = &qdev->rfields[field]; 89 u16 low_pos, hi_pos, low_bit; 90 91 low_pos = f->lsb / BITS_PER_TYPE(*data); 92 hi_pos = f->msb / BITS_PER_TYPE(*data); 93 low_bit = f->lsb % BITS_PER_TYPE(*data); 94 95 data[low_pos++] |= value << low_bit; 96 if (low_pos <= hi_pos) 97 data[low_pos++] |= (u32)(value >> (32 - low_bit)); 98 if (low_pos <= hi_pos) 99 data[low_pos] |= (u32)(value >> (64 - low_bit)); 100 } 101 102 static inline int qdma_reg_write(const struct qdma_device *qdev, 103 const u32 *data, enum qdma_regs reg) 104 { 105 const struct qdma_reg *r = &qdev->roffs[reg]; 106 int ret; 107 108 if (r->count > 1) 109 ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); 110 else 111 ret = regmap_write(qdev->regmap, r->off, *data); 112 113 return ret; 114 } 115 116 static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, 117 enum qdma_regs reg) 118 { 119 const struct qdma_reg *r = &qdev->roffs[reg]; 120 int ret; 121 122 if (r->count > 1) 123 ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); 124 else 125 ret = regmap_read(qdev->regmap, r->off, data); 126 127 return ret; 128 } 129 130 static int qdma_context_cmd_execute(const struct qdma_device *qdev, 131 enum qdma_ctxt_type type, 132 enum qdma_ctxt_cmd cmd, u16 index) 133 { 134 u32 value = 0; 135 int ret; 136 137 qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); 138 qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); 139 qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); 140 141 ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); 142 if (ret) 143 return ret; 144 145 ret = regmap_read_poll_timeout(qdev->regmap, 146 QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), 147 value, 148 !qdma_get_field(qdev, &value, 149 QDMA_REGF_CMD_BUSY), 150 QDMA_POLL_INTRVL_US, 151 QDMA_POLL_TIMEOUT_US); 152 if (ret) { 153 qdma_err(qdev, "Context command execution timed out"); 154 return ret; 155 } 156 157 return 0; 158 } 159 160 static int qdma_context_write_data(const struct qdma_device *qdev, 161 const u32 *data) 162 { 163 u32 mask[QDMA_CTXT_REGMAP_LEN]; 164 int ret; 165 166 memset(mask, ~0, sizeof(mask)); 167 168 ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); 169 if (ret) 170 return ret; 171 172 ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); 173 if (ret) 174 return ret; 175 176 return 0; 177 } 178 179 static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, 180 const struct qdma_ctxt_sw_desc *ctxt, 181 u32 *data) 182 { 183 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 184 qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); 185 qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); 186 qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); 187 188 qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); 189 qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); 190 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); 191 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); 192 qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); 193 qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); 194 qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); 195 qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); 196 qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); 197 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); 198 qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); 199 } 200 201 static void qdma_prep_intr_context(const struct qdma_device *qdev, 202 const struct qdma_ctxt_intr *ctxt, 203 u32 *data) 204 { 205 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 206 qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); 207 qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); 208 qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); 209 qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); 210 qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); 211 qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); 212 } 213 214 static void qdma_prep_fmap_context(const struct qdma_device *qdev, 215 const struct qdma_ctxt_fmap *ctxt, 216 u32 *data) 217 { 218 memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); 219 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); 220 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); 221 } 222 223 /* 224 * Program the indirect context register space 225 * 226 * Once the queue is enabled, context is dynamically updated by hardware. Any 227 * modification of the context through this API when the queue is enabled can 228 * result in unexpected behavior. Reading the context when the queue is enabled 229 * is not recommended as it can result in reduced performance. 230 */ 231 static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, 232 enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) 233 { 234 int ret; 235 236 mutex_lock(&qdev->ctxt_lock); 237 if (cmd == QDMA_CTXT_WRITE) { 238 ret = qdma_context_write_data(qdev, ctxt); 239 if (ret) 240 goto failed; 241 } 242 243 ret = qdma_context_cmd_execute(qdev, type, cmd, index); 244 if (ret) 245 goto failed; 246 247 if (cmd == QDMA_CTXT_READ) { 248 ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); 249 if (ret) 250 goto failed; 251 } 252 253 failed: 254 mutex_unlock(&qdev->ctxt_lock); 255 256 return ret; 257 } 258 259 static int qdma_check_queue_status(struct qdma_device *qdev, 260 enum dma_transfer_direction dir, u16 qid) 261 { 262 u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; 263 enum qdma_ctxt_type type; 264 int ret; 265 266 if (dir == DMA_MEM_TO_DEV) 267 type = QDMA_CTXT_DESC_SW_H2C; 268 else 269 type = QDMA_CTXT_DESC_SW_C2H; 270 ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); 271 if (ret) 272 return ret; 273 274 status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); 275 if (status) { 276 qdma_err(qdev, "queue %d already in use", qid); 277 return -EBUSY; 278 } 279 280 return 0; 281 } 282 283 static int qdma_clear_queue_context(const struct qdma_queue *queue) 284 { 285 static const enum qdma_ctxt_type h2c_types[] = { 286 QDMA_CTXT_DESC_SW_H2C, 287 QDMA_CTXT_DESC_HW_H2C, 288 QDMA_CTXT_DESC_CR_H2C, 289 QDMA_CTXT_PFTCH, 290 }; 291 static const enum qdma_ctxt_type c2h_types[] = { 292 QDMA_CTXT_DESC_SW_C2H, 293 QDMA_CTXT_DESC_HW_C2H, 294 QDMA_CTXT_DESC_CR_C2H, 295 QDMA_CTXT_PFTCH, 296 }; 297 struct qdma_device *qdev = queue->qdev; 298 const enum qdma_ctxt_type *type; 299 int ret, num, i; 300 301 if (queue->dir == DMA_MEM_TO_DEV) { 302 type = h2c_types; 303 num = ARRAY_SIZE(h2c_types); 304 } else { 305 type = c2h_types; 306 num = ARRAY_SIZE(c2h_types); 307 } 308 for (i = 0; i < num; i++) { 309 ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, 310 queue->qid, NULL); 311 if (ret) { 312 qdma_err(qdev, "Failed to clear ctxt %d", type[i]); 313 return ret; 314 } 315 } 316 317 return 0; 318 } 319 320 static int qdma_setup_fmap_context(struct qdma_device *qdev) 321 { 322 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 323 struct qdma_ctxt_fmap fmap; 324 int ret; 325 326 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, 327 qdev->fid, NULL); 328 if (ret) { 329 qdma_err(qdev, "Failed clearing context"); 330 return ret; 331 } 332 333 fmap.qbase = 0; 334 fmap.qmax = qdev->chan_num * 2; 335 qdma_prep_fmap_context(qdev, &fmap, ctxt); 336 ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, 337 qdev->fid, ctxt); 338 if (ret) 339 qdma_err(qdev, "Failed setup fmap, ret %d", ret); 340 341 return ret; 342 } 343 344 static int qdma_setup_queue_context(struct qdma_device *qdev, 345 const struct qdma_ctxt_sw_desc *sw_desc, 346 enum dma_transfer_direction dir, u16 qid) 347 { 348 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 349 enum qdma_ctxt_type type; 350 int ret; 351 352 if (dir == DMA_MEM_TO_DEV) 353 type = QDMA_CTXT_DESC_SW_H2C; 354 else 355 type = QDMA_CTXT_DESC_SW_C2H; 356 357 qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); 358 /* Setup SW descriptor context */ 359 ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); 360 if (ret) 361 qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); 362 363 return ret; 364 } 365 366 /* 367 * Enable or disable memory-mapped DMA engines 368 * 1: enable, 0: disable 369 */ 370 static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) 371 { 372 int ret; 373 374 ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); 375 ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); 376 377 return ret; 378 } 379 380 static int qdma_get_hw_info(struct qdma_device *qdev) 381 { 382 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); 383 u32 value = 0; 384 int ret; 385 386 ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); 387 if (ret) 388 return ret; 389 390 value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; 391 if (pdata->max_mm_channels * 2 > value) { 392 qdma_err(qdev, "not enough hw queues %d", value); 393 return -EINVAL; 394 } 395 qdev->chan_num = pdata->max_mm_channels; 396 397 ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); 398 if (ret) 399 return ret; 400 401 qdma_info(qdev, "max channel %d, function id %d", 402 qdev->chan_num, qdev->fid); 403 404 return 0; 405 } 406 407 static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) 408 { 409 struct qdma_device *qdev = queue->qdev; 410 411 return regmap_write(qdev->regmap, queue->pidx_reg, 412 pidx | QDMA_QUEUE_ARM_BIT); 413 } 414 415 static inline int qdma_update_cidx(const struct qdma_queue *queue, 416 u16 ridx, u16 cidx) 417 { 418 struct qdma_device *qdev = queue->qdev; 419 420 return regmap_write(qdev->regmap, queue->cidx_reg, 421 ((u32)ridx << 16) | cidx); 422 } 423 424 /** 425 * qdma_free_vdesc - Free descriptor 426 * @vdesc: Virtual DMA descriptor 427 */ 428 static void qdma_free_vdesc(struct virt_dma_desc *vdesc) 429 { 430 struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); 431 432 kfree(vd); 433 } 434 435 static int qdma_alloc_queues(struct qdma_device *qdev, 436 enum dma_transfer_direction dir) 437 { 438 struct qdma_queue *q, **queues; 439 u32 i, pidx_base; 440 int ret; 441 442 if (dir == DMA_MEM_TO_DEV) { 443 queues = &qdev->h2c_queues; 444 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); 445 } else { 446 queues = &qdev->c2h_queues; 447 pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); 448 } 449 450 *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), 451 GFP_KERNEL); 452 if (!*queues) 453 return -ENOMEM; 454 455 for (i = 0; i < qdev->chan_num; i++) { 456 ret = qdma_check_queue_status(qdev, dir, i); 457 if (ret) 458 return ret; 459 460 q = &(*queues)[i]; 461 q->ring_size = QDMA_DEFAULT_RING_SIZE; 462 q->idx_mask = q->ring_size - 2; 463 q->qdev = qdev; 464 q->dir = dir; 465 q->qid = i; 466 q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; 467 q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + 468 i * QDMA_DMAP_REG_STRIDE; 469 q->vchan.desc_free = qdma_free_vdesc; 470 vchan_init(&q->vchan, &qdev->dma_dev); 471 } 472 473 return 0; 474 } 475 476 static int qdma_device_verify(struct qdma_device *qdev) 477 { 478 u32 value; 479 int ret; 480 481 ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); 482 if (ret) 483 return ret; 484 485 value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); 486 if (value != QDMA_IDENTIFIER) { 487 qdma_err(qdev, "Invalid identifier"); 488 return -ENODEV; 489 } 490 qdev->rfields = qdma_regfs_default; 491 qdev->roffs = qdma_regos_default; 492 493 return 0; 494 } 495 496 static int qdma_device_setup(struct qdma_device *qdev) 497 { 498 u32 ring_sz = QDMA_DEFAULT_RING_SIZE; 499 int ret = 0; 500 501 ret = qdma_setup_fmap_context(qdev); 502 if (ret) { 503 qdma_err(qdev, "Failed setup fmap context"); 504 return ret; 505 } 506 507 /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ 508 ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); 509 if (ret) { 510 qdma_err(qdev, "Failed to setup ring %d of size %ld", 511 QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); 512 return ret; 513 } 514 515 /* Enable memory-mapped DMA engine in both directions */ 516 ret = qdma_sgdma_control(qdev, 1); 517 if (ret) { 518 qdma_err(qdev, "Failed to SGDMA with error %d", ret); 519 return ret; 520 } 521 522 ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); 523 if (ret) { 524 qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); 525 return ret; 526 } 527 528 ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); 529 if (ret) { 530 qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); 531 return ret; 532 } 533 534 return 0; 535 } 536 537 /** 538 * qdma_free_queue_resources() - Free queue resources 539 * @chan: DMA channel 540 */ 541 static void qdma_free_queue_resources(struct dma_chan *chan) 542 { 543 struct qdma_queue *queue = to_qdma_queue(chan); 544 struct qdma_device *qdev = queue->qdev; 545 struct qdma_platdata *pdata; 546 547 qdma_clear_queue_context(queue); 548 vchan_free_chan_resources(&queue->vchan); 549 pdata = dev_get_platdata(&qdev->pdev->dev); 550 dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, 551 queue->desc_base, queue->dma_desc_base); 552 } 553 554 /** 555 * qdma_alloc_queue_resources() - Allocate queue resources 556 * @chan: DMA channel 557 */ 558 static int qdma_alloc_queue_resources(struct dma_chan *chan) 559 { 560 struct qdma_queue *queue = to_qdma_queue(chan); 561 struct qdma_device *qdev = queue->qdev; 562 struct qdma_ctxt_sw_desc desc; 563 struct qdma_platdata *pdata; 564 size_t size; 565 int ret; 566 567 ret = qdma_clear_queue_context(queue); 568 if (ret) 569 return ret; 570 571 pdata = dev_get_platdata(&qdev->pdev->dev); 572 size = queue->ring_size * QDMA_MM_DESC_SIZE; 573 queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, 574 &queue->dma_desc_base, 575 GFP_KERNEL); 576 if (!queue->desc_base) { 577 qdma_err(qdev, "Failed to allocate descriptor ring"); 578 return -ENOMEM; 579 } 580 581 /* Setup SW descriptor queue context for DMA memory map */ 582 desc.vec = qdma_get_intr_ring_idx(qdev); 583 desc.desc_base = queue->dma_desc_base; 584 ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); 585 if (ret) { 586 qdma_err(qdev, "Failed to setup SW desc ctxt for %s", 587 chan->name); 588 dma_free_coherent(pdata->dma_dev, size, queue->desc_base, 589 queue->dma_desc_base); 590 return ret; 591 } 592 593 queue->pidx = 0; 594 queue->cidx = 0; 595 596 return 0; 597 } 598 599 static bool qdma_filter_fn(struct dma_chan *chan, void *param) 600 { 601 struct qdma_queue *queue = to_qdma_queue(chan); 602 struct qdma_queue_info *info = param; 603 604 return info->dir == queue->dir; 605 } 606 607 static int qdma_xfer_start(struct qdma_queue *queue) 608 { 609 struct qdma_device *qdev = queue->qdev; 610 int ret; 611 612 if (!vchan_next_desc(&queue->vchan)) 613 return 0; 614 615 qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", 616 queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); 617 618 ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); 619 if (ret) { 620 qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", 621 queue->pidx, CHAN_STR(queue), queue->qid); 622 } 623 624 return ret; 625 } 626 627 static void qdma_issue_pending(struct dma_chan *chan) 628 { 629 struct qdma_queue *queue = to_qdma_queue(chan); 630 unsigned long flags; 631 632 spin_lock_irqsave(&queue->vchan.lock, flags); 633 if (vchan_issue_pending(&queue->vchan)) { 634 if (queue->submitted_vdesc) { 635 queue->issued_vdesc = queue->submitted_vdesc; 636 queue->submitted_vdesc = NULL; 637 } 638 qdma_xfer_start(queue); 639 } 640 641 spin_unlock_irqrestore(&queue->vchan.lock, flags); 642 } 643 644 static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) 645 { 646 struct qdma_mm_desc *desc; 647 648 if (((q->pidx + 1) & q->idx_mask) == q->cidx) 649 return NULL; 650 651 desc = q->desc_base + q->pidx; 652 q->pidx = (q->pidx + 1) & q->idx_mask; 653 654 return desc; 655 } 656 657 static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) 658 { 659 struct qdma_mm_desc *desc; 660 struct scatterlist *sg; 661 u64 addr, *src, *dst; 662 u32 rest, len; 663 int ret = 0; 664 u32 i; 665 666 if (!vdesc->sg_len) 667 return 0; 668 669 if (q->dir == DMA_MEM_TO_DEV) { 670 dst = &vdesc->dev_addr; 671 src = &addr; 672 } else { 673 dst = &addr; 674 src = &vdesc->dev_addr; 675 } 676 677 for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { 678 addr = sg_dma_address(sg) + vdesc->sg_off; 679 rest = sg_dma_len(sg) - vdesc->sg_off; 680 while (rest) { 681 len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); 682 desc = qdma_get_desc(q); 683 if (!desc) { 684 ret = -EBUSY; 685 goto out; 686 } 687 688 desc->src_addr = cpu_to_le64(*src); 689 desc->dst_addr = cpu_to_le64(*dst); 690 desc->len = cpu_to_le32(len); 691 692 vdesc->dev_addr += len; 693 vdesc->sg_off += len; 694 vdesc->pending_descs++; 695 addr += len; 696 rest -= len; 697 } 698 vdesc->sg_off = 0; 699 } 700 out: 701 vdesc->sg_len -= i; 702 vdesc->pidx = q->pidx; 703 return ret; 704 } 705 706 static void qdma_fill_pending_vdesc(struct qdma_queue *q) 707 { 708 struct virt_dma_chan *vc = &q->vchan; 709 struct qdma_mm_vdesc *vdesc = NULL; 710 struct virt_dma_desc *vd; 711 int ret; 712 713 if (!list_empty(&vc->desc_issued)) { 714 vd = &q->issued_vdesc->vdesc; 715 list_for_each_entry_from(vd, &vc->desc_issued, node) { 716 vdesc = to_qdma_vdesc(vd); 717 ret = qdma_hw_enqueue(q, vdesc); 718 if (ret) { 719 q->issued_vdesc = vdesc; 720 return; 721 } 722 } 723 q->issued_vdesc = vdesc; 724 } 725 726 if (list_empty(&vc->desc_submitted)) 727 return; 728 729 if (q->submitted_vdesc) 730 vd = &q->submitted_vdesc->vdesc; 731 else 732 vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); 733 734 list_for_each_entry_from(vd, &vc->desc_submitted, node) { 735 vdesc = to_qdma_vdesc(vd); 736 ret = qdma_hw_enqueue(q, vdesc); 737 if (ret) 738 break; 739 } 740 q->submitted_vdesc = vdesc; 741 } 742 743 static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) 744 { 745 struct virt_dma_chan *vc = to_virt_chan(tx->chan); 746 struct qdma_queue *q = to_qdma_queue(&vc->chan); 747 struct virt_dma_desc *vd; 748 unsigned long flags; 749 dma_cookie_t cookie; 750 751 vd = container_of(tx, struct virt_dma_desc, tx); 752 spin_lock_irqsave(&vc->lock, flags); 753 cookie = dma_cookie_assign(tx); 754 755 list_move_tail(&vd->node, &vc->desc_submitted); 756 qdma_fill_pending_vdesc(q); 757 spin_unlock_irqrestore(&vc->lock, flags); 758 759 return cookie; 760 } 761 762 static struct dma_async_tx_descriptor * 763 qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, 764 unsigned int sg_len, enum dma_transfer_direction dir, 765 unsigned long flags, void *context) 766 { 767 struct qdma_queue *q = to_qdma_queue(chan); 768 struct dma_async_tx_descriptor *tx; 769 struct qdma_mm_vdesc *vdesc; 770 771 vdesc = kzalloc_obj(*vdesc, GFP_NOWAIT); 772 if (!vdesc) 773 return NULL; 774 vdesc->sgl = sgl; 775 vdesc->sg_len = sg_len; 776 if (dir == DMA_MEM_TO_DEV) 777 vdesc->dev_addr = q->cfg.dst_addr; 778 else 779 vdesc->dev_addr = q->cfg.src_addr; 780 781 tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); 782 tx->tx_submit = qdma_tx_submit; 783 784 return tx; 785 } 786 787 static int qdma_device_config(struct dma_chan *chan, 788 struct dma_slave_config *cfg) 789 { 790 struct qdma_queue *q = to_qdma_queue(chan); 791 792 memcpy(&q->cfg, cfg, sizeof(*cfg)); 793 794 return 0; 795 } 796 797 static int qdma_arm_err_intr(const struct qdma_device *qdev) 798 { 799 u32 value = 0; 800 801 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); 802 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); 803 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); 804 805 return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); 806 } 807 808 static irqreturn_t qdma_error_isr(int irq, void *data) 809 { 810 struct qdma_device *qdev = data; 811 u32 err_stat = 0; 812 int ret; 813 814 ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); 815 if (ret) { 816 qdma_err(qdev, "read error state failed, ret %d", ret); 817 goto out; 818 } 819 820 qdma_err(qdev, "global error %d", err_stat); 821 ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); 822 if (ret) 823 qdma_err(qdev, "clear error state failed, ret %d", ret); 824 825 out: 826 qdma_arm_err_intr(qdev); 827 return IRQ_HANDLED; 828 } 829 830 static irqreturn_t qdma_queue_isr(int irq, void *data) 831 { 832 struct qdma_intr_ring *intr = data; 833 struct qdma_queue *q = NULL; 834 struct qdma_device *qdev; 835 u32 index, comp_desc; 836 u64 intr_ent; 837 u8 color; 838 int ret; 839 u16 qid; 840 841 qdev = intr->qdev; 842 index = intr->cidx; 843 while (1) { 844 struct virt_dma_desc *vd; 845 struct qdma_mm_vdesc *vdesc; 846 unsigned long flags; 847 u32 cidx; 848 849 intr_ent = le64_to_cpu(intr->base[index]); 850 color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); 851 if (color != intr->color) 852 break; 853 854 qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); 855 if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) 856 q = qdev->c2h_queues; 857 else 858 q = qdev->h2c_queues; 859 q += qid; 860 861 cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); 862 863 spin_lock_irqsave(&q->vchan.lock, flags); 864 comp_desc = (cidx - q->cidx) & q->idx_mask; 865 866 vd = vchan_next_desc(&q->vchan); 867 if (!vd) 868 goto skip; 869 870 vdesc = to_qdma_vdesc(vd); 871 while (comp_desc > vdesc->pending_descs) { 872 list_del(&vd->node); 873 vchan_cookie_complete(vd); 874 comp_desc -= vdesc->pending_descs; 875 vd = vchan_next_desc(&q->vchan); 876 vdesc = to_qdma_vdesc(vd); 877 } 878 vdesc->pending_descs -= comp_desc; 879 if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { 880 list_del(&vd->node); 881 vchan_cookie_complete(vd); 882 } 883 q->cidx = cidx; 884 885 qdma_fill_pending_vdesc(q); 886 qdma_xfer_start(q); 887 888 skip: 889 spin_unlock_irqrestore(&q->vchan.lock, flags); 890 891 /* 892 * Wrap the index value and flip the expected color value if 893 * interrupt aggregation PIDX has wrapped around. 894 */ 895 index++; 896 index &= QDMA_INTR_RING_IDX_MASK; 897 if (!index) 898 intr->color = !intr->color; 899 } 900 901 /* 902 * Update the software interrupt aggregation ring CIDX if a valid entry 903 * was found. 904 */ 905 if (q) { 906 qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); 907 908 /* 909 * Record the last read index of status descriptor from the 910 * interrupt aggregation ring. 911 */ 912 intr->cidx = index; 913 914 ret = qdma_update_cidx(q, intr->ridx, index); 915 if (ret) { 916 qdma_err(qdev, "Failed to update IRQ CIDX"); 917 return IRQ_NONE; 918 } 919 } 920 921 return IRQ_HANDLED; 922 } 923 924 static int qdma_init_error_irq(struct qdma_device *qdev) 925 { 926 struct device *dev = &qdev->pdev->dev; 927 int ret; 928 u32 vec; 929 930 vec = qdev->queue_irq_start - 1; 931 932 ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, 933 IRQF_ONESHOT, "amd-qdma-error", qdev); 934 if (ret) { 935 qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); 936 return ret; 937 } 938 939 ret = qdma_arm_err_intr(qdev); 940 if (ret) 941 qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); 942 943 return ret; 944 } 945 946 static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) 947 { 948 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); 949 struct device *dev = &qdev->pdev->dev; 950 u32 ctxt[QDMA_CTXT_REGMAP_LEN]; 951 struct qdma_intr_ring *ring; 952 struct qdma_ctxt_intr intr_ctxt; 953 u32 vector; 954 int ret, i; 955 956 qdev->qintr_ring_num = qdev->queue_irq_num; 957 qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, 958 sizeof(*qdev->qintr_rings), 959 GFP_KERNEL); 960 if (!qdev->qintr_rings) 961 return -ENOMEM; 962 963 vector = qdev->queue_irq_start; 964 for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { 965 ring = &qdev->qintr_rings[i]; 966 ring->qdev = qdev; 967 ring->msix_id = qdev->err_irq_idx + i + 1; 968 ring->ridx = i; 969 ring->color = 1; 970 ring->base = dmam_alloc_coherent(pdata->dma_dev, 971 QDMA_INTR_RING_SIZE, 972 &ring->dev_base, GFP_KERNEL); 973 if (!ring->base) { 974 qdma_err(qdev, "Failed to alloc intr ring %d", i); 975 return -ENOMEM; 976 } 977 intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); 978 intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; 979 intr_ctxt.vec = ring->msix_id; 980 intr_ctxt.valid = true; 981 intr_ctxt.color = true; 982 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, 983 QDMA_CTXT_CLEAR, ring->ridx, NULL); 984 if (ret) { 985 qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); 986 return ret; 987 } 988 989 qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); 990 ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, 991 QDMA_CTXT_WRITE, ring->ridx, ctxt); 992 if (ret) { 993 qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); 994 return ret; 995 } 996 997 ret = devm_request_threaded_irq(dev, vector, NULL, 998 qdma_queue_isr, IRQF_ONESHOT, 999 "amd-qdma-queue", ring); 1000 if (ret) { 1001 qdma_err(qdev, "Failed to request irq %d", vector); 1002 return ret; 1003 } 1004 } 1005 1006 return 0; 1007 } 1008 1009 static int qdma_intr_init(struct qdma_device *qdev) 1010 { 1011 int ret; 1012 1013 ret = qdma_init_error_irq(qdev); 1014 if (ret) { 1015 qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); 1016 return ret; 1017 } 1018 1019 ret = qdmam_alloc_qintr_rings(qdev); 1020 if (ret) { 1021 qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); 1022 return ret; 1023 } 1024 1025 return 0; 1026 } 1027 1028 static void amd_qdma_remove(struct platform_device *pdev) 1029 { 1030 struct qdma_device *qdev = platform_get_drvdata(pdev); 1031 1032 qdma_sgdma_control(qdev, 0); 1033 dma_async_device_unregister(&qdev->dma_dev); 1034 1035 mutex_destroy(&qdev->ctxt_lock); 1036 } 1037 1038 static int amd_qdma_probe(struct platform_device *pdev) 1039 { 1040 struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); 1041 struct qdma_device *qdev; 1042 struct resource *res; 1043 void __iomem *regs; 1044 int ret; 1045 1046 qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); 1047 if (!qdev) 1048 return -ENOMEM; 1049 1050 platform_set_drvdata(pdev, qdev); 1051 qdev->pdev = pdev; 1052 mutex_init(&qdev->ctxt_lock); 1053 1054 res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1055 if (!res) { 1056 qdma_err(qdev, "Failed to get IRQ resource"); 1057 ret = -ENODEV; 1058 goto failed; 1059 } 1060 qdev->err_irq_idx = pdata->irq_index; 1061 qdev->queue_irq_start = res->start + 1; 1062 qdev->queue_irq_num = resource_size(res) - 1; 1063 1064 regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); 1065 if (IS_ERR(regs)) { 1066 ret = PTR_ERR(regs); 1067 qdma_err(qdev, "Failed to map IO resource, err %d", ret); 1068 goto failed; 1069 } 1070 1071 qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, 1072 &qdma_regmap_config); 1073 if (IS_ERR(qdev->regmap)) { 1074 ret = PTR_ERR(qdev->regmap); 1075 qdma_err(qdev, "Regmap init failed, err %d", ret); 1076 goto failed; 1077 } 1078 1079 ret = qdma_device_verify(qdev); 1080 if (ret) 1081 goto failed; 1082 1083 ret = qdma_get_hw_info(qdev); 1084 if (ret) 1085 goto failed; 1086 1087 INIT_LIST_HEAD(&qdev->dma_dev.channels); 1088 1089 ret = qdma_device_setup(qdev); 1090 if (ret) 1091 goto failed; 1092 1093 ret = qdma_intr_init(qdev); 1094 if (ret) { 1095 qdma_err(qdev, "Failed to initialize IRQs %d", ret); 1096 goto failed_disable_engine; 1097 } 1098 1099 dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); 1100 dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); 1101 1102 qdev->dma_dev.dev = &pdev->dev; 1103 qdev->dma_dev.filter.map = pdata->device_map; 1104 qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; 1105 qdev->dma_dev.filter.fn = qdma_filter_fn; 1106 qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; 1107 qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; 1108 qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; 1109 qdev->dma_dev.device_config = qdma_device_config; 1110 qdev->dma_dev.device_issue_pending = qdma_issue_pending; 1111 qdev->dma_dev.device_tx_status = dma_cookie_status; 1112 qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1113 1114 ret = dma_async_device_register(&qdev->dma_dev); 1115 if (ret) { 1116 qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); 1117 goto failed_disable_engine; 1118 } 1119 1120 return 0; 1121 1122 failed_disable_engine: 1123 qdma_sgdma_control(qdev, 0); 1124 failed: 1125 mutex_destroy(&qdev->ctxt_lock); 1126 qdma_err(qdev, "Failed to probe AMD QDMA driver"); 1127 return ret; 1128 } 1129 1130 static struct platform_driver amd_qdma_driver = { 1131 .driver = { 1132 .name = "amd-qdma", 1133 }, 1134 .probe = amd_qdma_probe, 1135 .remove = amd_qdma_remove, 1136 }; 1137 1138 module_platform_driver(amd_qdma_driver); 1139 1140 MODULE_DESCRIPTION("AMD QDMA driver"); 1141 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 1142 MODULE_LICENSE("GPL"); 1143