1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Microchip Switchtec(tm) DMA Controller Driver 4 * Copyright (c) 2025, Kelvin Cao <kelvin.cao@microchip.com> 5 * Copyright (c) 2025, Microchip Corporation 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/dmaengine.h> 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/delay.h> 14 #include <linux/iopoll.h> 15 16 #include "dmaengine.h" 17 18 MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine"); 19 MODULE_LICENSE("GPL"); 20 MODULE_AUTHOR("Kelvin Cao"); 21 22 #define SWITCHTEC_DMAC_CHAN_CTRL_OFFSET 0x1000 23 #define SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET 0x160000 24 25 #define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE 0x1000 26 #define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE 0x80 27 28 #define SWITCHTEC_REG_CAP 0x80 29 #define SWITCHTEC_REG_CHAN_CNT 0x84 30 #define SWITCHTEC_REG_TAG_LIMIT 0x90 31 #define SWITCHTEC_REG_CHAN_STS_VEC 0x94 32 #define SWITCHTEC_REG_SE_BUF_CNT 0x98 33 #define SWITCHTEC_REG_SE_BUF_BASE 0x9a 34 35 #define SWITCHTEC_DESC_MAX_SIZE 0x100000 36 37 #define SWITCHTEC_CHAN_CTRL_PAUSE BIT(0) 38 #define SWITCHTEC_CHAN_CTRL_HALT BIT(1) 39 #define SWITCHTEC_CHAN_CTRL_RESET BIT(2) 40 #define SWITCHTEC_CHAN_CTRL_ERR_PAUSE BIT(3) 41 42 #define SWITCHTEC_CHAN_STS_PAUSED BIT(9) 43 #define SWITCHTEC_CHAN_STS_HALTED BIT(10) 44 #define SWITCHTEC_CHAN_STS_PAUSED_MASK GENMASK(29, 13) 45 46 #define SWITCHTEC_INVALID_HFID 0xffff 47 48 #define SWITCHTEC_DMA_SQ_SIZE SZ_32K 49 #define SWITCHTEC_DMA_CQ_SIZE SZ_32K 50 51 #define SWITCHTEC_DMA_RING_SIZE SZ_32K 52 53 static const char * const channel_status_str[] = { 54 [13] = "received a VDM with length error status", 55 [14] = "received a VDM or Cpl with Unsupported Request error status", 56 [15] = "received a VDM or Cpl with Completion Abort error status", 57 [16] = "received a VDM with ECRC error status", 58 [17] = "received a VDM with EP error status", 59 [18] = "received a VDM with Reserved Cpl error status", 60 [19] = "received only part of split SE CplD", 61 [20] = "the ISP_DMAC detected a Completion Time Out", 62 [21] = "received a Cpl with Unsupported Request status", 63 [22] = "received a Cpl with Completion Abort status", 64 [23] = "received a Cpl with a reserved status", 65 [24] = "received a TLP with ECRC error status in its metadata", 66 [25] = "received a TLP with the EP bit set in the header", 67 [26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID", 68 [27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt", 69 [28] = "a reserved opcode was detected in an SE", 70 [29] = "received a SE Cpl with error status", 71 }; 72 73 struct chan_hw_regs { 74 u16 cq_head; 75 u16 rsvd1; 76 u16 sq_tail; 77 u16 rsvd2; 78 u8 ctrl; 79 u8 rsvd3[3]; 80 u16 status; 81 u16 rsvd4; 82 }; 83 84 #define PERF_BURST_SCALE_MASK GENMASK_U32(3, 2) 85 #define PERF_MRRS_MASK GENMASK_U32(6, 4) 86 #define PERF_INTERVAL_MASK GENMASK_U32(10, 8) 87 #define PERF_BURST_SIZE_MASK GENMASK_U32(14, 12) 88 #define PERF_ARB_WEIGHT_MASK GENMASK_U32(31, 24) 89 90 #define SE_BUF_BASE_MASK GENMASK_U32(10, 2) 91 #define SE_BUF_LEN_MASK GENMASK_U32(20, 12) 92 #define SE_THRESH_MASK GENMASK_U32(31, 23) 93 94 #define SWITCHTEC_CHAN_ENABLE BIT(1) 95 96 struct chan_fw_regs { 97 u32 valid_en_se; 98 u32 cq_base_lo; 99 u32 cq_base_hi; 100 u16 cq_size; 101 u16 rsvd1; 102 u32 sq_base_lo; 103 u32 sq_base_hi; 104 u16 sq_size; 105 u16 rsvd2; 106 u32 int_vec; 107 u32 perf_cfg; 108 u32 rsvd3; 109 u32 perf_latency_selector; 110 u32 perf_fetched_se_cnt_lo; 111 u32 perf_fetched_se_cnt_hi; 112 u32 perf_byte_cnt_lo; 113 u32 perf_byte_cnt_hi; 114 u32 rsvd4; 115 u16 perf_se_pending; 116 u16 perf_se_buf_empty; 117 u32 perf_chan_idle; 118 u32 perf_lat_max; 119 u32 perf_lat_min; 120 u32 perf_lat_last; 121 u16 sq_current; 122 u16 sq_phase; 123 u16 cq_current; 124 u16 cq_phase; 125 }; 126 127 struct switchtec_dma_chan { 128 struct switchtec_dma_dev *swdma_dev; 129 struct dma_chan dma_chan; 130 struct chan_hw_regs __iomem *mmio_chan_hw; 131 struct chan_fw_regs __iomem *mmio_chan_fw; 132 133 /* Serialize hardware control register access */ 134 spinlock_t hw_ctrl_lock; 135 136 struct tasklet_struct desc_task; 137 138 /* Serialize descriptor preparation */ 139 spinlock_t submit_lock; 140 bool ring_active; 141 int cid; 142 143 /* Serialize completion processing */ 144 spinlock_t complete_lock; 145 bool comp_ring_active; 146 147 /* channel index and irq */ 148 int index; 149 int irq; 150 151 /* 152 * In driver context, head is advanced by producer while 153 * tail is advanced by consumer. 154 */ 155 156 /* the head and tail for both desc_ring and hw_sq */ 157 int head; 158 int tail; 159 int phase_tag; 160 struct switchtec_dma_hw_se_desc *hw_sq; 161 dma_addr_t dma_addr_sq; 162 163 /* the tail for hw_cq */ 164 int cq_tail; 165 struct switchtec_dma_hw_ce *hw_cq; 166 dma_addr_t dma_addr_cq; 167 168 struct list_head list; 169 170 struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE]; 171 }; 172 173 struct switchtec_dma_dev { 174 struct dma_device dma_dev; 175 struct pci_dev __rcu *pdev; 176 void __iomem *bar; 177 178 struct switchtec_dma_chan **swdma_chans; 179 int chan_cnt; 180 int chan_status_irq; 181 }; 182 183 enum chan_op { 184 ENABLE_CHAN, 185 DISABLE_CHAN, 186 }; 187 188 enum switchtec_dma_opcode { 189 SWITCHTEC_DMA_OPC_MEMCPY = 0, 190 SWITCHTEC_DMA_OPC_RDIMM = 0x1, 191 SWITCHTEC_DMA_OPC_WRIMM = 0x2, 192 SWITCHTEC_DMA_OPC_RHI = 0x6, 193 SWITCHTEC_DMA_OPC_NOP = 0x7, 194 }; 195 196 struct switchtec_dma_hw_se_desc { 197 u8 opc; 198 u8 ctrl; 199 __le16 tlp_setting; 200 __le16 rsvd1; 201 __le16 cid; 202 __le32 byte_cnt; 203 __le32 addr_lo; /* SADDR_LO/WIADDR_LO */ 204 __le32 addr_hi; /* SADDR_HI/WIADDR_HI */ 205 __le32 daddr_lo; 206 __le32 daddr_hi; 207 __le16 dfid; 208 __le16 sfid; 209 }; 210 211 #define SWITCHTEC_SE_DFM BIT(5) 212 #define SWITCHTEC_SE_LIOF BIT(6) 213 #define SWITCHTEC_SE_BRR BIT(7) 214 #define SWITCHTEC_SE_CID_MASK GENMASK(15, 0) 215 216 #define SWITCHTEC_CE_SC_LEN_ERR BIT(0) 217 #define SWITCHTEC_CE_SC_UR BIT(1) 218 #define SWITCHTEC_CE_SC_CA BIT(2) 219 #define SWITCHTEC_CE_SC_RSVD_CPL BIT(3) 220 #define SWITCHTEC_CE_SC_ECRC_ERR BIT(4) 221 #define SWITCHTEC_CE_SC_EP_SET BIT(5) 222 #define SWITCHTEC_CE_SC_D_RD_CTO BIT(8) 223 #define SWITCHTEC_CE_SC_D_RIMM_UR BIT(9) 224 #define SWITCHTEC_CE_SC_D_RIMM_CA BIT(10) 225 #define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL BIT(11) 226 #define SWITCHTEC_CE_SC_D_ECRC BIT(12) 227 #define SWITCHTEC_CE_SC_D_EP_SET BIT(13) 228 #define SWITCHTEC_CE_SC_D_BAD_CONNID BIT(14) 229 #define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR BIT(15) 230 #define SWITCHTEC_CE_SC_D_INVD_CMD BIT(16) 231 #define SWITCHTEC_CE_SC_MASK GENMASK(16, 0) 232 233 struct switchtec_dma_hw_ce { 234 __le32 rdimm_cpl_dw0; 235 __le32 rdimm_cpl_dw1; 236 __le32 rsvd1; 237 __le32 cpl_byte_cnt; 238 __le16 sq_head; 239 __le16 rsvd2; 240 __le32 rsvd3; 241 __le32 sts_code; 242 __le16 cid; 243 __le16 phase_tag; 244 }; 245 246 struct switchtec_dma_desc { 247 struct dma_async_tx_descriptor txd; 248 struct switchtec_dma_hw_se_desc *hw; 249 u32 orig_size; 250 bool completed; 251 }; 252 253 static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask, 254 bool set) 255 { 256 u32 status; 257 258 return readl_poll_timeout_atomic(&chan_hw->status, status, 259 (set && (status & mask)) || 260 (!set && !(status & mask)), 261 10, 100 * USEC_PER_MSEC); 262 } 263 264 static int halt_channel(struct switchtec_dma_chan *swdma_chan) 265 { 266 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 267 struct pci_dev *pdev; 268 int ret; 269 270 rcu_read_lock(); 271 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 272 if (!pdev) { 273 ret = -ENODEV; 274 goto unlock_and_exit; 275 } 276 277 spin_lock(&swdma_chan->hw_ctrl_lock); 278 writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl); 279 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true); 280 spin_unlock(&swdma_chan->hw_ctrl_lock); 281 282 unlock_and_exit: 283 rcu_read_unlock(); 284 return ret; 285 } 286 287 static int unhalt_channel(struct switchtec_dma_chan *swdma_chan) 288 { 289 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 290 struct pci_dev *pdev; 291 u8 ctrl; 292 int ret; 293 294 rcu_read_lock(); 295 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 296 if (!pdev) { 297 ret = -ENODEV; 298 goto unlock_and_exit; 299 } 300 301 spin_lock(&swdma_chan->hw_ctrl_lock); 302 ctrl = readb(&chan_hw->ctrl); 303 ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT; 304 writeb(ctrl, &chan_hw->ctrl); 305 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false); 306 spin_unlock(&swdma_chan->hw_ctrl_lock); 307 308 unlock_and_exit: 309 rcu_read_unlock(); 310 return ret; 311 } 312 313 static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw) 314 { 315 readl(&chan_hw->cq_head); 316 } 317 318 static int reset_channel(struct switchtec_dma_chan *swdma_chan) 319 { 320 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 321 struct pci_dev *pdev; 322 323 rcu_read_lock(); 324 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 325 if (!pdev) { 326 rcu_read_unlock(); 327 return -ENODEV; 328 } 329 330 spin_lock(&swdma_chan->hw_ctrl_lock); 331 writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE, 332 &chan_hw->ctrl); 333 flush_pci_write(chan_hw); 334 335 udelay(1000); 336 337 writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl); 338 spin_unlock(&swdma_chan->hw_ctrl_lock); 339 flush_pci_write(chan_hw); 340 341 rcu_read_unlock(); 342 return 0; 343 } 344 345 static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan) 346 { 347 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 348 struct pci_dev *pdev; 349 350 rcu_read_lock(); 351 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 352 if (!pdev) { 353 rcu_read_unlock(); 354 return -ENODEV; 355 } 356 357 spin_lock(&swdma_chan->hw_ctrl_lock); 358 writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); 359 spin_unlock(&swdma_chan->hw_ctrl_lock); 360 361 flush_pci_write(chan_hw); 362 363 rcu_read_unlock(); 364 365 /* wait 60ms to ensure no pending CEs */ 366 mdelay(60); 367 368 return reset_channel(swdma_chan); 369 } 370 371 static int channel_op(struct switchtec_dma_chan *swdma_chan, int op) 372 { 373 struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; 374 struct pci_dev *pdev; 375 u32 valid_en_se; 376 377 rcu_read_lock(); 378 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 379 if (!pdev) { 380 rcu_read_unlock(); 381 return -ENODEV; 382 } 383 384 valid_en_se = readl(&chan_fw->valid_en_se); 385 if (op == ENABLE_CHAN) 386 valid_en_se |= SWITCHTEC_CHAN_ENABLE; 387 else 388 valid_en_se &= ~SWITCHTEC_CHAN_ENABLE; 389 390 writel(valid_en_se, &chan_fw->valid_en_se); 391 392 rcu_read_unlock(); 393 return 0; 394 } 395 396 static int enable_channel(struct switchtec_dma_chan *swdma_chan) 397 { 398 return channel_op(swdma_chan, ENABLE_CHAN); 399 } 400 401 static int disable_channel(struct switchtec_dma_chan *swdma_chan) 402 { 403 return channel_op(swdma_chan, DISABLE_CHAN); 404 } 405 406 static void 407 switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan) 408 { 409 struct device *chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan); 410 struct switchtec_dma_desc *desc; 411 struct switchtec_dma_hw_ce *ce; 412 struct dmaengine_result res; 413 int tail, cid, se_idx, i; 414 __le16 phase_tag; 415 u32 sts_code; 416 __le32 *p; 417 418 do { 419 spin_lock_bh(&swdma_chan->complete_lock); 420 if (!swdma_chan->comp_ring_active) { 421 spin_unlock_bh(&swdma_chan->complete_lock); 422 break; 423 } 424 425 ce = &swdma_chan->hw_cq[swdma_chan->cq_tail]; 426 /* 427 * phase_tag is updated by hardware, ensure the value is 428 * not from the cache 429 */ 430 phase_tag = smp_load_acquire(&ce->phase_tag); 431 if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) { 432 spin_unlock_bh(&swdma_chan->complete_lock); 433 break; 434 } 435 436 cid = le16_to_cpu(ce->cid); 437 se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1); 438 desc = swdma_chan->desc_ring[se_idx]; 439 440 tail = swdma_chan->tail; 441 442 res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt); 443 444 sts_code = le32_to_cpu(ce->sts_code); 445 446 if (!(sts_code & SWITCHTEC_CE_SC_MASK)) { 447 res.result = DMA_TRANS_NOERROR; 448 } else { 449 if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO) 450 res.result = DMA_TRANS_READ_FAILED; 451 else 452 res.result = DMA_TRANS_WRITE_FAILED; 453 454 dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid, 455 (u32)(sts_code & SWITCHTEC_CE_SC_MASK)); 456 457 p = (__le32 *)ce; 458 for (i = 0; i < sizeof(*ce) / 4; i++) { 459 dev_err(chan_dev, "CE DW%d: 0x%08x\n", i, 460 le32_to_cpu(*p)); 461 p++; 462 } 463 } 464 465 desc->completed = true; 466 467 swdma_chan->cq_tail++; 468 swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1; 469 470 rcu_read_lock(); 471 if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { 472 rcu_read_unlock(); 473 spin_unlock_bh(&swdma_chan->complete_lock); 474 return; 475 } 476 writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head); 477 rcu_read_unlock(); 478 479 if (swdma_chan->cq_tail == 0) 480 swdma_chan->phase_tag = !swdma_chan->phase_tag; 481 482 /* Out of order CE */ 483 if (se_idx != tail) { 484 spin_unlock_bh(&swdma_chan->complete_lock); 485 continue; 486 } 487 488 do { 489 dma_cookie_complete(&desc->txd); 490 dma_descriptor_unmap(&desc->txd); 491 dmaengine_desc_get_callback_invoke(&desc->txd, &res); 492 desc->txd.callback = NULL; 493 desc->txd.callback_result = NULL; 494 desc->completed = false; 495 496 tail++; 497 tail &= SWITCHTEC_DMA_SQ_SIZE - 1; 498 499 /* 500 * Ensure the desc updates are visible before updating 501 * the tail index 502 */ 503 smp_store_release(&swdma_chan->tail, tail); 504 desc = swdma_chan->desc_ring[swdma_chan->tail]; 505 if (!desc->completed) 506 break; 507 } while (CIRC_CNT(READ_ONCE(swdma_chan->head), swdma_chan->tail, 508 SWITCHTEC_DMA_SQ_SIZE)); 509 510 spin_unlock_bh(&swdma_chan->complete_lock); 511 } while (1); 512 } 513 514 static void 515 switchtec_dma_abort_desc(struct switchtec_dma_chan *swdma_chan, int force) 516 { 517 struct switchtec_dma_desc *desc; 518 struct dmaengine_result res; 519 520 if (!force) 521 switchtec_dma_cleanup_completed(swdma_chan); 522 523 spin_lock_bh(&swdma_chan->complete_lock); 524 525 while (CIRC_CNT(swdma_chan->head, swdma_chan->tail, 526 SWITCHTEC_DMA_SQ_SIZE) >= 1) { 527 desc = swdma_chan->desc_ring[swdma_chan->tail]; 528 529 res.residue = desc->orig_size; 530 res.result = DMA_TRANS_ABORTED; 531 532 dma_cookie_complete(&desc->txd); 533 dma_descriptor_unmap(&desc->txd); 534 if (!force) 535 dmaengine_desc_get_callback_invoke(&desc->txd, &res); 536 desc->txd.callback = NULL; 537 desc->txd.callback_result = NULL; 538 539 swdma_chan->tail++; 540 swdma_chan->tail &= SWITCHTEC_DMA_SQ_SIZE - 1; 541 } 542 543 spin_unlock_bh(&swdma_chan->complete_lock); 544 } 545 546 static void switchtec_dma_chan_stop(struct switchtec_dma_chan *swdma_chan) 547 { 548 int rc; 549 550 rc = halt_channel(swdma_chan); 551 if (rc) 552 return; 553 554 rcu_read_lock(); 555 if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { 556 rcu_read_unlock(); 557 return; 558 } 559 560 writel(0, &swdma_chan->mmio_chan_fw->sq_base_lo); 561 writel(0, &swdma_chan->mmio_chan_fw->sq_base_hi); 562 writel(0, &swdma_chan->mmio_chan_fw->cq_base_lo); 563 writel(0, &swdma_chan->mmio_chan_fw->cq_base_hi); 564 565 rcu_read_unlock(); 566 } 567 568 static int switchtec_dma_terminate_all(struct dma_chan *chan) 569 { 570 struct switchtec_dma_chan *swdma_chan = 571 container_of(chan, struct switchtec_dma_chan, dma_chan); 572 573 spin_lock_bh(&swdma_chan->complete_lock); 574 swdma_chan->comp_ring_active = false; 575 spin_unlock_bh(&swdma_chan->complete_lock); 576 577 return pause_reset_channel(swdma_chan); 578 } 579 580 static void switchtec_dma_synchronize(struct dma_chan *chan) 581 { 582 struct switchtec_dma_chan *swdma_chan = 583 container_of(chan, struct switchtec_dma_chan, dma_chan); 584 585 int rc; 586 587 switchtec_dma_abort_desc(swdma_chan, 1); 588 589 rc = enable_channel(swdma_chan); 590 if (rc) 591 return; 592 593 rc = reset_channel(swdma_chan); 594 if (rc) 595 return; 596 597 rc = unhalt_channel(swdma_chan); 598 if (rc) 599 return; 600 601 spin_lock_bh(&swdma_chan->submit_lock); 602 swdma_chan->head = 0; 603 spin_unlock_bh(&swdma_chan->submit_lock); 604 605 spin_lock_bh(&swdma_chan->complete_lock); 606 swdma_chan->comp_ring_active = true; 607 swdma_chan->phase_tag = 0; 608 swdma_chan->tail = 0; 609 swdma_chan->cq_tail = 0; 610 swdma_chan->cid = 0; 611 dma_cookie_init(chan); 612 spin_unlock_bh(&swdma_chan->complete_lock); 613 } 614 615 static struct dma_async_tx_descriptor * 616 switchtec_dma_prep_desc(struct dma_chan *c, u16 dst_fid, dma_addr_t dma_dst, 617 u16 src_fid, dma_addr_t dma_src, u64 data, 618 size_t len, unsigned long flags) 619 __acquires(swdma_chan->submit_lock) 620 { 621 struct switchtec_dma_chan *swdma_chan = 622 container_of(c, struct switchtec_dma_chan, dma_chan); 623 struct switchtec_dma_desc *desc; 624 int head, tail; 625 626 spin_lock_bh(&swdma_chan->submit_lock); 627 628 if (!swdma_chan->ring_active) 629 goto err_unlock; 630 631 tail = READ_ONCE(swdma_chan->tail); 632 head = swdma_chan->head; 633 634 if (!CIRC_SPACE(head, tail, SWITCHTEC_DMA_RING_SIZE)) 635 goto err_unlock; 636 637 desc = swdma_chan->desc_ring[head]; 638 639 if (src_fid != SWITCHTEC_INVALID_HFID && 640 dst_fid != SWITCHTEC_INVALID_HFID) 641 desc->hw->ctrl |= SWITCHTEC_SE_DFM; 642 643 if (flags & DMA_PREP_INTERRUPT) 644 desc->hw->ctrl |= SWITCHTEC_SE_LIOF; 645 646 if (flags & DMA_PREP_FENCE) 647 desc->hw->ctrl |= SWITCHTEC_SE_BRR; 648 649 desc->txd.flags = flags; 650 651 desc->completed = false; 652 desc->hw->opc = SWITCHTEC_DMA_OPC_MEMCPY; 653 desc->hw->addr_lo = cpu_to_le32(lower_32_bits(dma_src)); 654 desc->hw->addr_hi = cpu_to_le32(upper_32_bits(dma_src)); 655 desc->hw->daddr_lo = cpu_to_le32(lower_32_bits(dma_dst)); 656 desc->hw->daddr_hi = cpu_to_le32(upper_32_bits(dma_dst)); 657 desc->hw->byte_cnt = cpu_to_le32(len); 658 desc->hw->tlp_setting = 0; 659 desc->hw->dfid = cpu_to_le16(dst_fid); 660 desc->hw->sfid = cpu_to_le16(src_fid); 661 swdma_chan->cid &= SWITCHTEC_SE_CID_MASK; 662 desc->hw->cid = cpu_to_le16(swdma_chan->cid++); 663 desc->orig_size = len; 664 665 /* return with the lock held, it will be released in tx_submit */ 666 667 return &desc->txd; 668 669 err_unlock: 670 /* 671 * Keep sparse happy by restoring an even lock count on 672 * this lock. 673 */ 674 __acquire(swdma_chan->submit_lock); 675 676 spin_unlock_bh(&swdma_chan->submit_lock); 677 return NULL; 678 } 679 680 static struct dma_async_tx_descriptor * 681 switchtec_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dst, 682 dma_addr_t dma_src, size_t len, unsigned long flags) 683 __acquires(swdma_chan->submit_lock) 684 { 685 if (len > SWITCHTEC_DESC_MAX_SIZE) { 686 /* 687 * Keep sparse happy by restoring an even lock count on 688 * this lock. 689 */ 690 __acquire(swdma_chan->submit_lock); 691 return NULL; 692 } 693 694 return switchtec_dma_prep_desc(c, SWITCHTEC_INVALID_HFID, dma_dst, 695 SWITCHTEC_INVALID_HFID, dma_src, 0, len, 696 flags); 697 } 698 699 static dma_cookie_t 700 switchtec_dma_tx_submit(struct dma_async_tx_descriptor *desc) 701 __releases(swdma_chan->submit_lock) 702 { 703 struct switchtec_dma_chan *swdma_chan = 704 container_of(desc->chan, struct switchtec_dma_chan, dma_chan); 705 dma_cookie_t cookie; 706 int head; 707 708 head = swdma_chan->head + 1; 709 head &= SWITCHTEC_DMA_RING_SIZE - 1; 710 711 /* 712 * Ensure the desc updates are visible before updating the head index 713 */ 714 smp_store_release(&swdma_chan->head, head); 715 716 cookie = dma_cookie_assign(desc); 717 718 spin_unlock_bh(&swdma_chan->submit_lock); 719 720 return cookie; 721 } 722 723 static enum dma_status switchtec_dma_tx_status(struct dma_chan *chan, 724 dma_cookie_t cookie, struct dma_tx_state *txstate) 725 { 726 struct switchtec_dma_chan *swdma_chan = 727 container_of(chan, struct switchtec_dma_chan, dma_chan); 728 enum dma_status ret; 729 730 ret = dma_cookie_status(chan, cookie, txstate); 731 if (ret == DMA_COMPLETE) 732 return ret; 733 734 /* 735 * For jobs where the interrupts are disabled, this is the only place 736 * to process the completions returned by the hardware. Callers that 737 * disable interrupts must call tx_status() to determine when a job 738 * is done, so it is safe to process completions here. If a job has 739 * interrupts enabled, then the completions will normally be processed 740 * in the tasklet that is triggered by the interrupt and tx_status() 741 * does not need to be called. 742 */ 743 switchtec_dma_cleanup_completed(swdma_chan); 744 745 return dma_cookie_status(chan, cookie, txstate); 746 } 747 748 static void switchtec_dma_issue_pending(struct dma_chan *chan) 749 { 750 struct switchtec_dma_chan *swdma_chan = 751 container_of(chan, struct switchtec_dma_chan, dma_chan); 752 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; 753 754 /* 755 * The sq_tail register is actually for the head of the 756 * submisssion queue. Chip has the opposite define of head/tail 757 * to the Linux kernel. 758 */ 759 760 rcu_read_lock(); 761 if (!rcu_dereference(swdma_dev->pdev)) { 762 rcu_read_unlock(); 763 return; 764 } 765 766 spin_lock_bh(&swdma_chan->submit_lock); 767 writew(swdma_chan->head, &swdma_chan->mmio_chan_hw->sq_tail); 768 spin_unlock_bh(&swdma_chan->submit_lock); 769 770 rcu_read_unlock(); 771 } 772 773 static int switchtec_dma_pause(struct dma_chan *chan) 774 { 775 struct switchtec_dma_chan *swdma_chan = 776 container_of(chan, struct switchtec_dma_chan, dma_chan); 777 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 778 struct pci_dev *pdev; 779 int ret; 780 781 rcu_read_lock(); 782 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 783 if (!pdev) { 784 ret = -ENODEV; 785 goto unlock_and_exit; 786 } 787 788 spin_lock(&swdma_chan->hw_ctrl_lock); 789 writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); 790 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, true); 791 spin_unlock(&swdma_chan->hw_ctrl_lock); 792 793 unlock_and_exit: 794 rcu_read_unlock(); 795 return ret; 796 } 797 798 static int switchtec_dma_resume(struct dma_chan *chan) 799 { 800 struct switchtec_dma_chan *swdma_chan = 801 container_of(chan, struct switchtec_dma_chan, dma_chan); 802 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; 803 struct pci_dev *pdev; 804 int ret; 805 806 rcu_read_lock(); 807 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); 808 if (!pdev) { 809 ret = -ENODEV; 810 goto unlock_and_exit; 811 } 812 813 spin_lock(&swdma_chan->hw_ctrl_lock); 814 writeb(0, &chan_hw->ctrl); 815 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, false); 816 spin_unlock(&swdma_chan->hw_ctrl_lock); 817 818 unlock_and_exit: 819 rcu_read_unlock(); 820 return ret; 821 } 822 823 static void switchtec_dma_desc_task(unsigned long data) 824 { 825 struct switchtec_dma_chan *swdma_chan = (void *)data; 826 827 switchtec_dma_cleanup_completed(swdma_chan); 828 } 829 830 static irqreturn_t switchtec_dma_isr(int irq, void *chan) 831 { 832 struct switchtec_dma_chan *swdma_chan = chan; 833 834 if (swdma_chan->comp_ring_active) 835 tasklet_schedule(&swdma_chan->desc_task); 836 837 return IRQ_HANDLED; 838 } 839 840 static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma) 841 { 842 struct switchtec_dma_dev *swdma_dev = dma; 843 struct dma_device *dma_dev = &swdma_dev->dma_dev; 844 struct switchtec_dma_chan *swdma_chan; 845 struct chan_hw_regs __iomem *chan_hw; 846 struct device *chan_dev; 847 struct dma_chan *chan; 848 u32 chan_status; 849 int bit; 850 851 list_for_each_entry(chan, &dma_dev->channels, device_node) { 852 swdma_chan = container_of(chan, struct switchtec_dma_chan, 853 dma_chan); 854 chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan); 855 chan_hw = swdma_chan->mmio_chan_hw; 856 857 rcu_read_lock(); 858 if (!rcu_dereference(swdma_dev->pdev)) { 859 rcu_read_unlock(); 860 goto out; 861 } 862 863 chan_status = readl(&chan_hw->status); 864 chan_status &= SWITCHTEC_CHAN_STS_PAUSED_MASK; 865 rcu_read_unlock(); 866 867 bit = ffs(chan_status); 868 if (!bit) 869 dev_dbg(chan_dev, "No pause bit set.\n"); 870 else 871 dev_err(chan_dev, "Paused, %s\n", 872 channel_status_str[bit - 1]); 873 } 874 875 out: 876 return IRQ_HANDLED; 877 } 878 879 static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan) 880 { 881 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; 882 size_t size; 883 int i; 884 885 size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); 886 if (swdma_chan->hw_sq) 887 dma_free_coherent(swdma_dev->dma_dev.dev, size, 888 swdma_chan->hw_sq, swdma_chan->dma_addr_sq); 889 swdma_chan->hw_sq = NULL; 890 891 size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); 892 if (swdma_chan->hw_cq) 893 dma_free_coherent(swdma_dev->dma_dev.dev, size, 894 swdma_chan->hw_cq, swdma_chan->dma_addr_cq); 895 swdma_chan->hw_cq = NULL; 896 897 for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) { 898 kfree(swdma_chan->desc_ring[i]); 899 swdma_chan->desc_ring[i] = NULL; 900 } 901 } 902 903 static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan) 904 { 905 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; 906 struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; 907 struct switchtec_dma_desc *desc; 908 struct pci_dev *pdev; 909 size_t size; 910 int rc, i; 911 912 swdma_chan->head = 0; 913 swdma_chan->tail = 0; 914 swdma_chan->cq_tail = 0; 915 916 size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); 917 swdma_chan->hw_sq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, 918 &swdma_chan->dma_addr_sq, 919 GFP_NOWAIT); 920 if (!swdma_chan->hw_sq) { 921 rc = -ENOMEM; 922 goto free_and_exit; 923 } 924 925 size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); 926 swdma_chan->hw_cq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, 927 &swdma_chan->dma_addr_cq, 928 GFP_NOWAIT); 929 if (!swdma_chan->hw_cq) { 930 rc = -ENOMEM; 931 goto free_and_exit; 932 } 933 934 /* reset host phase tag */ 935 swdma_chan->phase_tag = 0; 936 937 for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) { 938 desc = kzalloc_obj(*desc, GFP_NOWAIT); 939 if (!desc) { 940 rc = -ENOMEM; 941 goto free_and_exit; 942 } 943 944 dma_async_tx_descriptor_init(&desc->txd, &swdma_chan->dma_chan); 945 desc->txd.tx_submit = switchtec_dma_tx_submit; 946 desc->hw = &swdma_chan->hw_sq[i]; 947 desc->completed = true; 948 949 swdma_chan->desc_ring[i] = desc; 950 } 951 952 rcu_read_lock(); 953 pdev = rcu_dereference(swdma_dev->pdev); 954 if (!pdev) { 955 rcu_read_unlock(); 956 rc = -ENODEV; 957 goto free_and_exit; 958 } 959 960 /* set sq/cq */ 961 writel(lower_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_lo); 962 writel(upper_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_hi); 963 writel(lower_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_lo); 964 writel(upper_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_hi); 965 966 writew(SWITCHTEC_DMA_SQ_SIZE, &swdma_chan->mmio_chan_fw->sq_size); 967 writew(SWITCHTEC_DMA_CQ_SIZE, &swdma_chan->mmio_chan_fw->cq_size); 968 969 rcu_read_unlock(); 970 return 0; 971 972 free_and_exit: 973 switchtec_dma_free_desc(swdma_chan); 974 return rc; 975 } 976 977 static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan) 978 { 979 struct switchtec_dma_chan *swdma_chan = 980 container_of(chan, struct switchtec_dma_chan, dma_chan); 981 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; 982 u32 perf_cfg; 983 int rc; 984 985 rc = switchtec_dma_alloc_desc(swdma_chan); 986 if (rc) 987 return rc; 988 989 rc = enable_channel(swdma_chan); 990 if (rc) 991 goto err_free_desc; 992 993 rc = reset_channel(swdma_chan); 994 if (rc) 995 goto err_disable_channel; 996 997 rc = unhalt_channel(swdma_chan); 998 if (rc) 999 goto err_disable_channel; 1000 1001 swdma_chan->ring_active = true; 1002 swdma_chan->comp_ring_active = true; 1003 swdma_chan->cid = 0; 1004 1005 dma_cookie_init(chan); 1006 1007 rcu_read_lock(); 1008 if (!rcu_dereference(swdma_dev->pdev)) { 1009 rcu_read_unlock(); 1010 rc = -ENODEV; 1011 goto err_ring_inactive; 1012 } 1013 1014 perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg); 1015 rcu_read_unlock(); 1016 1017 dev_dbg(dmaengine_chan_dev(chan), "Burst Size: 0x%x\n", 1018 FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg)); 1019 1020 dev_dbg(dmaengine_chan_dev(chan), "Burst Scale: 0x%x\n", 1021 FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg)); 1022 1023 dev_dbg(dmaengine_chan_dev(chan), "Interval: 0x%x\n", 1024 FIELD_GET(PERF_INTERVAL_MASK, perf_cfg)); 1025 1026 dev_dbg(dmaengine_chan_dev(chan), "Arb Weight: 0x%x\n", 1027 FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg)); 1028 1029 dev_dbg(dmaengine_chan_dev(chan), "MRRS: 0x%x\n", 1030 FIELD_GET(PERF_MRRS_MASK, perf_cfg)); 1031 1032 return SWITCHTEC_DMA_SQ_SIZE; 1033 1034 err_ring_inactive: 1035 spin_lock_bh(&swdma_chan->submit_lock); 1036 swdma_chan->ring_active = false; 1037 spin_unlock_bh(&swdma_chan->submit_lock); 1038 1039 spin_lock_bh(&swdma_chan->complete_lock); 1040 swdma_chan->comp_ring_active = false; 1041 spin_unlock_bh(&swdma_chan->complete_lock); 1042 err_disable_channel: 1043 disable_channel(swdma_chan); 1044 err_free_desc: 1045 switchtec_dma_free_desc(swdma_chan); 1046 return rc; 1047 } 1048 1049 static void switchtec_dma_free_chan_resources(struct dma_chan *chan) 1050 { 1051 struct switchtec_dma_chan *swdma_chan = 1052 container_of(chan, struct switchtec_dma_chan, dma_chan); 1053 1054 spin_lock_bh(&swdma_chan->submit_lock); 1055 swdma_chan->ring_active = false; 1056 spin_unlock_bh(&swdma_chan->submit_lock); 1057 1058 spin_lock_bh(&swdma_chan->complete_lock); 1059 swdma_chan->comp_ring_active = false; 1060 spin_unlock_bh(&swdma_chan->complete_lock); 1061 1062 switchtec_dma_chan_stop(swdma_chan); 1063 switchtec_dma_abort_desc(swdma_chan, 0); 1064 switchtec_dma_free_desc(swdma_chan); 1065 1066 disable_channel(swdma_chan); 1067 } 1068 1069 static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev, 1070 struct pci_dev *pdev, int i) 1071 { 1072 struct dma_device *dma = &swdma_dev->dma_dev; 1073 struct switchtec_dma_chan *swdma_chan; 1074 u32 valid_en_se, thresh; 1075 int se_buf_len, irq, rc; 1076 struct dma_chan *chan; 1077 1078 swdma_chan = kzalloc_obj(*swdma_chan); 1079 if (!swdma_chan) 1080 return -ENOMEM; 1081 1082 swdma_chan->phase_tag = 0; 1083 swdma_chan->index = i; 1084 swdma_chan->swdma_dev = swdma_dev; 1085 1086 spin_lock_init(&swdma_chan->hw_ctrl_lock); 1087 spin_lock_init(&swdma_chan->submit_lock); 1088 spin_lock_init(&swdma_chan->complete_lock); 1089 tasklet_init(&swdma_chan->desc_task, switchtec_dma_desc_task, 1090 (unsigned long)swdma_chan); 1091 1092 swdma_chan->mmio_chan_fw = 1093 swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET + 1094 i * SWITCHTEC_DMA_CHAN_FW_REGS_SIZE; 1095 swdma_chan->mmio_chan_hw = 1096 swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CTRL_OFFSET + 1097 i * SWITCHTEC_DMA_CHAN_HW_REGS_SIZE; 1098 1099 swdma_dev->swdma_chans[i] = swdma_chan; 1100 1101 rc = pause_reset_channel(swdma_chan); 1102 if (rc) 1103 goto free_and_exit; 1104 1105 /* init perf tuner */ 1106 writel(FIELD_PREP(PERF_BURST_SCALE_MASK, 1) | 1107 FIELD_PREP(PERF_MRRS_MASK, 3) | 1108 FIELD_PREP(PERF_BURST_SIZE_MASK, 6) | 1109 FIELD_PREP(PERF_ARB_WEIGHT_MASK, 1), 1110 &swdma_chan->mmio_chan_fw->perf_cfg); 1111 1112 valid_en_se = readl(&swdma_chan->mmio_chan_fw->valid_en_se); 1113 1114 dev_dbg(&pdev->dev, "Channel %d: SE buffer base %d\n", i, 1115 FIELD_GET(SE_BUF_BASE_MASK, valid_en_se)); 1116 1117 se_buf_len = FIELD_GET(SE_BUF_LEN_MASK, valid_en_se); 1118 dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len); 1119 1120 thresh = se_buf_len / 2; 1121 valid_en_se |= FIELD_PREP(SE_THRESH_MASK, thresh); 1122 writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se); 1123 1124 /* request irqs */ 1125 irq = readl(&swdma_chan->mmio_chan_fw->int_vec); 1126 dev_dbg(&pdev->dev, "Channel %d: CE irq vector %d\n", i, irq); 1127 1128 rc = pci_request_irq(pdev, irq, switchtec_dma_isr, NULL, swdma_chan, 1129 KBUILD_MODNAME); 1130 if (rc) 1131 goto free_and_exit; 1132 1133 swdma_chan->irq = irq; 1134 1135 chan = &swdma_chan->dma_chan; 1136 chan->device = dma; 1137 dma_cookie_init(chan); 1138 1139 list_add_tail(&chan->device_node, &dma->channels); 1140 1141 return 0; 1142 1143 free_and_exit: 1144 kfree(swdma_chan); 1145 return rc; 1146 } 1147 1148 static int switchtec_dma_chan_free(struct pci_dev *pdev, 1149 struct switchtec_dma_chan *swdma_chan) 1150 { 1151 spin_lock_bh(&swdma_chan->submit_lock); 1152 swdma_chan->ring_active = false; 1153 spin_unlock_bh(&swdma_chan->submit_lock); 1154 1155 spin_lock_bh(&swdma_chan->complete_lock); 1156 swdma_chan->comp_ring_active = false; 1157 spin_unlock_bh(&swdma_chan->complete_lock); 1158 1159 pci_free_irq(pdev, swdma_chan->irq, swdma_chan); 1160 tasklet_kill(&swdma_chan->desc_task); 1161 1162 switchtec_dma_chan_stop(swdma_chan); 1163 1164 return 0; 1165 } 1166 1167 static void switchtec_dma_chans_release(struct pci_dev *pdev, 1168 struct switchtec_dma_dev *swdma_dev) 1169 { 1170 int i; 1171 1172 for (i = 0; i < swdma_dev->chan_cnt; i++) 1173 switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); 1174 } 1175 1176 static void switchtec_dma_chans_free(struct switchtec_dma_dev *swdma_dev) 1177 { 1178 int i; 1179 1180 for (i = 0; i < swdma_dev->chan_cnt; i++) { 1181 list_del(&swdma_dev->swdma_chans[i]->dma_chan.device_node); 1182 kfree(swdma_dev->swdma_chans[i]); 1183 } 1184 1185 kfree(swdma_dev->swdma_chans); 1186 } 1187 1188 static void switchtec_dma_chans_disable(struct pci_dev *pdev, 1189 struct switchtec_dma_dev *swdma_dev) 1190 { 1191 if (swdma_dev->chan_status_irq >= 0) { 1192 pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); 1193 swdma_dev->chan_status_irq = -1; 1194 } 1195 } 1196 1197 static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, 1198 struct pci_dev *pdev, int chan_cnt) 1199 { 1200 struct dma_device *dma = &swdma_dev->dma_dev; 1201 int base, cnt, rc, i; 1202 1203 swdma_dev->swdma_chans = kzalloc_objs(*swdma_dev->swdma_chans, chan_cnt); 1204 1205 if (!swdma_dev->swdma_chans) 1206 return -ENOMEM; 1207 1208 base = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_BASE); 1209 cnt = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_CNT); 1210 1211 dev_dbg(&pdev->dev, "EP SE buffer base %d\n", base); 1212 dev_dbg(&pdev->dev, "EP SE buffer count %d\n", cnt); 1213 1214 INIT_LIST_HEAD(&dma->channels); 1215 1216 for (i = 0; i < chan_cnt; i++) { 1217 rc = switchtec_dma_chan_init(swdma_dev, pdev, i); 1218 if (rc) { 1219 dev_err(&pdev->dev, "Channel %d: init channel failed\n", 1220 i); 1221 swdma_dev->chan_cnt = i; 1222 goto err_exit; 1223 } 1224 } 1225 1226 return chan_cnt; 1227 1228 err_exit: 1229 switchtec_dma_chans_disable(pdev, swdma_dev); 1230 switchtec_dma_chans_release(pdev, swdma_dev); 1231 switchtec_dma_chans_free(swdma_dev); 1232 1233 return rc; 1234 } 1235 1236 static void switchtec_dma_release(struct dma_device *dma_dev) 1237 { 1238 struct switchtec_dma_dev *swdma_dev = 1239 container_of(dma_dev, struct switchtec_dma_dev, dma_dev); 1240 1241 switchtec_dma_chans_free(swdma_dev); 1242 1243 put_device(dma_dev->dev); 1244 kfree(swdma_dev); 1245 } 1246 1247 static int switchtec_dma_create(struct pci_dev *pdev) 1248 { 1249 struct switchtec_dma_dev *swdma_dev; 1250 int chan_cnt, nr_vecs, irq, rc; 1251 struct dma_device *dma; 1252 struct dma_chan *chan; 1253 1254 /* 1255 * Create the switchtec dma device 1256 */ 1257 swdma_dev = kzalloc_obj(*swdma_dev); 1258 if (!swdma_dev) 1259 return -ENOMEM; 1260 1261 swdma_dev->chan_status_irq = -1; 1262 1263 swdma_dev->bar = ioremap(pci_resource_start(pdev, 0), 1264 pci_resource_len(pdev, 0)); 1265 1266 RCU_INIT_POINTER(swdma_dev->pdev, pdev); 1267 1268 nr_vecs = pci_msix_vec_count(pdev); 1269 rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); 1270 if (rc < 0) 1271 goto err_exit; 1272 1273 irq = readw(swdma_dev->bar + SWITCHTEC_REG_CHAN_STS_VEC); 1274 pci_dbg(pdev, "Channel pause irq vector %d\n", irq); 1275 1276 rc = pci_request_irq(pdev, irq, NULL, switchtec_dma_chan_status_isr, 1277 swdma_dev, KBUILD_MODNAME); 1278 if (rc) 1279 goto err_exit; 1280 1281 swdma_dev->chan_status_irq = irq; 1282 1283 chan_cnt = readl(swdma_dev->bar + SWITCHTEC_REG_CHAN_CNT); 1284 if (!chan_cnt) { 1285 pci_err(pdev, "No channel configured.\n"); 1286 rc = -ENXIO; 1287 goto err_exit; 1288 } 1289 1290 chan_cnt = switchtec_dma_chans_enumerate(swdma_dev, pdev, chan_cnt); 1291 if (chan_cnt < 0) { 1292 pci_err(pdev, "Failed to enumerate dma channels: %d\n", 1293 chan_cnt); 1294 rc = -ENXIO; 1295 goto err_exit; 1296 } 1297 1298 swdma_dev->chan_cnt = chan_cnt; 1299 1300 dma = &swdma_dev->dma_dev; 1301 dma->copy_align = DMAENGINE_ALIGN_8_BYTES; 1302 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 1303 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 1304 dma->dev = get_device(&pdev->dev); 1305 1306 dma->device_alloc_chan_resources = switchtec_dma_alloc_chan_resources; 1307 dma->device_free_chan_resources = switchtec_dma_free_chan_resources; 1308 dma->device_prep_dma_memcpy = switchtec_dma_prep_memcpy; 1309 dma->device_tx_status = switchtec_dma_tx_status; 1310 dma->device_issue_pending = switchtec_dma_issue_pending; 1311 dma->device_pause = switchtec_dma_pause; 1312 dma->device_resume = switchtec_dma_resume; 1313 dma->device_terminate_all = switchtec_dma_terminate_all; 1314 dma->device_synchronize = switchtec_dma_synchronize; 1315 dma->device_release = switchtec_dma_release; 1316 1317 rc = dma_async_device_register(dma); 1318 if (rc) { 1319 pci_err(pdev, "Failed to register dma device: %d\n", rc); 1320 goto err_chans_release_exit; 1321 } 1322 1323 pci_dbg(pdev, "Channel count: %d\n", chan_cnt); 1324 1325 list_for_each_entry(chan, &dma->channels, device_node) 1326 pci_dbg(pdev, "%s\n", dma_chan_name(chan)); 1327 1328 pci_set_drvdata(pdev, swdma_dev); 1329 1330 return 0; 1331 1332 err_chans_release_exit: 1333 switchtec_dma_chans_disable(pdev, swdma_dev); 1334 switchtec_dma_chans_release(pdev, swdma_dev); 1335 switchtec_dma_chans_free(swdma_dev); 1336 1337 err_exit: 1338 if (swdma_dev->chan_status_irq >= 0) 1339 pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); 1340 1341 iounmap(swdma_dev->bar); 1342 kfree(swdma_dev); 1343 return rc; 1344 } 1345 1346 static int switchtec_dma_probe(struct pci_dev *pdev, 1347 const struct pci_device_id *id) 1348 { 1349 int rc; 1350 1351 rc = pci_enable_device(pdev); 1352 if (rc) 1353 return rc; 1354 1355 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1356 1357 rc = pci_request_mem_regions(pdev, KBUILD_MODNAME); 1358 if (rc) 1359 goto err_disable; 1360 1361 pci_set_master(pdev); 1362 1363 rc = switchtec_dma_create(pdev); 1364 if (rc) 1365 goto err_free; 1366 1367 return 0; 1368 1369 err_free: 1370 pci_free_irq_vectors(pdev); 1371 pci_release_mem_regions(pdev); 1372 1373 err_disable: 1374 pci_disable_device(pdev); 1375 1376 return rc; 1377 } 1378 1379 static void switchtec_dma_remove(struct pci_dev *pdev) 1380 { 1381 struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev); 1382 void __iomem *bar = swdma_dev->bar; 1383 1384 switchtec_dma_chans_release(pdev, swdma_dev); 1385 1386 rcu_assign_pointer(swdma_dev->pdev, NULL); 1387 synchronize_rcu(); 1388 1389 pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); 1390 1391 pci_free_irq_vectors(pdev); 1392 1393 dma_async_device_unregister(&swdma_dev->dma_dev); 1394 1395 iounmap(bar); 1396 pci_release_mem_regions(pdev); 1397 pci_disable_device(pdev); 1398 } 1399 1400 /* 1401 * Also use the class code to identify the devices, as some of the 1402 * device IDs are also used for other devices with other classes by 1403 * Microsemi. 1404 */ 1405 #define SW_ID(vendor_id, device_id) \ 1406 { \ 1407 .vendor = vendor_id, \ 1408 .device = device_id, \ 1409 .subvendor = PCI_ANY_ID, \ 1410 .subdevice = PCI_ANY_ID, \ 1411 .class = PCI_CLASS_SYSTEM_OTHER << 8, \ 1412 .class_mask = 0xffffffff, \ 1413 } 1414 1415 static const struct pci_device_id switchtec_dma_pci_tbl[] = { 1416 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4000), /* PFX 100XG4 */ 1417 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4084), /* PFX 84XG4 */ 1418 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4068), /* PFX 68XG4 */ 1419 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4052), /* PFX 52XG4 */ 1420 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4036), /* PFX 36XG4 */ 1421 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4028), /* PFX 28XG4 */ 1422 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4100), /* PSX 100XG4 */ 1423 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4184), /* PSX 84XG4 */ 1424 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4168), /* PSX 68XG4 */ 1425 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4152), /* PSX 52XG4 */ 1426 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4136), /* PSX 36XG4 */ 1427 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4128), /* PSX 28XG4 */ 1428 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4352), /* PFXA 52XG4 */ 1429 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4336), /* PFXA 36XG4 */ 1430 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4328), /* PFXA 28XG4 */ 1431 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4452), /* PSXA 52XG4 */ 1432 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4436), /* PSXA 36XG4 */ 1433 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4428), /* PSXA 28XG4 */ 1434 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5000), /* PFX 100XG5 */ 1435 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5084), /* PFX 84XG5 */ 1436 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5068), /* PFX 68XG5 */ 1437 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5052), /* PFX 52XG5 */ 1438 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5036), /* PFX 36XG5 */ 1439 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5028), /* PFX 28XG5 */ 1440 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5100), /* PSX 100XG5 */ 1441 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5184), /* PSX 84XG5 */ 1442 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5168), /* PSX 68XG5 */ 1443 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5152), /* PSX 52XG5 */ 1444 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5136), /* PSX 36XG5 */ 1445 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5128), /* PSX 28XG5 */ 1446 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5300), /* PFXA 100XG5 */ 1447 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5384), /* PFXA 84XG5 */ 1448 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5368), /* PFXA 68XG5 */ 1449 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5352), /* PFXA 52XG5 */ 1450 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5336), /* PFXA 36XG5 */ 1451 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5328), /* PFXA 28XG5 */ 1452 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5400), /* PSXA 100XG5 */ 1453 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5484), /* PSXA 84XG5 */ 1454 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5468), /* PSXA 68XG5 */ 1455 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5452), /* PSXA 52XG5 */ 1456 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5436), /* PSXA 36XG5 */ 1457 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5428), /* PSXA 28XG5 */ 1458 SW_ID(PCI_VENDOR_ID_EFAR, 0x1001), /* PCI1001 16XG4 */ 1459 SW_ID(PCI_VENDOR_ID_EFAR, 0x1002), /* PCI1002 16XG4 */ 1460 SW_ID(PCI_VENDOR_ID_EFAR, 0x1003), /* PCI1003 16XG4 */ 1461 SW_ID(PCI_VENDOR_ID_EFAR, 0x1004), /* PCI1004 16XG4 */ 1462 SW_ID(PCI_VENDOR_ID_EFAR, 0x1005), /* PCI1005 16XG4 */ 1463 SW_ID(PCI_VENDOR_ID_EFAR, 0x1006), /* PCI1006 16XG4 */ 1464 SW_ID(PCI_VENDOR_ID_EFAR, 0x1008), /* PCI1008 16XG4 */ 1465 {0} 1466 }; 1467 MODULE_DEVICE_TABLE(pci, switchtec_dma_pci_tbl); 1468 1469 static struct pci_driver switchtec_dma_pci_driver = { 1470 .name = KBUILD_MODNAME, 1471 .id_table = switchtec_dma_pci_tbl, 1472 .probe = switchtec_dma_probe, 1473 .remove = switchtec_dma_remove, 1474 }; 1475 module_pci_driver(switchtec_dma_pci_driver); 1476