1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * DMA driver for Xilinx Video DMA Engine 4 * 5 * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved. 6 * 7 * Based on the Freescale DMA driver. 8 * 9 * Description: 10 * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP 11 * core that provides high-bandwidth direct memory access between memory 12 * and AXI4-Stream type video target peripherals. The core provides efficient 13 * two dimensional DMA operations with independent asynchronous read (S2MM) 14 * and write (MM2S) channel operation. It can be configured to have either 15 * one channel or two channels. If configured as two channels, one is to 16 * transmit to the video device (MM2S) and another is to receive from the 17 * video device (S2MM). Initialization, status, interrupt and management 18 * registers are accessed through an AXI4-Lite slave interface. 19 * 20 * The AXI Direct Memory Access (AXI DMA) core is a soft Xilinx IP core that 21 * provides high-bandwidth one dimensional direct memory access between memory 22 * and AXI4-Stream target peripherals. It supports one receive and one 23 * transmit channel, both of them optional at synthesis time. 24 * 25 * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory 26 * Access (DMA) between a memory-mapped source address and a memory-mapped 27 * destination address. 28 * 29 * The AXI Multichannel Direct Memory Access (AXI MCDMA) core is a soft 30 * Xilinx IP that provides high-bandwidth direct memory access between 31 * memory and AXI4-Stream target peripherals. It provides scatter gather 32 * (SG) interface with multiple channels independent configuration support. 33 * 34 */ 35 36 #include <linux/bitops.h> 37 #include <linux/dmapool.h> 38 #include <linux/dma/xilinx_dma.h> 39 #include <linux/init.h> 40 #include <linux/interrupt.h> 41 #include <linux/io.h> 42 #include <linux/iopoll.h> 43 #include <linux/module.h> 44 #include <linux/of.h> 45 #include <linux/of_dma.h> 46 #include <linux/of_irq.h> 47 #include <linux/platform_device.h> 48 #include <linux/slab.h> 49 #include <linux/string_choices.h> 50 #include <linux/clk.h> 51 #include <linux/io-64-nonatomic-lo-hi.h> 52 53 #include "../dmaengine.h" 54 55 /* Register/Descriptor Offsets */ 56 #define XILINX_DMA_MM2S_CTRL_OFFSET 0x0000 57 #define XILINX_DMA_S2MM_CTRL_OFFSET 0x0030 58 #define XILINX_VDMA_MM2S_DESC_OFFSET 0x0050 59 #define XILINX_VDMA_S2MM_DESC_OFFSET 0x00a0 60 61 /* Control Registers */ 62 #define XILINX_DMA_REG_DMACR 0x0000 63 #define XILINX_DMA_DMACR_DELAY_MAX 0xff 64 #define XILINX_DMA_DMACR_DELAY_SHIFT 24 65 #define XILINX_DMA_DMACR_FRAME_COUNT_MAX 0xff 66 #define XILINX_DMA_DMACR_FRAME_COUNT_SHIFT 16 67 #define XILINX_DMA_DMACR_ERR_IRQ BIT(14) 68 #define XILINX_DMA_DMACR_DLY_CNT_IRQ BIT(13) 69 #define XILINX_DMA_DMACR_FRM_CNT_IRQ BIT(12) 70 #define XILINX_DMA_DMACR_MASTER_SHIFT 8 71 #define XILINX_DMA_DMACR_FSYNCSRC_SHIFT 5 72 #define XILINX_DMA_DMACR_FRAMECNT_EN BIT(4) 73 #define XILINX_DMA_DMACR_GENLOCK_EN BIT(3) 74 #define XILINX_DMA_DMACR_RESET BIT(2) 75 #define XILINX_DMA_DMACR_CIRC_EN BIT(1) 76 #define XILINX_DMA_DMACR_RUNSTOP BIT(0) 77 #define XILINX_DMA_DMACR_FSYNCSRC_MASK GENMASK(6, 5) 78 #define XILINX_DMA_DMACR_DELAY_MASK GENMASK(31, 24) 79 #define XILINX_DMA_DMACR_FRAME_COUNT_MASK GENMASK(23, 16) 80 #define XILINX_DMA_DMACR_MASTER_MASK GENMASK(11, 8) 81 82 #define XILINX_DMA_REG_DMASR 0x0004 83 #define XILINX_DMA_DMASR_EOL_LATE_ERR BIT(15) 84 #define XILINX_DMA_DMASR_ERR_IRQ BIT(14) 85 #define XILINX_DMA_DMASR_DLY_CNT_IRQ BIT(13) 86 #define XILINX_DMA_DMASR_FRM_CNT_IRQ BIT(12) 87 #define XILINX_DMA_DMASR_SOF_LATE_ERR BIT(11) 88 #define XILINX_DMA_DMASR_SG_DEC_ERR BIT(10) 89 #define XILINX_DMA_DMASR_SG_SLV_ERR BIT(9) 90 #define XILINX_DMA_DMASR_EOF_EARLY_ERR BIT(8) 91 #define XILINX_DMA_DMASR_SOF_EARLY_ERR BIT(7) 92 #define XILINX_DMA_DMASR_DMA_DEC_ERR BIT(6) 93 #define XILINX_DMA_DMASR_DMA_SLAVE_ERR BIT(5) 94 #define XILINX_DMA_DMASR_DMA_INT_ERR BIT(4) 95 #define XILINX_DMA_DMASR_SG_MASK BIT(3) 96 #define XILINX_DMA_DMASR_IDLE BIT(1) 97 #define XILINX_DMA_DMASR_HALTED BIT(0) 98 #define XILINX_DMA_DMASR_DELAY_MASK GENMASK(31, 24) 99 #define XILINX_DMA_DMASR_FRAME_COUNT_MASK GENMASK(23, 16) 100 101 #define XILINX_DMA_REG_CURDESC 0x0008 102 #define XILINX_DMA_REG_TAILDESC 0x0010 103 #define XILINX_DMA_REG_REG_INDEX 0x0014 104 #define XILINX_DMA_REG_FRMSTORE 0x0018 105 #define XILINX_DMA_REG_THRESHOLD 0x001c 106 #define XILINX_DMA_REG_FRMPTR_STS 0x0024 107 #define XILINX_DMA_REG_PARK_PTR 0x0028 108 #define XILINX_DMA_PARK_PTR_WR_REF_SHIFT 8 109 #define XILINX_DMA_PARK_PTR_WR_REF_MASK GENMASK(12, 8) 110 #define XILINX_DMA_PARK_PTR_RD_REF_SHIFT 0 111 #define XILINX_DMA_PARK_PTR_RD_REF_MASK GENMASK(4, 0) 112 #define XILINX_DMA_REG_VDMA_VERSION 0x002c 113 114 /* Register Direct Mode Registers */ 115 #define XILINX_DMA_REG_VSIZE 0x0000 116 #define XILINX_DMA_VSIZE_MASK GENMASK(12, 0) 117 #define XILINX_DMA_REG_HSIZE 0x0004 118 #define XILINX_DMA_HSIZE_MASK GENMASK(15, 0) 119 120 #define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 121 #define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 122 #define XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT 0 123 124 #define XILINX_VDMA_REG_START_ADDRESS(n) (0x000c + 4 * (n)) 125 #define XILINX_VDMA_REG_START_ADDRESS_64(n) (0x000c + 8 * (n)) 126 127 #define XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP 0x00ec 128 #define XILINX_VDMA_ENABLE_VERTICAL_FLIP BIT(0) 129 130 /* HW specific definitions */ 131 #define XILINX_MCDMA_MAX_CHANS_PER_DEVICE 0x20 132 #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x2 133 #define XILINX_CDMA_MAX_CHANS_PER_DEVICE 0x1 134 #define XILINX_DMA_DFAULT_ADDRWIDTH 0x20 135 136 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ 137 (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ 138 XILINX_DMA_DMASR_DLY_CNT_IRQ | \ 139 XILINX_DMA_DMASR_ERR_IRQ) 140 141 #define XILINX_DMA_DMASR_ALL_ERR_MASK \ 142 (XILINX_DMA_DMASR_EOL_LATE_ERR | \ 143 XILINX_DMA_DMASR_SOF_LATE_ERR | \ 144 XILINX_DMA_DMASR_SG_DEC_ERR | \ 145 XILINX_DMA_DMASR_SG_SLV_ERR | \ 146 XILINX_DMA_DMASR_EOF_EARLY_ERR | \ 147 XILINX_DMA_DMASR_SOF_EARLY_ERR | \ 148 XILINX_DMA_DMASR_DMA_DEC_ERR | \ 149 XILINX_DMA_DMASR_DMA_SLAVE_ERR | \ 150 XILINX_DMA_DMASR_DMA_INT_ERR) 151 152 /* 153 * Recoverable errors are DMA Internal error, SOF Early, EOF Early 154 * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC 155 * is enabled in the h/w system. 156 */ 157 #define XILINX_DMA_DMASR_ERR_RECOVER_MASK \ 158 (XILINX_DMA_DMASR_SOF_LATE_ERR | \ 159 XILINX_DMA_DMASR_EOF_EARLY_ERR | \ 160 XILINX_DMA_DMASR_SOF_EARLY_ERR | \ 161 XILINX_DMA_DMASR_DMA_INT_ERR) 162 163 /* Axi VDMA Flush on Fsync bits */ 164 #define XILINX_DMA_FLUSH_S2MM 3 165 #define XILINX_DMA_FLUSH_MM2S 2 166 #define XILINX_DMA_FLUSH_BOTH 1 167 168 /* Timeout for polling various registers */ 169 #define XILINX_DMA_POLL_TIMEOUT_US 1000000 170 /* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ 171 #define XILINX_DMA_POLL_DELAY_US 10 172 173 /* AXI DMA Specific Registers/Offsets */ 174 #define XILINX_DMA_REG_SRCDSTADDR 0x18 175 #define XILINX_DMA_REG_BTT 0x28 176 177 /* AXI DMA Specific Masks/Bit fields */ 178 #define XILINX_DMA_MAX_TRANS_LEN_MIN 8 179 #define XILINX_DMA_MAX_TRANS_LEN_MAX 23 180 #define XILINX_DMA_V2_MAX_TRANS_LEN_MAX 26 181 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) 182 #define XILINX_DMA_CR_DELAY_MAX GENMASK(31, 24) 183 #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) 184 #define XILINX_DMA_CR_COALESCE_SHIFT 16 185 #define XILINX_DMA_CR_DELAY_SHIFT 24 186 #define XILINX_DMA_BD_SOP BIT(27) 187 #define XILINX_DMA_BD_EOP BIT(26) 188 #define XILINX_DMA_BD_COMP_MASK BIT(31) 189 #define XILINX_DMA_COALESCE_MAX 255 190 #define XILINX_DMA_NUM_DESCS 512 191 #define XILINX_DMA_NUM_APP_WORDS 5 192 193 /* AXI CDMA Specific Registers/Offsets */ 194 #define XILINX_CDMA_REG_SRCADDR 0x18 195 #define XILINX_CDMA_REG_DSTADDR 0x20 196 197 /* AXI CDMA Specific Masks */ 198 #define XILINX_CDMA_CR_SGMODE BIT(3) 199 200 #define xilinx_prep_dma_addr_t(addr) \ 201 ((dma_addr_t)((u64)addr##_##msb << 32 | (addr))) 202 203 /* AXI MCDMA Specific Registers/Offsets */ 204 #define XILINX_MCDMA_MM2S_CTRL_OFFSET 0x0000 205 #define XILINX_MCDMA_S2MM_CTRL_OFFSET 0x0500 206 #define XILINX_MCDMA_CHEN_OFFSET 0x0008 207 #define XILINX_MCDMA_CH_ERR_OFFSET 0x0010 208 #define XILINX_MCDMA_RXINT_SER_OFFSET 0x0020 209 #define XILINX_MCDMA_TXINT_SER_OFFSET 0x0028 210 #define XILINX_MCDMA_CHAN_CR_OFFSET(x) (0x40 + (x) * 0x40) 211 #define XILINX_MCDMA_CHAN_SR_OFFSET(x) (0x44 + (x) * 0x40) 212 #define XILINX_MCDMA_CHAN_CDESC_OFFSET(x) (0x48 + (x) * 0x40) 213 #define XILINX_MCDMA_CHAN_TDESC_OFFSET(x) (0x50 + (x) * 0x40) 214 215 /* AXI MCDMA Specific Masks/Shifts */ 216 #define XILINX_MCDMA_COALESCE_SHIFT 16 217 #define XILINX_MCDMA_COALESCE_MAX 24 218 #define XILINX_MCDMA_IRQ_ALL_MASK GENMASK(7, 5) 219 #define XILINX_MCDMA_COALESCE_MASK GENMASK(23, 16) 220 #define XILINX_MCDMA_CR_RUNSTOP_MASK BIT(0) 221 #define XILINX_MCDMA_IRQ_IOC_MASK BIT(5) 222 #define XILINX_MCDMA_IRQ_DELAY_MASK BIT(6) 223 #define XILINX_MCDMA_IRQ_ERR_MASK BIT(7) 224 #define XILINX_MCDMA_BD_EOP BIT(30) 225 #define XILINX_MCDMA_BD_SOP BIT(31) 226 227 /** 228 * struct xilinx_vdma_desc_hw - Hardware Descriptor 229 * @next_desc: Next Descriptor Pointer @0x00 230 * @pad1: Reserved @0x04 231 * @buf_addr: Buffer address @0x08 232 * @buf_addr_msb: MSB of Buffer address @0x0C 233 * @vsize: Vertical Size @0x10 234 * @hsize: Horizontal Size @0x14 235 * @stride: Number of bytes between the first 236 * pixels of each horizontal line @0x18 237 */ 238 struct xilinx_vdma_desc_hw { 239 u32 next_desc; 240 u32 pad1; 241 u32 buf_addr; 242 u32 buf_addr_msb; 243 u32 vsize; 244 u32 hsize; 245 u32 stride; 246 } __aligned(64); 247 248 /** 249 * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA 250 * @next_desc: Next Descriptor Pointer @0x00 251 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04 252 * @buf_addr: Buffer address @0x08 253 * @buf_addr_msb: MSB of Buffer address @0x0C 254 * @reserved1: Reserved @0x10 255 * @reserved2: Reserved @0x14 256 * @control: Control field @0x18 257 * @status: Status field @0x1C 258 * @app: APP Fields @0x20 - 0x30 259 */ 260 struct xilinx_axidma_desc_hw { 261 u32 next_desc; 262 u32 next_desc_msb; 263 u32 buf_addr; 264 u32 buf_addr_msb; 265 u32 reserved1; 266 u32 reserved2; 267 u32 control; 268 u32 status; 269 u32 app[XILINX_DMA_NUM_APP_WORDS]; 270 } __aligned(64); 271 272 /** 273 * struct xilinx_aximcdma_desc_hw - Hardware Descriptor for AXI MCDMA 274 * @next_desc: Next Descriptor Pointer @0x00 275 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04 276 * @buf_addr: Buffer address @0x08 277 * @buf_addr_msb: MSB of Buffer address @0x0C 278 * @rsvd: Reserved field @0x10 279 * @control: Control Information field @0x14 280 * @status: Status field @0x18 281 * @sideband_status: Status of sideband signals @0x1C 282 * @app: APP Fields @0x20 - 0x30 283 */ 284 struct xilinx_aximcdma_desc_hw { 285 u32 next_desc; 286 u32 next_desc_msb; 287 u32 buf_addr; 288 u32 buf_addr_msb; 289 u32 rsvd; 290 u32 control; 291 u32 status; 292 u32 sideband_status; 293 u32 app[XILINX_DMA_NUM_APP_WORDS]; 294 } __aligned(64); 295 296 /** 297 * struct xilinx_cdma_desc_hw - Hardware Descriptor 298 * @next_desc: Next Descriptor Pointer @0x00 299 * @next_desc_msb: Next Descriptor Pointer MSB @0x04 300 * @src_addr: Source address @0x08 301 * @src_addr_msb: Source address MSB @0x0C 302 * @dest_addr: Destination address @0x10 303 * @dest_addr_msb: Destination address MSB @0x14 304 * @control: Control field @0x18 305 * @status: Status field @0x1C 306 */ 307 struct xilinx_cdma_desc_hw { 308 u32 next_desc; 309 u32 next_desc_msb; 310 u32 src_addr; 311 u32 src_addr_msb; 312 u32 dest_addr; 313 u32 dest_addr_msb; 314 u32 control; 315 u32 status; 316 } __aligned(64); 317 318 /** 319 * struct xilinx_vdma_tx_segment - Descriptor segment 320 * @hw: Hardware descriptor 321 * @node: Node in the descriptor segments list 322 * @phys: Physical address of segment 323 */ 324 struct xilinx_vdma_tx_segment { 325 struct xilinx_vdma_desc_hw hw; 326 struct list_head node; 327 dma_addr_t phys; 328 } __aligned(64); 329 330 /** 331 * struct xilinx_axidma_tx_segment - Descriptor segment 332 * @hw: Hardware descriptor 333 * @node: Node in the descriptor segments list 334 * @phys: Physical address of segment 335 */ 336 struct xilinx_axidma_tx_segment { 337 struct xilinx_axidma_desc_hw hw; 338 struct list_head node; 339 dma_addr_t phys; 340 } __aligned(64); 341 342 /** 343 * struct xilinx_aximcdma_tx_segment - Descriptor segment 344 * @hw: Hardware descriptor 345 * @node: Node in the descriptor segments list 346 * @phys: Physical address of segment 347 */ 348 struct xilinx_aximcdma_tx_segment { 349 struct xilinx_aximcdma_desc_hw hw; 350 struct list_head node; 351 dma_addr_t phys; 352 } __aligned(64); 353 354 /** 355 * struct xilinx_cdma_tx_segment - Descriptor segment 356 * @hw: Hardware descriptor 357 * @node: Node in the descriptor segments list 358 * @phys: Physical address of segment 359 */ 360 struct xilinx_cdma_tx_segment { 361 struct xilinx_cdma_desc_hw hw; 362 struct list_head node; 363 dma_addr_t phys; 364 } __aligned(64); 365 366 /** 367 * struct xilinx_dma_tx_descriptor - Per Transaction structure 368 * @async_tx: Async transaction descriptor 369 * @segments: TX segments list 370 * @node: Node in the channel descriptors list 371 * @cyclic: Check for cyclic transfers. 372 * @err: Whether the descriptor has an error. 373 * @residue: Residue of the completed descriptor 374 */ 375 struct xilinx_dma_tx_descriptor { 376 struct dma_async_tx_descriptor async_tx; 377 struct list_head segments; 378 struct list_head node; 379 bool cyclic; 380 bool err; 381 u32 residue; 382 }; 383 384 /** 385 * struct xilinx_dma_chan - Driver specific DMA channel structure 386 * @xdev: Driver specific device structure 387 * @ctrl_offset: Control registers offset 388 * @desc_offset: TX descriptor registers offset 389 * @lock: Descriptor operation lock 390 * @pending_list: Descriptors waiting 391 * @active_list: Descriptors ready to submit 392 * @done_list: Complete descriptors 393 * @free_seg_list: Free descriptors 394 * @common: DMA common channel 395 * @desc_pool: Descriptors pool 396 * @dev: The dma device 397 * @irq: Channel IRQ 398 * @id: Channel ID 399 * @direction: Transfer direction 400 * @num_frms: Number of frames 401 * @has_sg: Support scatter transfers 402 * @cyclic: Check for cyclic transfers. 403 * @genlock: Support genlock mode 404 * @err: Channel has errors 405 * @idle: Check for channel idle 406 * @terminating: Check for channel being synchronized by user 407 * @tasklet: Cleanup work after irq 408 * @config: Device configuration info 409 * @flush_on_fsync: Flush on Frame sync 410 * @desc_pendingcount: Descriptor pending count 411 * @ext_addr: Indicates 64 bit addressing is supported by dma channel 412 * @desc_submitcount: Descriptor h/w submitted count 413 * @seg_v: Statically allocated segments base 414 * @seg_mv: Statically allocated segments base for MCDMA 415 * @seg_p: Physical allocated segments base 416 * @cyclic_seg_v: Statically allocated segment base for cyclic transfers 417 * @cyclic_seg_p: Physical allocated segments base for cyclic dma 418 * @start_transfer: Differentiate b/w DMA IP's transfer 419 * @stop_transfer: Differentiate b/w DMA IP's quiesce 420 * @tdest: TDEST value for mcdma 421 * @has_vflip: S2MM vertical flip 422 * @irq_delay: Interrupt delay timeout 423 */ 424 struct xilinx_dma_chan { 425 struct xilinx_dma_device *xdev; 426 u32 ctrl_offset; 427 u32 desc_offset; 428 spinlock_t lock; 429 struct list_head pending_list; 430 struct list_head active_list; 431 struct list_head done_list; 432 struct list_head free_seg_list; 433 struct dma_chan common; 434 struct dma_pool *desc_pool; 435 struct device *dev; 436 int irq; 437 int id; 438 enum dma_transfer_direction direction; 439 int num_frms; 440 bool has_sg; 441 bool cyclic; 442 bool genlock; 443 bool err; 444 bool idle; 445 bool terminating; 446 struct tasklet_struct tasklet; 447 struct xilinx_vdma_config config; 448 bool flush_on_fsync; 449 u32 desc_pendingcount; 450 bool ext_addr; 451 u32 desc_submitcount; 452 struct xilinx_axidma_tx_segment *seg_v; 453 struct xilinx_aximcdma_tx_segment *seg_mv; 454 dma_addr_t seg_p; 455 struct xilinx_axidma_tx_segment *cyclic_seg_v; 456 dma_addr_t cyclic_seg_p; 457 void (*start_transfer)(struct xilinx_dma_chan *chan); 458 int (*stop_transfer)(struct xilinx_dma_chan *chan); 459 u16 tdest; 460 bool has_vflip; 461 u8 irq_delay; 462 }; 463 464 /** 465 * enum xdma_ip_type - DMA IP type. 466 * 467 * @XDMA_TYPE_AXIDMA: Axi dma ip. 468 * @XDMA_TYPE_CDMA: Axi cdma ip. 469 * @XDMA_TYPE_VDMA: Axi vdma ip. 470 * @XDMA_TYPE_AXIMCDMA: Axi MCDMA ip. 471 * 472 */ 473 enum xdma_ip_type { 474 XDMA_TYPE_AXIDMA = 0, 475 XDMA_TYPE_CDMA, 476 XDMA_TYPE_VDMA, 477 XDMA_TYPE_AXIMCDMA 478 }; 479 480 struct xilinx_dma_config { 481 enum xdma_ip_type dmatype; 482 int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk, 483 struct clk **tx_clk, struct clk **txs_clk, 484 struct clk **rx_clk, struct clk **rxs_clk); 485 irqreturn_t (*irq_handler)(int irq, void *data); 486 const int max_channels; 487 }; 488 489 /** 490 * struct xilinx_dma_device - DMA device structure 491 * @regs: I/O mapped base address 492 * @dev: Device Structure 493 * @common: DMA device structure 494 * @chan: Driver specific DMA channel 495 * @flush_on_fsync: Flush on frame sync 496 * @ext_addr: Indicates 64 bit addressing is supported by dma device 497 * @pdev: Platform device structure pointer 498 * @dma_config: DMA config structure 499 * @axi_clk: DMA Axi4-lite interace clock 500 * @tx_clk: DMA mm2s clock 501 * @txs_clk: DMA mm2s stream clock 502 * @rx_clk: DMA s2mm clock 503 * @rxs_clk: DMA s2mm stream clock 504 * @s2mm_chan_id: DMA s2mm channel identifier 505 * @mm2s_chan_id: DMA mm2s channel identifier 506 * @max_buffer_len: Max buffer length 507 * @has_axistream_connected: AXI DMA connected to AXI Stream IP 508 */ 509 struct xilinx_dma_device { 510 void __iomem *regs; 511 struct device *dev; 512 struct dma_device common; 513 struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE]; 514 u32 flush_on_fsync; 515 bool ext_addr; 516 struct platform_device *pdev; 517 const struct xilinx_dma_config *dma_config; 518 struct clk *axi_clk; 519 struct clk *tx_clk; 520 struct clk *txs_clk; 521 struct clk *rx_clk; 522 struct clk *rxs_clk; 523 u32 s2mm_chan_id; 524 u32 mm2s_chan_id; 525 u32 max_buffer_len; 526 bool has_axistream_connected; 527 }; 528 529 /* Macros */ 530 #define to_xilinx_chan(chan) \ 531 container_of(chan, struct xilinx_dma_chan, common) 532 #define to_dma_tx_descriptor(tx) \ 533 container_of(tx, struct xilinx_dma_tx_descriptor, async_tx) 534 #define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \ 535 readl_poll_timeout_atomic(chan->xdev->regs + chan->ctrl_offset + reg, \ 536 val, cond, delay_us, timeout_us) 537 538 /* IO accessors */ 539 static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg) 540 { 541 return ioread32(chan->xdev->regs + reg); 542 } 543 544 static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value) 545 { 546 iowrite32(value, chan->xdev->regs + reg); 547 } 548 549 static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg, 550 u32 value) 551 { 552 dma_write(chan, chan->desc_offset + reg, value); 553 } 554 555 static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg) 556 { 557 return dma_read(chan, chan->ctrl_offset + reg); 558 } 559 560 static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg, 561 u32 value) 562 { 563 dma_write(chan, chan->ctrl_offset + reg, value); 564 } 565 566 static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg, 567 u32 clr) 568 { 569 dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr); 570 } 571 572 static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg, 573 u32 set) 574 { 575 dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set); 576 } 577 578 /** 579 * vdma_desc_write_64 - 64-bit descriptor write 580 * @chan: Driver specific VDMA channel 581 * @reg: Register to write 582 * @value_lsb: lower address of the descriptor. 583 * @value_msb: upper address of the descriptor. 584 * 585 * Since vdma driver is trying to write to a register offset which is not a 586 * multiple of 64 bits(ex : 0x5c), we are writing as two separate 32 bits 587 * instead of a single 64 bit register write. 588 */ 589 static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg, 590 u32 value_lsb, u32 value_msb) 591 { 592 /* Write the lsb 32 bits*/ 593 writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg); 594 595 /* Write the msb 32 bits */ 596 writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4); 597 } 598 599 static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value) 600 { 601 lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg); 602 } 603 604 static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg, 605 dma_addr_t addr) 606 { 607 if (chan->ext_addr) 608 dma_writeq(chan, reg, addr); 609 else 610 dma_ctrl_write(chan, reg, addr); 611 } 612 613 static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan, 614 struct xilinx_axidma_desc_hw *hw, 615 dma_addr_t buf_addr, size_t sg_used, 616 size_t period_len) 617 { 618 if (chan->ext_addr) { 619 hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len); 620 hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used + 621 period_len); 622 } else { 623 hw->buf_addr = buf_addr + sg_used + period_len; 624 } 625 } 626 627 static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan, 628 struct xilinx_aximcdma_desc_hw *hw, 629 dma_addr_t buf_addr, size_t sg_used) 630 { 631 if (chan->ext_addr) { 632 hw->buf_addr = lower_32_bits(buf_addr + sg_used); 633 hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used); 634 } else { 635 hw->buf_addr = buf_addr + sg_used; 636 } 637 } 638 639 /** 640 * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length 641 * @tx: async transaction descriptor 642 * @payload_len: metadata payload length 643 * @max_len: metadata max length 644 * Return: The app field pointer. 645 */ 646 static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, 647 size_t *payload_len, size_t *max_len) 648 { 649 struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); 650 struct xilinx_axidma_tx_segment *seg; 651 652 *max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS; 653 seg = list_first_entry(&desc->segments, 654 struct xilinx_axidma_tx_segment, node); 655 return seg->hw.app; 656 } 657 658 static const struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { 659 .get_ptr = xilinx_dma_get_metadata_ptr, 660 }; 661 662 /* ----------------------------------------------------------------------------- 663 * Descriptors and segments alloc and free 664 */ 665 666 /** 667 * xilinx_vdma_alloc_tx_segment - Allocate transaction segment 668 * @chan: Driver specific DMA channel 669 * 670 * Return: The allocated segment on success and NULL on failure. 671 */ 672 static struct xilinx_vdma_tx_segment * 673 xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan) 674 { 675 struct xilinx_vdma_tx_segment *segment; 676 dma_addr_t phys; 677 678 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); 679 if (!segment) 680 return NULL; 681 682 segment->phys = phys; 683 684 return segment; 685 } 686 687 /** 688 * xilinx_cdma_alloc_tx_segment - Allocate transaction segment 689 * @chan: Driver specific DMA channel 690 * 691 * Return: The allocated segment on success and NULL on failure. 692 */ 693 static struct xilinx_cdma_tx_segment * 694 xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan) 695 { 696 struct xilinx_cdma_tx_segment *segment; 697 dma_addr_t phys; 698 699 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); 700 if (!segment) 701 return NULL; 702 703 segment->phys = phys; 704 705 return segment; 706 } 707 708 /** 709 * xilinx_axidma_alloc_tx_segment - Allocate transaction segment 710 * @chan: Driver specific DMA channel 711 * 712 * Return: The allocated segment on success and NULL on failure. 713 */ 714 static struct xilinx_axidma_tx_segment * 715 xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan) 716 { 717 struct xilinx_axidma_tx_segment *segment = NULL; 718 unsigned long flags; 719 720 spin_lock_irqsave(&chan->lock, flags); 721 if (!list_empty(&chan->free_seg_list)) { 722 segment = list_first_entry(&chan->free_seg_list, 723 struct xilinx_axidma_tx_segment, 724 node); 725 list_del(&segment->node); 726 } 727 spin_unlock_irqrestore(&chan->lock, flags); 728 729 if (!segment) 730 dev_dbg(chan->dev, "Could not find free tx segment\n"); 731 732 return segment; 733 } 734 735 /** 736 * xilinx_aximcdma_alloc_tx_segment - Allocate transaction segment 737 * @chan: Driver specific DMA channel 738 * 739 * Return: The allocated segment on success and NULL on failure. 740 */ 741 static struct xilinx_aximcdma_tx_segment * 742 xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan) 743 { 744 struct xilinx_aximcdma_tx_segment *segment = NULL; 745 unsigned long flags; 746 747 spin_lock_irqsave(&chan->lock, flags); 748 if (!list_empty(&chan->free_seg_list)) { 749 segment = list_first_entry(&chan->free_seg_list, 750 struct xilinx_aximcdma_tx_segment, 751 node); 752 list_del(&segment->node); 753 } 754 spin_unlock_irqrestore(&chan->lock, flags); 755 756 return segment; 757 } 758 759 static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw) 760 { 761 u32 next_desc = hw->next_desc; 762 u32 next_desc_msb = hw->next_desc_msb; 763 764 memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw)); 765 766 hw->next_desc = next_desc; 767 hw->next_desc_msb = next_desc_msb; 768 } 769 770 static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw) 771 { 772 u32 next_desc = hw->next_desc; 773 u32 next_desc_msb = hw->next_desc_msb; 774 775 memset(hw, 0, sizeof(struct xilinx_aximcdma_desc_hw)); 776 777 hw->next_desc = next_desc; 778 hw->next_desc_msb = next_desc_msb; 779 } 780 781 /** 782 * xilinx_dma_free_tx_segment - Free transaction segment 783 * @chan: Driver specific DMA channel 784 * @segment: DMA transaction segment 785 */ 786 static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan, 787 struct xilinx_axidma_tx_segment *segment) 788 { 789 xilinx_dma_clean_hw_desc(&segment->hw); 790 791 list_add_tail(&segment->node, &chan->free_seg_list); 792 } 793 794 /** 795 * xilinx_mcdma_free_tx_segment - Free transaction segment 796 * @chan: Driver specific DMA channel 797 * @segment: DMA transaction segment 798 */ 799 static void xilinx_mcdma_free_tx_segment(struct xilinx_dma_chan *chan, 800 struct xilinx_aximcdma_tx_segment * 801 segment) 802 { 803 xilinx_mcdma_clean_hw_desc(&segment->hw); 804 805 list_add_tail(&segment->node, &chan->free_seg_list); 806 } 807 808 /** 809 * xilinx_cdma_free_tx_segment - Free transaction segment 810 * @chan: Driver specific DMA channel 811 * @segment: DMA transaction segment 812 */ 813 static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan, 814 struct xilinx_cdma_tx_segment *segment) 815 { 816 dma_pool_free(chan->desc_pool, segment, segment->phys); 817 } 818 819 /** 820 * xilinx_vdma_free_tx_segment - Free transaction segment 821 * @chan: Driver specific DMA channel 822 * @segment: DMA transaction segment 823 */ 824 static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan, 825 struct xilinx_vdma_tx_segment *segment) 826 { 827 dma_pool_free(chan->desc_pool, segment, segment->phys); 828 } 829 830 /** 831 * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor 832 * @chan: Driver specific DMA channel 833 * 834 * Return: The allocated descriptor on success and NULL on failure. 835 */ 836 static struct xilinx_dma_tx_descriptor * 837 xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) 838 { 839 struct xilinx_dma_tx_descriptor *desc; 840 841 desc = kzalloc_obj(*desc, GFP_NOWAIT); 842 if (!desc) 843 return NULL; 844 845 INIT_LIST_HEAD(&desc->segments); 846 847 return desc; 848 } 849 850 /** 851 * xilinx_dma_free_tx_descriptor - Free transaction descriptor 852 * @chan: Driver specific DMA channel 853 * @desc: DMA transaction descriptor 854 */ 855 static void 856 xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan, 857 struct xilinx_dma_tx_descriptor *desc) 858 { 859 struct xilinx_vdma_tx_segment *segment, *next; 860 struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next; 861 struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next; 862 struct xilinx_aximcdma_tx_segment *aximcdma_segment, *aximcdma_next; 863 864 if (!desc) 865 return; 866 867 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 868 list_for_each_entry_safe(segment, next, &desc->segments, node) { 869 list_del(&segment->node); 870 xilinx_vdma_free_tx_segment(chan, segment); 871 } 872 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 873 list_for_each_entry_safe(cdma_segment, cdma_next, 874 &desc->segments, node) { 875 list_del(&cdma_segment->node); 876 xilinx_cdma_free_tx_segment(chan, cdma_segment); 877 } 878 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 879 list_for_each_entry_safe(axidma_segment, axidma_next, 880 &desc->segments, node) { 881 list_del(&axidma_segment->node); 882 xilinx_dma_free_tx_segment(chan, axidma_segment); 883 } 884 } else { 885 list_for_each_entry_safe(aximcdma_segment, aximcdma_next, 886 &desc->segments, node) { 887 list_del(&aximcdma_segment->node); 888 xilinx_mcdma_free_tx_segment(chan, aximcdma_segment); 889 } 890 } 891 892 kfree(desc); 893 } 894 895 /* Required functions */ 896 897 /** 898 * xilinx_dma_free_desc_list - Free descriptors list 899 * @chan: Driver specific DMA channel 900 * @list: List to parse and delete the descriptor 901 */ 902 static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan, 903 struct list_head *list) 904 { 905 struct xilinx_dma_tx_descriptor *desc, *next; 906 907 list_for_each_entry_safe(desc, next, list, node) { 908 list_del(&desc->node); 909 xilinx_dma_free_tx_descriptor(chan, desc); 910 } 911 } 912 913 /** 914 * xilinx_dma_free_descriptors - Free channel descriptors 915 * @chan: Driver specific DMA channel 916 */ 917 static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan) 918 { 919 unsigned long flags; 920 921 spin_lock_irqsave(&chan->lock, flags); 922 923 xilinx_dma_free_desc_list(chan, &chan->pending_list); 924 xilinx_dma_free_desc_list(chan, &chan->done_list); 925 xilinx_dma_free_desc_list(chan, &chan->active_list); 926 927 spin_unlock_irqrestore(&chan->lock, flags); 928 } 929 930 /** 931 * xilinx_dma_free_chan_resources - Free channel resources 932 * @dchan: DMA channel 933 */ 934 static void xilinx_dma_free_chan_resources(struct dma_chan *dchan) 935 { 936 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 937 unsigned long flags; 938 939 dev_dbg(chan->dev, "Free all channel resources.\n"); 940 941 xilinx_dma_free_descriptors(chan); 942 943 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 944 spin_lock_irqsave(&chan->lock, flags); 945 INIT_LIST_HEAD(&chan->free_seg_list); 946 spin_unlock_irqrestore(&chan->lock, flags); 947 948 /* Free memory that is allocated for BD */ 949 dma_free_coherent(chan->dev, sizeof(*chan->seg_v) * 950 XILINX_DMA_NUM_DESCS, chan->seg_v, 951 chan->seg_p); 952 953 /* Free Memory that is allocated for cyclic DMA Mode */ 954 dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v), 955 chan->cyclic_seg_v, chan->cyclic_seg_p); 956 } 957 958 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { 959 spin_lock_irqsave(&chan->lock, flags); 960 INIT_LIST_HEAD(&chan->free_seg_list); 961 spin_unlock_irqrestore(&chan->lock, flags); 962 963 /* Free memory that is allocated for BD */ 964 dma_free_coherent(chan->dev, sizeof(*chan->seg_mv) * 965 XILINX_DMA_NUM_DESCS, chan->seg_mv, 966 chan->seg_p); 967 } 968 969 if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA && 970 chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) { 971 dma_pool_destroy(chan->desc_pool); 972 chan->desc_pool = NULL; 973 } 974 975 } 976 977 /** 978 * xilinx_dma_get_residue - Compute residue for a given descriptor 979 * @chan: Driver specific dma channel 980 * @desc: dma transaction descriptor 981 * 982 * Return: The number of residue bytes for the descriptor. 983 */ 984 static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, 985 struct xilinx_dma_tx_descriptor *desc) 986 { 987 struct xilinx_cdma_tx_segment *cdma_seg; 988 struct xilinx_axidma_tx_segment *axidma_seg; 989 struct xilinx_aximcdma_tx_segment *aximcdma_seg; 990 struct xilinx_cdma_desc_hw *cdma_hw; 991 struct xilinx_axidma_desc_hw *axidma_hw; 992 struct xilinx_aximcdma_desc_hw *aximcdma_hw; 993 struct list_head *entry; 994 u32 residue = 0; 995 996 list_for_each(entry, &desc->segments) { 997 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 998 cdma_seg = list_entry(entry, 999 struct xilinx_cdma_tx_segment, 1000 node); 1001 cdma_hw = &cdma_seg->hw; 1002 residue += (cdma_hw->control & chan->xdev->max_buffer_len) - 1003 (cdma_hw->status & chan->xdev->max_buffer_len); 1004 } else if (chan->xdev->dma_config->dmatype == 1005 XDMA_TYPE_AXIDMA) { 1006 axidma_seg = list_entry(entry, 1007 struct xilinx_axidma_tx_segment, 1008 node); 1009 axidma_hw = &axidma_seg->hw; 1010 residue += (axidma_hw->control & chan->xdev->max_buffer_len) - 1011 (axidma_hw->status & chan->xdev->max_buffer_len); 1012 } else { 1013 aximcdma_seg = 1014 list_entry(entry, 1015 struct xilinx_aximcdma_tx_segment, 1016 node); 1017 aximcdma_hw = &aximcdma_seg->hw; 1018 residue += 1019 (aximcdma_hw->control & chan->xdev->max_buffer_len) - 1020 (aximcdma_hw->status & chan->xdev->max_buffer_len); 1021 } 1022 } 1023 1024 return residue; 1025 } 1026 1027 static u32 1028 xilinx_dma_get_residue_axidma_direct_s2mm(struct xilinx_dma_chan *chan, 1029 struct xilinx_dma_tx_descriptor *desc) 1030 { 1031 struct xilinx_axidma_tx_segment *seg; 1032 struct xilinx_axidma_desc_hw *hw; 1033 u32 finished_len; 1034 1035 finished_len = dma_ctrl_read(chan, XILINX_DMA_REG_BTT); 1036 1037 seg = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, 1038 node); 1039 1040 hw = &seg->hw; 1041 1042 return hw->control - finished_len; 1043 } 1044 1045 /** 1046 * xilinx_dma_chan_handle_cyclic - Cyclic dma callback 1047 * @chan: Driver specific dma channel 1048 * @desc: dma transaction descriptor 1049 * @flags: flags for spin lock 1050 */ 1051 static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan, 1052 struct xilinx_dma_tx_descriptor *desc, 1053 unsigned long *flags) 1054 { 1055 struct dmaengine_desc_callback cb; 1056 1057 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1058 if (dmaengine_desc_callback_valid(&cb)) { 1059 spin_unlock_irqrestore(&chan->lock, *flags); 1060 dmaengine_desc_callback_invoke(&cb, NULL); 1061 spin_lock_irqsave(&chan->lock, *flags); 1062 } 1063 } 1064 1065 /** 1066 * xilinx_dma_chan_desc_cleanup - Clean channel descriptors 1067 * @chan: Driver specific DMA channel 1068 */ 1069 static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) 1070 { 1071 struct xilinx_dma_tx_descriptor *desc, *next; 1072 unsigned long flags; 1073 1074 spin_lock_irqsave(&chan->lock, flags); 1075 1076 list_for_each_entry_safe(desc, next, &chan->done_list, node) { 1077 struct dmaengine_result result; 1078 1079 if (desc->cyclic) { 1080 xilinx_dma_chan_handle_cyclic(chan, desc, &flags); 1081 break; 1082 } 1083 1084 /* Remove from the list of running transactions */ 1085 list_del(&desc->node); 1086 1087 if (unlikely(desc->err)) { 1088 if (chan->direction == DMA_DEV_TO_MEM) 1089 result.result = DMA_TRANS_READ_FAILED; 1090 else 1091 result.result = DMA_TRANS_WRITE_FAILED; 1092 } else { 1093 result.result = DMA_TRANS_NOERROR; 1094 } 1095 1096 result.residue = desc->residue; 1097 1098 /* Run the link descriptor callback function */ 1099 spin_unlock_irqrestore(&chan->lock, flags); 1100 dmaengine_desc_get_callback_invoke(&desc->async_tx, &result); 1101 spin_lock_irqsave(&chan->lock, flags); 1102 1103 /* Run any dependencies, then free the descriptor */ 1104 dma_run_dependencies(&desc->async_tx); 1105 xilinx_dma_free_tx_descriptor(chan, desc); 1106 1107 /* 1108 * While we ran a callback the user called a terminate function, 1109 * which takes care of cleaning up any remaining descriptors 1110 */ 1111 if (chan->terminating) 1112 break; 1113 } 1114 1115 spin_unlock_irqrestore(&chan->lock, flags); 1116 } 1117 1118 /** 1119 * xilinx_dma_do_tasklet - Schedule completion tasklet 1120 * @t: Pointer to the Xilinx DMA channel structure 1121 */ 1122 static void xilinx_dma_do_tasklet(struct tasklet_struct *t) 1123 { 1124 struct xilinx_dma_chan *chan = from_tasklet(chan, t, tasklet); 1125 1126 xilinx_dma_chan_desc_cleanup(chan); 1127 } 1128 1129 /** 1130 * xilinx_dma_alloc_chan_resources - Allocate channel resources 1131 * @dchan: DMA channel 1132 * 1133 * Return: '0' on success and failure value on error 1134 */ 1135 static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) 1136 { 1137 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1138 int i; 1139 1140 /* Has this channel already been allocated? */ 1141 if (chan->desc_pool) 1142 return 0; 1143 1144 /* 1145 * We need the descriptor to be aligned to 64bytes 1146 * for meeting Xilinx VDMA specification requirement. 1147 */ 1148 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 1149 /* Allocate the buffer descriptors. */ 1150 chan->seg_v = dma_alloc_coherent(chan->dev, 1151 sizeof(*chan->seg_v) * XILINX_DMA_NUM_DESCS, 1152 &chan->seg_p, GFP_KERNEL); 1153 if (!chan->seg_v) { 1154 dev_err(chan->dev, 1155 "unable to allocate channel %d descriptors\n", 1156 chan->id); 1157 return -ENOMEM; 1158 } 1159 /* 1160 * For cyclic DMA mode we need to program the tail Descriptor 1161 * register with a value which is not a part of the BD chain 1162 * so allocating a desc segment during channel allocation for 1163 * programming tail descriptor. 1164 */ 1165 chan->cyclic_seg_v = dma_alloc_coherent(chan->dev, 1166 sizeof(*chan->cyclic_seg_v), 1167 &chan->cyclic_seg_p, 1168 GFP_KERNEL); 1169 if (!chan->cyclic_seg_v) { 1170 dev_err(chan->dev, 1171 "unable to allocate desc segment for cyclic DMA\n"); 1172 dma_free_coherent(chan->dev, sizeof(*chan->seg_v) * 1173 XILINX_DMA_NUM_DESCS, chan->seg_v, 1174 chan->seg_p); 1175 return -ENOMEM; 1176 } 1177 chan->cyclic_seg_v->phys = chan->cyclic_seg_p; 1178 1179 for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) { 1180 chan->seg_v[i].hw.next_desc = 1181 lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) * 1182 ((i + 1) % XILINX_DMA_NUM_DESCS)); 1183 chan->seg_v[i].hw.next_desc_msb = 1184 upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) * 1185 ((i + 1) % XILINX_DMA_NUM_DESCS)); 1186 chan->seg_v[i].phys = chan->seg_p + 1187 sizeof(*chan->seg_v) * i; 1188 list_add_tail(&chan->seg_v[i].node, 1189 &chan->free_seg_list); 1190 } 1191 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { 1192 /* Allocate the buffer descriptors. */ 1193 chan->seg_mv = dma_alloc_coherent(chan->dev, 1194 sizeof(*chan->seg_mv) * 1195 XILINX_DMA_NUM_DESCS, 1196 &chan->seg_p, GFP_KERNEL); 1197 if (!chan->seg_mv) { 1198 dev_err(chan->dev, 1199 "unable to allocate channel %d descriptors\n", 1200 chan->id); 1201 return -ENOMEM; 1202 } 1203 for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) { 1204 chan->seg_mv[i].hw.next_desc = 1205 lower_32_bits(chan->seg_p + sizeof(*chan->seg_mv) * 1206 ((i + 1) % XILINX_DMA_NUM_DESCS)); 1207 chan->seg_mv[i].hw.next_desc_msb = 1208 upper_32_bits(chan->seg_p + sizeof(*chan->seg_mv) * 1209 ((i + 1) % XILINX_DMA_NUM_DESCS)); 1210 chan->seg_mv[i].phys = chan->seg_p + 1211 sizeof(*chan->seg_mv) * i; 1212 list_add_tail(&chan->seg_mv[i].node, 1213 &chan->free_seg_list); 1214 } 1215 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 1216 chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool", 1217 chan->dev, 1218 sizeof(struct xilinx_cdma_tx_segment), 1219 __alignof__(struct xilinx_cdma_tx_segment), 1220 0); 1221 } else { 1222 chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool", 1223 chan->dev, 1224 sizeof(struct xilinx_vdma_tx_segment), 1225 __alignof__(struct xilinx_vdma_tx_segment), 1226 0); 1227 } 1228 1229 if (!chan->desc_pool && 1230 ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) && 1231 chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) { 1232 dev_err(chan->dev, 1233 "unable to allocate channel %d descriptor pool\n", 1234 chan->id); 1235 return -ENOMEM; 1236 } 1237 1238 dma_cookie_init(dchan); 1239 1240 if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) 1241 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 1242 XILINX_CDMA_CR_SGMODE); 1243 1244 return 0; 1245 } 1246 1247 /** 1248 * xilinx_dma_calc_copysize - Calculate the amount of data to copy 1249 * @chan: Driver specific DMA channel 1250 * @size: Total data that needs to be copied 1251 * @done: Amount of data that has been already copied 1252 * 1253 * Return: Amount of data that has to be copied 1254 */ 1255 static int xilinx_dma_calc_copysize(struct xilinx_dma_chan *chan, 1256 int size, int done) 1257 { 1258 size_t copy; 1259 1260 copy = min_t(size_t, size - done, 1261 chan->xdev->max_buffer_len); 1262 1263 if ((copy + done < size) && 1264 chan->xdev->common.copy_align) { 1265 /* 1266 * If this is not the last descriptor, make sure 1267 * the next one will be properly aligned 1268 */ 1269 copy = rounddown(copy, 1270 (1 << chan->xdev->common.copy_align)); 1271 } 1272 return copy; 1273 } 1274 1275 /** 1276 * xilinx_dma_tx_status - Get DMA transaction status 1277 * @dchan: DMA channel 1278 * @cookie: Transaction identifier 1279 * @txstate: Transaction state 1280 * 1281 * Return: DMA transaction status 1282 */ 1283 static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan, 1284 dma_cookie_t cookie, 1285 struct dma_tx_state *txstate) 1286 { 1287 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1288 struct xilinx_dma_tx_descriptor *desc; 1289 enum dma_status ret; 1290 unsigned long flags; 1291 u32 residue = 0; 1292 1293 ret = dma_cookie_status(dchan, cookie, txstate); 1294 if (ret == DMA_COMPLETE || !txstate) 1295 return ret; 1296 1297 spin_lock_irqsave(&chan->lock, flags); 1298 if (!list_empty(&chan->active_list)) { 1299 desc = list_last_entry(&chan->active_list, 1300 struct xilinx_dma_tx_descriptor, node); 1301 /* 1302 * VDMA and simple mode do not support residue reporting, so the 1303 * residue field will always be 0. 1304 */ 1305 if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) 1306 residue = xilinx_dma_get_residue(chan, desc); 1307 } 1308 spin_unlock_irqrestore(&chan->lock, flags); 1309 1310 dma_set_residue(txstate, residue); 1311 1312 return ret; 1313 } 1314 1315 /** 1316 * xilinx_dma_stop_transfer - Halt DMA channel 1317 * @chan: Driver specific DMA channel 1318 * 1319 * Return: '0' on success and failure value on error 1320 */ 1321 static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) 1322 { 1323 u32 val; 1324 1325 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); 1326 1327 /* Wait for the hardware to halt */ 1328 return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, 1329 val & XILINX_DMA_DMASR_HALTED, 1330 XILINX_DMA_POLL_DELAY_US, 1331 XILINX_DMA_POLL_TIMEOUT_US); 1332 } 1333 1334 /** 1335 * xilinx_cdma_stop_transfer - Wait for the current transfer to complete 1336 * @chan: Driver specific DMA channel 1337 * 1338 * Return: '0' on success and failure value on error 1339 */ 1340 static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) 1341 { 1342 u32 val; 1343 1344 return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, 1345 val & XILINX_DMA_DMASR_IDLE, 1346 XILINX_DMA_POLL_DELAY_US, 1347 XILINX_DMA_POLL_TIMEOUT_US); 1348 } 1349 1350 /** 1351 * xilinx_dma_start - Start DMA channel 1352 * @chan: Driver specific DMA channel 1353 */ 1354 static void xilinx_dma_start(struct xilinx_dma_chan *chan) 1355 { 1356 int err; 1357 u32 val; 1358 1359 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); 1360 1361 /* Wait for the hardware to start */ 1362 err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, 1363 !(val & XILINX_DMA_DMASR_HALTED), 1364 XILINX_DMA_POLL_DELAY_US, 1365 XILINX_DMA_POLL_TIMEOUT_US); 1366 1367 if (err) { 1368 dev_err(chan->dev, "Cannot start channel %p: %x\n", 1369 chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 1370 1371 chan->err = true; 1372 } 1373 } 1374 1375 /** 1376 * xilinx_vdma_start_transfer - Starts VDMA transfer 1377 * @chan: Driver specific channel struct pointer 1378 */ 1379 static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) 1380 { 1381 struct xilinx_vdma_config *config = &chan->config; 1382 struct xilinx_dma_tx_descriptor *desc; 1383 u32 reg, j; 1384 struct xilinx_vdma_tx_segment *segment, *last = NULL; 1385 int i = 0; 1386 1387 /* This function was invoked with lock held */ 1388 if (chan->err) 1389 return; 1390 1391 if (!chan->idle) 1392 return; 1393 1394 if (list_empty(&chan->pending_list)) 1395 return; 1396 1397 desc = list_first_entry(&chan->pending_list, 1398 struct xilinx_dma_tx_descriptor, node); 1399 1400 /* Configure the hardware using info in the config structure */ 1401 if (chan->has_vflip) { 1402 reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP); 1403 reg &= ~XILINX_VDMA_ENABLE_VERTICAL_FLIP; 1404 reg |= config->vflip_en; 1405 dma_write(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP, 1406 reg); 1407 } 1408 1409 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 1410 1411 if (config->frm_cnt_en) 1412 reg |= XILINX_DMA_DMACR_FRAMECNT_EN; 1413 else 1414 reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN; 1415 1416 /* If not parking, enable circular mode */ 1417 if (config->park) 1418 reg &= ~XILINX_DMA_DMACR_CIRC_EN; 1419 else 1420 reg |= XILINX_DMA_DMACR_CIRC_EN; 1421 1422 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 1423 1424 if (config->park) { 1425 j = chan->desc_submitcount; 1426 reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); 1427 if (chan->direction == DMA_MEM_TO_DEV) { 1428 reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; 1429 reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; 1430 } else { 1431 reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; 1432 reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; 1433 } 1434 dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); 1435 } 1436 1437 /* Start the hardware */ 1438 xilinx_dma_start(chan); 1439 1440 if (chan->err) 1441 return; 1442 1443 /* Start the transfer */ 1444 if (chan->desc_submitcount < chan->num_frms) 1445 i = chan->desc_submitcount; 1446 1447 list_for_each_entry(segment, &desc->segments, node) { 1448 if (chan->ext_addr) 1449 vdma_desc_write_64(chan, 1450 XILINX_VDMA_REG_START_ADDRESS_64(i++), 1451 segment->hw.buf_addr, 1452 segment->hw.buf_addr_msb); 1453 else 1454 vdma_desc_write(chan, 1455 XILINX_VDMA_REG_START_ADDRESS(i++), 1456 segment->hw.buf_addr); 1457 1458 last = segment; 1459 } 1460 1461 if (!last) 1462 return; 1463 1464 /* HW expects these parameters to be same for one transaction */ 1465 vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); 1466 vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE, 1467 last->hw.stride); 1468 vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); 1469 1470 chan->desc_submitcount++; 1471 chan->desc_pendingcount--; 1472 list_move_tail(&desc->node, &chan->active_list); 1473 if (chan->desc_submitcount == chan->num_frms) 1474 chan->desc_submitcount = 0; 1475 1476 chan->idle = false; 1477 } 1478 1479 /** 1480 * xilinx_cdma_start_transfer - Starts cdma transfer 1481 * @chan: Driver specific channel struct pointer 1482 */ 1483 static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan) 1484 { 1485 struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; 1486 struct xilinx_cdma_tx_segment *tail_segment; 1487 u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR); 1488 1489 if (chan->err) 1490 return; 1491 1492 if (!chan->idle) 1493 return; 1494 1495 if (list_empty(&chan->pending_list)) 1496 return; 1497 1498 head_desc = list_first_entry(&chan->pending_list, 1499 struct xilinx_dma_tx_descriptor, node); 1500 tail_desc = list_last_entry(&chan->pending_list, 1501 struct xilinx_dma_tx_descriptor, node); 1502 tail_segment = list_last_entry(&tail_desc->segments, 1503 struct xilinx_cdma_tx_segment, node); 1504 1505 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { 1506 ctrl_reg &= ~XILINX_DMA_CR_COALESCE_MAX; 1507 ctrl_reg |= chan->desc_pendingcount << 1508 XILINX_DMA_CR_COALESCE_SHIFT; 1509 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg); 1510 } 1511 1512 if (chan->has_sg) { 1513 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, 1514 XILINX_CDMA_CR_SGMODE); 1515 1516 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 1517 XILINX_CDMA_CR_SGMODE); 1518 1519 xilinx_write(chan, XILINX_DMA_REG_CURDESC, 1520 head_desc->async_tx.phys); 1521 1522 /* Update tail ptr register which will start the transfer */ 1523 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1524 tail_segment->phys); 1525 } else { 1526 /* In simple mode */ 1527 struct xilinx_cdma_tx_segment *segment; 1528 struct xilinx_cdma_desc_hw *hw; 1529 1530 segment = list_first_entry(&head_desc->segments, 1531 struct xilinx_cdma_tx_segment, 1532 node); 1533 1534 hw = &segment->hw; 1535 1536 xilinx_write(chan, XILINX_CDMA_REG_SRCADDR, 1537 xilinx_prep_dma_addr_t(hw->src_addr)); 1538 xilinx_write(chan, XILINX_CDMA_REG_DSTADDR, 1539 xilinx_prep_dma_addr_t(hw->dest_addr)); 1540 1541 /* Start the transfer */ 1542 dma_ctrl_write(chan, XILINX_DMA_REG_BTT, 1543 hw->control & chan->xdev->max_buffer_len); 1544 } 1545 1546 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1547 chan->desc_pendingcount = 0; 1548 chan->idle = false; 1549 } 1550 1551 /** 1552 * xilinx_dma_start_transfer - Starts DMA transfer 1553 * @chan: Driver specific channel struct pointer 1554 */ 1555 static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) 1556 { 1557 struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; 1558 struct xilinx_axidma_tx_segment *tail_segment; 1559 u32 reg; 1560 1561 if (chan->err) 1562 return; 1563 1564 if (list_empty(&chan->pending_list)) { 1565 if (chan->cyclic) { 1566 struct xilinx_dma_tx_descriptor *desc; 1567 struct list_head *entry; 1568 1569 desc = list_last_entry(&chan->done_list, 1570 struct xilinx_dma_tx_descriptor, node); 1571 list_for_each(entry, &desc->segments) { 1572 struct xilinx_axidma_tx_segment *axidma_seg; 1573 struct xilinx_axidma_desc_hw *axidma_hw; 1574 axidma_seg = list_entry(entry, 1575 struct xilinx_axidma_tx_segment, 1576 node); 1577 axidma_hw = &axidma_seg->hw; 1578 axidma_hw->status = 0; 1579 } 1580 1581 list_splice_tail_init(&chan->done_list, &chan->active_list); 1582 chan->desc_pendingcount = 0; 1583 chan->idle = false; 1584 } 1585 return; 1586 } 1587 1588 /* 1589 * Direct (non-SG) mode has no descriptor queue: writing the BTT 1590 * register launches a transfer immediately, so a new transfer must 1591 * not be programmed while one is in flight. Keep such transfers 1592 * serialized. SG mode supports chaining onto a running transfer via 1593 * tail-pointer extension, so it is allowed to proceed when busy. 1594 */ 1595 if (!chan->has_sg && !chan->idle) 1596 return; 1597 1598 head_desc = list_first_entry(&chan->pending_list, 1599 struct xilinx_dma_tx_descriptor, node); 1600 tail_desc = list_last_entry(&chan->pending_list, 1601 struct xilinx_dma_tx_descriptor, node); 1602 tail_segment = list_last_entry(&tail_desc->segments, 1603 struct xilinx_axidma_tx_segment, node); 1604 1605 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 1606 1607 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { 1608 reg &= ~XILINX_DMA_CR_COALESCE_MAX; 1609 reg |= chan->desc_pendingcount << 1610 XILINX_DMA_CR_COALESCE_SHIFT; 1611 } 1612 1613 if (chan->has_sg && list_empty(&chan->active_list)) 1614 xilinx_write(chan, XILINX_DMA_REG_CURDESC, 1615 head_desc->async_tx.phys); 1616 reg &= ~XILINX_DMA_CR_DELAY_MAX; 1617 reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; 1618 reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; 1619 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 1620 1621 if (chan->idle) 1622 xilinx_dma_start(chan); 1623 1624 if (chan->err) 1625 return; 1626 1627 /* Start the transfer */ 1628 if (chan->has_sg) { 1629 if (chan->cyclic) 1630 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1631 chan->cyclic_seg_v->phys); 1632 else 1633 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1634 tail_segment->phys); 1635 } else { 1636 struct xilinx_axidma_tx_segment *segment; 1637 struct xilinx_axidma_desc_hw *hw; 1638 1639 segment = list_first_entry(&head_desc->segments, 1640 struct xilinx_axidma_tx_segment, 1641 node); 1642 hw = &segment->hw; 1643 1644 xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR, 1645 xilinx_prep_dma_addr_t(hw->buf_addr)); 1646 1647 /* Start the transfer */ 1648 dma_ctrl_write(chan, XILINX_DMA_REG_BTT, 1649 hw->control & chan->xdev->max_buffer_len); 1650 } 1651 1652 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1653 chan->desc_pendingcount = 0; 1654 chan->idle = false; 1655 } 1656 1657 /** 1658 * xilinx_mcdma_start_transfer - Starts MCDMA transfer 1659 * @chan: Driver specific channel struct pointer 1660 */ 1661 static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) 1662 { 1663 struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; 1664 struct xilinx_aximcdma_tx_segment *tail_segment; 1665 u32 reg; 1666 1667 /* 1668 * lock has been held by calling functions, so we don't need it 1669 * to take it here again. 1670 */ 1671 1672 if (chan->err) 1673 return; 1674 1675 if (list_empty(&chan->pending_list)) 1676 return; 1677 1678 head_desc = list_first_entry(&chan->pending_list, 1679 struct xilinx_dma_tx_descriptor, node); 1680 tail_desc = list_last_entry(&chan->pending_list, 1681 struct xilinx_dma_tx_descriptor, node); 1682 tail_segment = list_last_entry(&tail_desc->segments, 1683 struct xilinx_aximcdma_tx_segment, node); 1684 1685 reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest)); 1686 1687 if (chan->desc_pendingcount <= XILINX_MCDMA_COALESCE_MAX) { 1688 reg &= ~XILINX_MCDMA_COALESCE_MASK; 1689 reg |= chan->desc_pendingcount << 1690 XILINX_MCDMA_COALESCE_SHIFT; 1691 } 1692 1693 reg |= XILINX_MCDMA_IRQ_ALL_MASK; 1694 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); 1695 1696 /* Program current descriptor */ 1697 if (chan->has_sg && list_empty(&chan->active_list)) 1698 xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), 1699 head_desc->async_tx.phys); 1700 1701 /* Program channel enable register */ 1702 reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET); 1703 reg |= BIT(chan->tdest); 1704 dma_ctrl_write(chan, XILINX_MCDMA_CHEN_OFFSET, reg); 1705 1706 /* Start the fetch of BDs for the channel */ 1707 reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest)); 1708 reg |= XILINX_MCDMA_CR_RUNSTOP_MASK; 1709 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); 1710 1711 if (chan->idle) 1712 xilinx_dma_start(chan); 1713 1714 if (chan->err) 1715 return; 1716 1717 /* Start the transfer */ 1718 xilinx_write(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET(chan->tdest), 1719 tail_segment->phys); 1720 1721 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1722 chan->desc_pendingcount = 0; 1723 chan->idle = false; 1724 } 1725 1726 /** 1727 * xilinx_dma_issue_pending - Issue pending transactions 1728 * @dchan: DMA channel 1729 */ 1730 static void xilinx_dma_issue_pending(struct dma_chan *dchan) 1731 { 1732 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1733 unsigned long flags; 1734 1735 spin_lock_irqsave(&chan->lock, flags); 1736 chan->start_transfer(chan); 1737 spin_unlock_irqrestore(&chan->lock, flags); 1738 } 1739 1740 /** 1741 * xilinx_dma_device_config - Configure the DMA channel 1742 * @dchan: DMA channel 1743 * @config: channel configuration 1744 * 1745 * Return: 0 always. 1746 */ 1747 static int xilinx_dma_device_config(struct dma_chan *dchan, 1748 struct dma_slave_config *config) 1749 { 1750 return 0; 1751 } 1752 1753 /** 1754 * xilinx_dma_complete_descriptor - Mark the active descriptor as complete 1755 * @chan : xilinx DMA channel 1756 * 1757 * CONTEXT: hardirq 1758 */ 1759 static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) 1760 { 1761 struct xilinx_dma_tx_descriptor *desc, *next; 1762 1763 /* This function was invoked with lock held */ 1764 if (list_empty(&chan->active_list)) 1765 return; 1766 1767 list_for_each_entry_safe(desc, next, &chan->active_list, node) { 1768 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 1769 struct xilinx_axidma_tx_segment *seg; 1770 1771 seg = list_last_entry(&desc->segments, 1772 struct xilinx_axidma_tx_segment, node); 1773 if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) 1774 break; 1775 } 1776 if (chan->has_sg && chan->xdev->dma_config->dmatype != 1777 XDMA_TYPE_VDMA) 1778 desc->residue = xilinx_dma_get_residue(chan, desc); 1779 else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA && 1780 chan->direction == DMA_DEV_TO_MEM && !chan->has_sg) 1781 desc->residue = xilinx_dma_get_residue_axidma_direct_s2mm(chan, desc); 1782 else 1783 desc->residue = 0; 1784 desc->err = chan->err; 1785 1786 list_del(&desc->node); 1787 if (!desc->cyclic) 1788 dma_cookie_complete(&desc->async_tx); 1789 list_add_tail(&desc->node, &chan->done_list); 1790 } 1791 } 1792 1793 /** 1794 * xilinx_dma_reset - Reset DMA channel 1795 * @chan: Driver specific DMA channel 1796 * 1797 * Return: '0' on success and failure value on error 1798 */ 1799 static int xilinx_dma_reset(struct xilinx_dma_chan *chan) 1800 { 1801 int err; 1802 u32 tmp; 1803 1804 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET); 1805 1806 /* Wait for the hardware to finish reset */ 1807 err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, 1808 !(tmp & XILINX_DMA_DMACR_RESET), 1809 XILINX_DMA_POLL_DELAY_US, 1810 XILINX_DMA_POLL_TIMEOUT_US); 1811 1812 if (err) { 1813 dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", 1814 dma_ctrl_read(chan, XILINX_DMA_REG_DMACR), 1815 dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 1816 return -ETIMEDOUT; 1817 } 1818 1819 chan->err = false; 1820 chan->idle = true; 1821 chan->desc_pendingcount = 0; 1822 chan->desc_submitcount = 0; 1823 1824 return err; 1825 } 1826 1827 /** 1828 * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts 1829 * @chan: Driver specific DMA channel 1830 * 1831 * Return: '0' on success and failure value on error 1832 */ 1833 static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan) 1834 { 1835 int err; 1836 1837 /* Reset VDMA */ 1838 err = xilinx_dma_reset(chan); 1839 if (err) 1840 return err; 1841 1842 /* Enable interrupts */ 1843 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 1844 XILINX_DMA_DMAXR_ALL_IRQ_MASK); 1845 1846 return 0; 1847 } 1848 1849 /** 1850 * xilinx_mcdma_irq_handler - MCDMA Interrupt handler 1851 * @irq: IRQ number 1852 * @data: Pointer to the Xilinx MCDMA channel structure 1853 * 1854 * Return: IRQ_HANDLED/IRQ_NONE 1855 */ 1856 static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) 1857 { 1858 struct xilinx_dma_chan *chan = data; 1859 u32 status, ser_offset, chan_sermask, chan_offset = 0, chan_id; 1860 1861 if (chan->direction == DMA_DEV_TO_MEM) 1862 ser_offset = XILINX_MCDMA_RXINT_SER_OFFSET; 1863 else 1864 ser_offset = XILINX_MCDMA_TXINT_SER_OFFSET; 1865 1866 /* Read the channel id raising the interrupt*/ 1867 chan_sermask = dma_ctrl_read(chan, ser_offset); 1868 chan_id = ffs(chan_sermask); 1869 1870 if (!chan_id) 1871 return IRQ_NONE; 1872 1873 if (chan->direction == DMA_DEV_TO_MEM) 1874 chan_offset = chan->xdev->dma_config->max_channels / 2; 1875 1876 chan_offset = chan_offset + (chan_id - 1); 1877 chan = chan->xdev->chan[chan_offset]; 1878 /* Read the status and ack the interrupts. */ 1879 status = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest)); 1880 if (!(status & XILINX_MCDMA_IRQ_ALL_MASK)) 1881 return IRQ_NONE; 1882 1883 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest), 1884 status & XILINX_MCDMA_IRQ_ALL_MASK); 1885 1886 if (status & XILINX_MCDMA_IRQ_ERR_MASK) { 1887 dev_err(chan->dev, "Channel %p has errors %x cdr %x tdr %x\n", 1888 chan, 1889 dma_ctrl_read(chan, XILINX_MCDMA_CH_ERR_OFFSET), 1890 dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET 1891 (chan->tdest)), 1892 dma_ctrl_read(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET 1893 (chan->tdest))); 1894 chan->err = true; 1895 } 1896 1897 if (status & XILINX_MCDMA_IRQ_DELAY_MASK) { 1898 /* 1899 * Device takes too long to do the transfer when user requires 1900 * responsiveness. 1901 */ 1902 dev_dbg(chan->dev, "Inter-packet latency too long\n"); 1903 } 1904 1905 if (status & XILINX_MCDMA_IRQ_IOC_MASK) { 1906 spin_lock(&chan->lock); 1907 xilinx_dma_complete_descriptor(chan); 1908 if (list_empty(&chan->active_list)) { 1909 chan->idle = true; 1910 chan->start_transfer(chan); 1911 } 1912 spin_unlock(&chan->lock); 1913 } 1914 1915 tasklet_hi_schedule(&chan->tasklet); 1916 return IRQ_HANDLED; 1917 } 1918 1919 /** 1920 * xilinx_dma_irq_handler - DMA Interrupt handler 1921 * @irq: IRQ number 1922 * @data: Pointer to the Xilinx DMA channel structure 1923 * 1924 * Return: IRQ_HANDLED/IRQ_NONE 1925 */ 1926 static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) 1927 { 1928 struct xilinx_dma_chan *chan = data; 1929 u32 status; 1930 1931 /* Read the status and ack the interrupts. */ 1932 status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR); 1933 if (!(status & XILINX_DMA_DMAXR_ALL_IRQ_MASK)) 1934 return IRQ_NONE; 1935 1936 dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, 1937 status & XILINX_DMA_DMAXR_ALL_IRQ_MASK); 1938 1939 if (status & XILINX_DMA_DMASR_ERR_IRQ) { 1940 /* 1941 * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the 1942 * error is recoverable, ignore it. Otherwise flag the error. 1943 * 1944 * Only recoverable errors can be cleared in the DMASR register, 1945 * make sure not to write to other error bits to 1. 1946 */ 1947 u32 errors = status & XILINX_DMA_DMASR_ALL_ERR_MASK; 1948 1949 dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, 1950 errors & XILINX_DMA_DMASR_ERR_RECOVER_MASK); 1951 1952 if (!chan->flush_on_fsync || 1953 (errors & ~XILINX_DMA_DMASR_ERR_RECOVER_MASK)) { 1954 dev_err(chan->dev, 1955 "Channel %p has errors %x, cdr %x tdr %x\n", 1956 chan, errors, 1957 dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC), 1958 dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC)); 1959 chan->err = true; 1960 } 1961 } 1962 1963 if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ | 1964 XILINX_DMA_DMASR_DLY_CNT_IRQ)) { 1965 spin_lock(&chan->lock); 1966 xilinx_dma_complete_descriptor(chan); 1967 if (list_empty(&chan->active_list)) { 1968 chan->idle = true; 1969 chan->start_transfer(chan); 1970 } 1971 spin_unlock(&chan->lock); 1972 } 1973 1974 tasklet_schedule(&chan->tasklet); 1975 return IRQ_HANDLED; 1976 } 1977 1978 /** 1979 * append_desc_queue - Queuing descriptor 1980 * @chan: Driver specific dma channel 1981 * @desc: dma transaction descriptor 1982 */ 1983 static void append_desc_queue(struct xilinx_dma_chan *chan, 1984 struct xilinx_dma_tx_descriptor *desc) 1985 { 1986 struct xilinx_vdma_tx_segment *tail_segment; 1987 struct xilinx_dma_tx_descriptor *tail_desc; 1988 struct xilinx_axidma_tx_segment *axidma_tail_segment; 1989 struct xilinx_aximcdma_tx_segment *aximcdma_tail_segment; 1990 struct xilinx_cdma_tx_segment *cdma_tail_segment; 1991 1992 if (list_empty(&chan->pending_list)) 1993 goto append; 1994 1995 /* 1996 * Add the hardware descriptor to the chain of hardware descriptors 1997 * that already exists in memory. 1998 */ 1999 tail_desc = list_last_entry(&chan->pending_list, 2000 struct xilinx_dma_tx_descriptor, node); 2001 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 2002 tail_segment = list_last_entry(&tail_desc->segments, 2003 struct xilinx_vdma_tx_segment, 2004 node); 2005 tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 2006 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 2007 cdma_tail_segment = list_last_entry(&tail_desc->segments, 2008 struct xilinx_cdma_tx_segment, 2009 node); 2010 cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 2011 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 2012 axidma_tail_segment = list_last_entry(&tail_desc->segments, 2013 struct xilinx_axidma_tx_segment, 2014 node); 2015 axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 2016 } else { 2017 aximcdma_tail_segment = 2018 list_last_entry(&tail_desc->segments, 2019 struct xilinx_aximcdma_tx_segment, 2020 node); 2021 aximcdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 2022 } 2023 2024 /* 2025 * Add the software descriptor and all children to the list 2026 * of pending transactions 2027 */ 2028 append: 2029 list_add_tail(&desc->node, &chan->pending_list); 2030 chan->desc_pendingcount++; 2031 2032 if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) 2033 && unlikely(chan->desc_pendingcount > chan->num_frms)) { 2034 dev_dbg(chan->dev, "desc pendingcount is too high\n"); 2035 chan->desc_pendingcount = chan->num_frms; 2036 } 2037 } 2038 2039 /** 2040 * xilinx_dma_tx_submit - Submit DMA transaction 2041 * @tx: Async transaction descriptor 2042 * 2043 * Return: cookie value on success and failure value on error 2044 */ 2045 static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx) 2046 { 2047 struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); 2048 struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan); 2049 dma_cookie_t cookie; 2050 unsigned long flags; 2051 int err; 2052 2053 if (chan->cyclic) { 2054 xilinx_dma_free_tx_descriptor(chan, desc); 2055 return -EBUSY; 2056 } 2057 2058 if (chan->err) { 2059 /* 2060 * If reset fails, need to hard reset the system. 2061 * Channel is no longer functional 2062 */ 2063 err = xilinx_dma_chan_reset(chan); 2064 if (err < 0) 2065 return err; 2066 } 2067 2068 spin_lock_irqsave(&chan->lock, flags); 2069 2070 cookie = dma_cookie_assign(tx); 2071 2072 /* Put this transaction onto the tail of the pending queue */ 2073 append_desc_queue(chan, desc); 2074 2075 if (desc->cyclic) 2076 chan->cyclic = true; 2077 2078 chan->terminating = false; 2079 2080 spin_unlock_irqrestore(&chan->lock, flags); 2081 2082 return cookie; 2083 } 2084 2085 /** 2086 * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a 2087 * DMA_SLAVE transaction 2088 * @dchan: DMA channel 2089 * @xt: Interleaved template pointer 2090 * @flags: transfer ack flags 2091 * 2092 * Return: Async transaction descriptor on success and NULL on failure 2093 */ 2094 static struct dma_async_tx_descriptor * 2095 xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, 2096 struct dma_interleaved_template *xt, 2097 unsigned long flags) 2098 { 2099 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2100 struct xilinx_dma_tx_descriptor *desc; 2101 struct xilinx_vdma_tx_segment *segment; 2102 struct xilinx_vdma_desc_hw *hw; 2103 2104 if (!is_slave_direction(xt->dir)) 2105 return NULL; 2106 2107 if (!xt->numf || !xt->sgl[0].size) 2108 return NULL; 2109 2110 if (xt->numf & ~XILINX_DMA_VSIZE_MASK || 2111 xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK) 2112 return NULL; 2113 2114 if (xt->frame_size != 1) 2115 return NULL; 2116 2117 /* Allocate a transaction descriptor. */ 2118 desc = xilinx_dma_alloc_tx_descriptor(chan); 2119 if (!desc) 2120 return NULL; 2121 2122 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2123 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2124 async_tx_ack(&desc->async_tx); 2125 2126 /* Allocate the link descriptor from DMA pool */ 2127 segment = xilinx_vdma_alloc_tx_segment(chan); 2128 if (!segment) 2129 goto error; 2130 2131 /* Fill in the hardware descriptor */ 2132 hw = &segment->hw; 2133 hw->vsize = xt->numf; 2134 hw->hsize = xt->sgl[0].size; 2135 hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) << 2136 XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT; 2137 hw->stride |= chan->config.frm_dly << 2138 XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT; 2139 2140 if (xt->dir != DMA_MEM_TO_DEV) { 2141 if (chan->ext_addr) { 2142 hw->buf_addr = lower_32_bits(xt->dst_start); 2143 hw->buf_addr_msb = upper_32_bits(xt->dst_start); 2144 } else { 2145 hw->buf_addr = xt->dst_start; 2146 } 2147 } else { 2148 if (chan->ext_addr) { 2149 hw->buf_addr = lower_32_bits(xt->src_start); 2150 hw->buf_addr_msb = upper_32_bits(xt->src_start); 2151 } else { 2152 hw->buf_addr = xt->src_start; 2153 } 2154 } 2155 2156 /* Insert the segment into the descriptor segments list. */ 2157 list_add_tail(&segment->node, &desc->segments); 2158 2159 /* Link the last hardware descriptor with the first. */ 2160 segment = list_first_entry(&desc->segments, 2161 struct xilinx_vdma_tx_segment, node); 2162 desc->async_tx.phys = segment->phys; 2163 2164 return &desc->async_tx; 2165 2166 error: 2167 xilinx_dma_free_tx_descriptor(chan, desc); 2168 return NULL; 2169 } 2170 2171 /** 2172 * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction 2173 * @dchan: DMA channel 2174 * @dma_dst: destination address 2175 * @dma_src: source address 2176 * @len: transfer length 2177 * @flags: transfer ack flags 2178 * 2179 * Return: Async transaction descriptor on success and NULL on failure 2180 */ 2181 static struct dma_async_tx_descriptor * 2182 xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, 2183 dma_addr_t dma_src, size_t len, unsigned long flags) 2184 { 2185 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2186 struct xilinx_dma_tx_descriptor *desc; 2187 struct xilinx_cdma_tx_segment *segment; 2188 struct xilinx_cdma_desc_hw *hw; 2189 2190 if (!len || len > chan->xdev->max_buffer_len) 2191 return NULL; 2192 2193 desc = xilinx_dma_alloc_tx_descriptor(chan); 2194 if (!desc) 2195 return NULL; 2196 2197 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2198 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2199 2200 /* Allocate the link descriptor from DMA pool */ 2201 segment = xilinx_cdma_alloc_tx_segment(chan); 2202 if (!segment) 2203 goto error; 2204 2205 hw = &segment->hw; 2206 hw->control = len; 2207 hw->src_addr = dma_src; 2208 hw->dest_addr = dma_dst; 2209 if (chan->ext_addr) { 2210 hw->src_addr_msb = upper_32_bits(dma_src); 2211 hw->dest_addr_msb = upper_32_bits(dma_dst); 2212 } 2213 2214 /* Insert the segment into the descriptor segments list. */ 2215 list_add_tail(&segment->node, &desc->segments); 2216 2217 desc->async_tx.phys = segment->phys; 2218 hw->next_desc = segment->phys; 2219 2220 return &desc->async_tx; 2221 2222 error: 2223 xilinx_dma_free_tx_descriptor(chan, desc); 2224 return NULL; 2225 } 2226 2227 /** 2228 * xilinx_dma_prep_peripheral_dma_vec - prepare descriptors for a DMA_SLAVE 2229 * transaction from DMA vectors 2230 * @dchan: DMA channel 2231 * @vecs: Array of DMA vectors that should be transferred 2232 * @nb: number of entries in @vecs 2233 * @direction: DMA direction 2234 * @flags: transfer ack flags 2235 * 2236 * Return: Async transaction descriptor on success and NULL on failure 2237 */ 2238 static struct dma_async_tx_descriptor *xilinx_dma_prep_peripheral_dma_vec( 2239 struct dma_chan *dchan, const struct dma_vec *vecs, size_t nb, 2240 enum dma_transfer_direction direction, unsigned long flags) 2241 { 2242 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2243 struct xilinx_dma_tx_descriptor *desc; 2244 struct xilinx_axidma_tx_segment *segment, *head, *prev = NULL; 2245 size_t copy; 2246 size_t sg_used; 2247 unsigned int i; 2248 2249 if (!is_slave_direction(direction) || direction != chan->direction) 2250 return NULL; 2251 2252 desc = xilinx_dma_alloc_tx_descriptor(chan); 2253 if (!desc) 2254 return NULL; 2255 2256 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2257 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2258 2259 /* Build transactions using information from DMA vectors */ 2260 for (i = 0; i < nb; i++) { 2261 sg_used = 0; 2262 2263 /* Loop until the entire dma_vec entry is used */ 2264 while (sg_used < vecs[i].len) { 2265 struct xilinx_axidma_desc_hw *hw; 2266 2267 /* Get a free segment */ 2268 segment = xilinx_axidma_alloc_tx_segment(chan); 2269 if (!segment) 2270 goto error; 2271 2272 /* 2273 * Calculate the maximum number of bytes to transfer, 2274 * making sure it is less than the hw limit 2275 */ 2276 copy = xilinx_dma_calc_copysize(chan, vecs[i].len, 2277 sg_used); 2278 hw = &segment->hw; 2279 2280 /* Fill in the descriptor */ 2281 xilinx_axidma_buf(chan, hw, vecs[i].addr, sg_used, 0); 2282 hw->control = copy; 2283 2284 if (prev) 2285 prev->hw.next_desc = segment->phys; 2286 2287 prev = segment; 2288 sg_used += copy; 2289 2290 /* 2291 * Insert the segment into the descriptor segments 2292 * list. 2293 */ 2294 list_add_tail(&segment->node, &desc->segments); 2295 } 2296 } 2297 2298 head = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, node); 2299 desc->async_tx.phys = head->phys; 2300 2301 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 2302 if (chan->direction == DMA_MEM_TO_DEV) { 2303 segment->hw.control |= XILINX_DMA_BD_SOP; 2304 segment = list_last_entry(&desc->segments, 2305 struct xilinx_axidma_tx_segment, 2306 node); 2307 segment->hw.control |= XILINX_DMA_BD_EOP; 2308 } 2309 2310 if (chan->xdev->has_axistream_connected) 2311 desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; 2312 2313 return &desc->async_tx; 2314 2315 error: 2316 xilinx_dma_free_tx_descriptor(chan, desc); 2317 return NULL; 2318 } 2319 2320 /** 2321 * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 2322 * @dchan: DMA channel 2323 * @sgl: scatterlist to transfer to/from 2324 * @sg_len: number of entries in @scatterlist 2325 * @direction: DMA direction 2326 * @flags: transfer ack flags 2327 * @context: APP words of the descriptor 2328 * 2329 * Return: Async transaction descriptor on success and NULL on failure 2330 */ 2331 static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( 2332 struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, 2333 enum dma_transfer_direction direction, unsigned long flags, 2334 void *context) 2335 { 2336 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2337 struct xilinx_dma_tx_descriptor *desc; 2338 struct xilinx_axidma_tx_segment *segment = NULL; 2339 u32 *app_w = (u32 *)context; 2340 struct scatterlist *sg; 2341 size_t copy; 2342 size_t sg_used; 2343 unsigned int i; 2344 2345 if (!is_slave_direction(direction)) 2346 return NULL; 2347 2348 /* Allocate a transaction descriptor. */ 2349 desc = xilinx_dma_alloc_tx_descriptor(chan); 2350 if (!desc) 2351 return NULL; 2352 2353 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2354 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2355 2356 /* Build transactions using information in the scatter gather list */ 2357 for_each_sg(sgl, sg, sg_len, i) { 2358 sg_used = 0; 2359 2360 /* Loop until the entire scatterlist entry is used */ 2361 while (sg_used < sg_dma_len(sg)) { 2362 struct xilinx_axidma_desc_hw *hw; 2363 2364 /* Get a free segment */ 2365 segment = xilinx_axidma_alloc_tx_segment(chan); 2366 if (!segment) 2367 goto error; 2368 2369 /* 2370 * Calculate the maximum number of bytes to transfer, 2371 * making sure it is less than the hw limit 2372 */ 2373 copy = xilinx_dma_calc_copysize(chan, sg_dma_len(sg), 2374 sg_used); 2375 hw = &segment->hw; 2376 2377 /* Fill in the descriptor */ 2378 xilinx_axidma_buf(chan, hw, sg_dma_address(sg), 2379 sg_used, 0); 2380 2381 hw->control = copy; 2382 2383 if (chan->direction == DMA_MEM_TO_DEV) { 2384 if (app_w) 2385 memcpy(hw->app, app_w, sizeof(u32) * 2386 XILINX_DMA_NUM_APP_WORDS); 2387 } 2388 2389 sg_used += copy; 2390 2391 /* 2392 * Insert the segment into the descriptor segments 2393 * list. 2394 */ 2395 list_add_tail(&segment->node, &desc->segments); 2396 } 2397 } 2398 2399 segment = list_first_entry(&desc->segments, 2400 struct xilinx_axidma_tx_segment, node); 2401 desc->async_tx.phys = segment->phys; 2402 2403 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 2404 if (chan->direction == DMA_MEM_TO_DEV) { 2405 segment->hw.control |= XILINX_DMA_BD_SOP; 2406 segment = list_last_entry(&desc->segments, 2407 struct xilinx_axidma_tx_segment, 2408 node); 2409 segment->hw.control |= XILINX_DMA_BD_EOP; 2410 } 2411 2412 if (chan->xdev->has_axistream_connected) 2413 desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; 2414 2415 return &desc->async_tx; 2416 2417 error: 2418 xilinx_dma_free_tx_descriptor(chan, desc); 2419 return NULL; 2420 } 2421 2422 /** 2423 * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction 2424 * @dchan: DMA channel 2425 * @buf_addr: Physical address of the buffer 2426 * @buf_len: Total length of the cyclic buffers 2427 * @period_len: length of individual cyclic buffer 2428 * @direction: DMA direction 2429 * @flags: transfer ack flags 2430 * 2431 * Return: Async transaction descriptor on success and NULL on failure 2432 */ 2433 static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic( 2434 struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len, 2435 size_t period_len, enum dma_transfer_direction direction, 2436 unsigned long flags) 2437 { 2438 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2439 struct xilinx_dma_tx_descriptor *desc; 2440 struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL; 2441 size_t copy, sg_used; 2442 unsigned int num_periods; 2443 int i; 2444 u32 reg; 2445 2446 if (!period_len) 2447 return NULL; 2448 2449 num_periods = buf_len / period_len; 2450 2451 if (!num_periods) 2452 return NULL; 2453 2454 if (!is_slave_direction(direction)) 2455 return NULL; 2456 2457 /* Allocate a transaction descriptor. */ 2458 desc = xilinx_dma_alloc_tx_descriptor(chan); 2459 if (!desc) 2460 return NULL; 2461 2462 chan->direction = direction; 2463 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2464 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2465 2466 for (i = 0; i < num_periods; ++i) { 2467 sg_used = 0; 2468 2469 while (sg_used < period_len) { 2470 struct xilinx_axidma_desc_hw *hw; 2471 2472 /* Get a free segment */ 2473 segment = xilinx_axidma_alloc_tx_segment(chan); 2474 if (!segment) 2475 goto error; 2476 2477 /* 2478 * Calculate the maximum number of bytes to transfer, 2479 * making sure it is less than the hw limit 2480 */ 2481 copy = xilinx_dma_calc_copysize(chan, period_len, 2482 sg_used); 2483 hw = &segment->hw; 2484 xilinx_axidma_buf(chan, hw, buf_addr, sg_used, 2485 period_len * i); 2486 hw->control = copy; 2487 2488 if (prev) 2489 prev->hw.next_desc = segment->phys; 2490 2491 prev = segment; 2492 sg_used += copy; 2493 2494 /* 2495 * Insert the segment into the descriptor segments 2496 * list. 2497 */ 2498 list_add_tail(&segment->node, &desc->segments); 2499 } 2500 } 2501 2502 head_segment = list_first_entry(&desc->segments, 2503 struct xilinx_axidma_tx_segment, node); 2504 desc->async_tx.phys = head_segment->phys; 2505 2506 desc->cyclic = true; 2507 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 2508 reg |= XILINX_DMA_CR_CYCLIC_BD_EN_MASK; 2509 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 2510 2511 segment = list_last_entry(&desc->segments, 2512 struct xilinx_axidma_tx_segment, 2513 node); 2514 segment->hw.next_desc = (u32) head_segment->phys; 2515 2516 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 2517 if (direction == DMA_MEM_TO_DEV) { 2518 head_segment->hw.control |= XILINX_DMA_BD_SOP; 2519 segment->hw.control |= XILINX_DMA_BD_EOP; 2520 } 2521 2522 return &desc->async_tx; 2523 2524 error: 2525 xilinx_dma_free_tx_descriptor(chan, desc); 2526 return NULL; 2527 } 2528 2529 /** 2530 * xilinx_mcdma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 2531 * @dchan: DMA channel 2532 * @sgl: scatterlist to transfer to/from 2533 * @sg_len: number of entries in @scatterlist 2534 * @direction: DMA direction 2535 * @flags: transfer ack flags 2536 * @context: APP words of the descriptor 2537 * 2538 * Return: Async transaction descriptor on success and NULL on failure 2539 */ 2540 static struct dma_async_tx_descriptor * 2541 xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 2542 unsigned int sg_len, 2543 enum dma_transfer_direction direction, 2544 unsigned long flags, void *context) 2545 { 2546 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2547 struct xilinx_dma_tx_descriptor *desc; 2548 struct xilinx_aximcdma_tx_segment *segment = NULL; 2549 u32 *app_w = (u32 *)context; 2550 struct scatterlist *sg; 2551 size_t copy; 2552 size_t sg_used; 2553 unsigned int i; 2554 2555 if (!is_slave_direction(direction)) 2556 return NULL; 2557 2558 /* Allocate a transaction descriptor. */ 2559 desc = xilinx_dma_alloc_tx_descriptor(chan); 2560 if (!desc) 2561 return NULL; 2562 2563 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 2564 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 2565 2566 /* Build transactions using information in the scatter gather list */ 2567 for_each_sg(sgl, sg, sg_len, i) { 2568 sg_used = 0; 2569 2570 /* Loop until the entire scatterlist entry is used */ 2571 while (sg_used < sg_dma_len(sg)) { 2572 struct xilinx_aximcdma_desc_hw *hw; 2573 2574 /* Get a free segment */ 2575 segment = xilinx_aximcdma_alloc_tx_segment(chan); 2576 if (!segment) 2577 goto error; 2578 2579 /* 2580 * Calculate the maximum number of bytes to transfer, 2581 * making sure it is less than the hw limit 2582 */ 2583 copy = min_t(size_t, sg_dma_len(sg) - sg_used, 2584 chan->xdev->max_buffer_len); 2585 hw = &segment->hw; 2586 2587 /* Fill in the descriptor */ 2588 xilinx_aximcdma_buf(chan, hw, sg_dma_address(sg), 2589 sg_used); 2590 hw->control = copy; 2591 2592 if (chan->direction == DMA_MEM_TO_DEV && app_w) { 2593 memcpy(hw->app, app_w, sizeof(u32) * 2594 XILINX_DMA_NUM_APP_WORDS); 2595 } 2596 2597 sg_used += copy; 2598 /* 2599 * Insert the segment into the descriptor segments 2600 * list. 2601 */ 2602 list_add_tail(&segment->node, &desc->segments); 2603 } 2604 } 2605 2606 segment = list_first_entry(&desc->segments, 2607 struct xilinx_aximcdma_tx_segment, node); 2608 desc->async_tx.phys = segment->phys; 2609 2610 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 2611 if (chan->direction == DMA_MEM_TO_DEV) { 2612 segment->hw.control |= XILINX_MCDMA_BD_SOP; 2613 segment = list_last_entry(&desc->segments, 2614 struct xilinx_aximcdma_tx_segment, 2615 node); 2616 segment->hw.control |= XILINX_MCDMA_BD_EOP; 2617 } 2618 2619 return &desc->async_tx; 2620 2621 error: 2622 xilinx_dma_free_tx_descriptor(chan, desc); 2623 2624 return NULL; 2625 } 2626 2627 /** 2628 * xilinx_dma_terminate_all - Halt the channel and free descriptors 2629 * @dchan: Driver specific DMA Channel pointer 2630 * 2631 * Return: '0' always. 2632 */ 2633 static int xilinx_dma_terminate_all(struct dma_chan *dchan) 2634 { 2635 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2636 u32 reg; 2637 int err; 2638 2639 if (!chan->cyclic) { 2640 err = chan->stop_transfer(chan); 2641 if (err) { 2642 dev_err(chan->dev, "Cannot stop channel %p: %x\n", 2643 chan, dma_ctrl_read(chan, 2644 XILINX_DMA_REG_DMASR)); 2645 chan->err = true; 2646 } 2647 } 2648 2649 xilinx_dma_chan_reset(chan); 2650 /* Remove and free all of the descriptors in the lists */ 2651 chan->terminating = true; 2652 xilinx_dma_free_descriptors(chan); 2653 chan->idle = true; 2654 2655 if (chan->cyclic) { 2656 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 2657 reg &= ~XILINX_DMA_CR_CYCLIC_BD_EN_MASK; 2658 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 2659 chan->cyclic = false; 2660 } 2661 2662 if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) 2663 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, 2664 XILINX_CDMA_CR_SGMODE); 2665 2666 return 0; 2667 } 2668 2669 static void xilinx_dma_synchronize(struct dma_chan *dchan) 2670 { 2671 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2672 2673 tasklet_kill(&chan->tasklet); 2674 } 2675 2676 /** 2677 * xilinx_vdma_channel_set_config - Configure VDMA channel 2678 * Run-time configuration for Axi VDMA, supports: 2679 * . halt the channel 2680 * . configure interrupt coalescing and inter-packet delay threshold 2681 * . start/stop parking 2682 * . enable genlock 2683 * 2684 * @dchan: DMA channel 2685 * @cfg: VDMA device configuration pointer 2686 * 2687 * Return: '0' on success and failure value on error 2688 */ 2689 int xilinx_vdma_channel_set_config(struct dma_chan *dchan, 2690 struct xilinx_vdma_config *cfg) 2691 { 2692 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2693 u32 dmacr; 2694 2695 if (cfg->reset) 2696 return xilinx_dma_chan_reset(chan); 2697 2698 dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 2699 2700 chan->config.frm_dly = cfg->frm_dly; 2701 chan->config.park = cfg->park; 2702 2703 /* genlock settings */ 2704 chan->config.gen_lock = cfg->gen_lock; 2705 chan->config.master = cfg->master; 2706 2707 dmacr &= ~XILINX_DMA_DMACR_GENLOCK_EN; 2708 if (cfg->gen_lock && chan->genlock) { 2709 dmacr |= XILINX_DMA_DMACR_GENLOCK_EN; 2710 dmacr &= ~XILINX_DMA_DMACR_MASTER_MASK; 2711 dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT; 2712 } 2713 2714 chan->config.frm_cnt_en = cfg->frm_cnt_en; 2715 chan->config.vflip_en = cfg->vflip_en; 2716 2717 if (cfg->park) 2718 chan->config.park_frm = cfg->park_frm; 2719 else 2720 chan->config.park_frm = -1; 2721 2722 chan->config.coalesc = cfg->coalesc; 2723 chan->config.delay = cfg->delay; 2724 2725 if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) { 2726 dmacr &= ~XILINX_DMA_DMACR_FRAME_COUNT_MASK; 2727 dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT; 2728 chan->config.coalesc = cfg->coalesc; 2729 } 2730 2731 if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) { 2732 dmacr &= ~XILINX_DMA_DMACR_DELAY_MASK; 2733 dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT; 2734 chan->config.delay = cfg->delay; 2735 } 2736 2737 /* FSync Source selection */ 2738 dmacr &= ~XILINX_DMA_DMACR_FSYNCSRC_MASK; 2739 dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT; 2740 2741 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr); 2742 2743 return 0; 2744 } 2745 EXPORT_SYMBOL(xilinx_vdma_channel_set_config); 2746 2747 /* ----------------------------------------------------------------------------- 2748 * Probe and remove 2749 */ 2750 2751 /** 2752 * xilinx_dma_chan_remove - Per Channel remove function 2753 * @chan: Driver specific DMA channel 2754 */ 2755 static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan) 2756 { 2757 /* Disable all interrupts */ 2758 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, 2759 XILINX_DMA_DMAXR_ALL_IRQ_MASK); 2760 2761 if (chan->irq > 0) 2762 free_irq(chan->irq, chan); 2763 2764 tasklet_kill(&chan->tasklet); 2765 2766 list_del(&chan->common.device_node); 2767 } 2768 2769 static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2770 struct clk **tx_clk, struct clk **rx_clk, 2771 struct clk **sg_clk, struct clk **tmp_clk) 2772 { 2773 int err; 2774 2775 *tmp_clk = NULL; 2776 2777 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2778 if (IS_ERR(*axi_clk)) 2779 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); 2780 2781 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); 2782 if (IS_ERR(*tx_clk)) 2783 *tx_clk = NULL; 2784 2785 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); 2786 if (IS_ERR(*rx_clk)) 2787 *rx_clk = NULL; 2788 2789 *sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk"); 2790 if (IS_ERR(*sg_clk)) 2791 *sg_clk = NULL; 2792 2793 err = clk_prepare_enable(*axi_clk); 2794 if (err) { 2795 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err); 2796 return err; 2797 } 2798 2799 err = clk_prepare_enable(*tx_clk); 2800 if (err) { 2801 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 2802 goto err_disable_axiclk; 2803 } 2804 2805 err = clk_prepare_enable(*rx_clk); 2806 if (err) { 2807 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 2808 goto err_disable_txclk; 2809 } 2810 2811 err = clk_prepare_enable(*sg_clk); 2812 if (err) { 2813 dev_err(&pdev->dev, "failed to enable sg_clk (%d)\n", err); 2814 goto err_disable_rxclk; 2815 } 2816 2817 return 0; 2818 2819 err_disable_rxclk: 2820 clk_disable_unprepare(*rx_clk); 2821 err_disable_txclk: 2822 clk_disable_unprepare(*tx_clk); 2823 err_disable_axiclk: 2824 clk_disable_unprepare(*axi_clk); 2825 2826 return err; 2827 } 2828 2829 static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2830 struct clk **dev_clk, struct clk **tmp_clk, 2831 struct clk **tmp1_clk, struct clk **tmp2_clk) 2832 { 2833 int err; 2834 2835 *tmp_clk = NULL; 2836 *tmp1_clk = NULL; 2837 *tmp2_clk = NULL; 2838 2839 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2840 if (IS_ERR(*axi_clk)) 2841 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); 2842 2843 *dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk"); 2844 if (IS_ERR(*dev_clk)) 2845 return dev_err_probe(&pdev->dev, PTR_ERR(*dev_clk), "failed to get dev_clk\n"); 2846 2847 err = clk_prepare_enable(*axi_clk); 2848 if (err) { 2849 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err); 2850 return err; 2851 } 2852 2853 err = clk_prepare_enable(*dev_clk); 2854 if (err) { 2855 dev_err(&pdev->dev, "failed to enable dev_clk (%d)\n", err); 2856 goto err_disable_axiclk; 2857 } 2858 2859 return 0; 2860 2861 err_disable_axiclk: 2862 clk_disable_unprepare(*axi_clk); 2863 2864 return err; 2865 } 2866 2867 static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2868 struct clk **tx_clk, struct clk **txs_clk, 2869 struct clk **rx_clk, struct clk **rxs_clk) 2870 { 2871 int err; 2872 2873 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2874 if (IS_ERR(*axi_clk)) 2875 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); 2876 2877 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); 2878 if (IS_ERR(*tx_clk)) 2879 *tx_clk = NULL; 2880 2881 *txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk"); 2882 if (IS_ERR(*txs_clk)) 2883 *txs_clk = NULL; 2884 2885 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); 2886 if (IS_ERR(*rx_clk)) 2887 *rx_clk = NULL; 2888 2889 *rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk"); 2890 if (IS_ERR(*rxs_clk)) 2891 *rxs_clk = NULL; 2892 2893 err = clk_prepare_enable(*axi_clk); 2894 if (err) { 2895 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", 2896 err); 2897 return err; 2898 } 2899 2900 err = clk_prepare_enable(*tx_clk); 2901 if (err) { 2902 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 2903 goto err_disable_axiclk; 2904 } 2905 2906 err = clk_prepare_enable(*txs_clk); 2907 if (err) { 2908 dev_err(&pdev->dev, "failed to enable txs_clk (%d)\n", err); 2909 goto err_disable_txclk; 2910 } 2911 2912 err = clk_prepare_enable(*rx_clk); 2913 if (err) { 2914 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 2915 goto err_disable_txsclk; 2916 } 2917 2918 err = clk_prepare_enable(*rxs_clk); 2919 if (err) { 2920 dev_err(&pdev->dev, "failed to enable rxs_clk (%d)\n", err); 2921 goto err_disable_rxclk; 2922 } 2923 2924 return 0; 2925 2926 err_disable_rxclk: 2927 clk_disable_unprepare(*rx_clk); 2928 err_disable_txsclk: 2929 clk_disable_unprepare(*txs_clk); 2930 err_disable_txclk: 2931 clk_disable_unprepare(*tx_clk); 2932 err_disable_axiclk: 2933 clk_disable_unprepare(*axi_clk); 2934 2935 return err; 2936 } 2937 2938 static void xdma_disable_allclks(struct xilinx_dma_device *xdev) 2939 { 2940 clk_disable_unprepare(xdev->rxs_clk); 2941 clk_disable_unprepare(xdev->rx_clk); 2942 clk_disable_unprepare(xdev->txs_clk); 2943 clk_disable_unprepare(xdev->tx_clk); 2944 clk_disable_unprepare(xdev->axi_clk); 2945 } 2946 2947 /** 2948 * xilinx_dma_chan_probe - Per Channel Probing 2949 * It get channel features from the device tree entry and 2950 * initialize special channel handling routines 2951 * 2952 * @xdev: Driver specific device structure 2953 * @node: Device node 2954 * 2955 * Return: '0' on success and failure value on error 2956 */ 2957 static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, 2958 struct device_node *node) 2959 { 2960 struct xilinx_dma_chan *chan; 2961 bool has_dre = false; 2962 u32 value, width; 2963 int err; 2964 2965 /* Allocate and initialize the channel structure */ 2966 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); 2967 if (!chan) 2968 return -ENOMEM; 2969 2970 chan->dev = xdev->dev; 2971 chan->xdev = xdev; 2972 chan->desc_pendingcount = 0x0; 2973 chan->ext_addr = xdev->ext_addr; 2974 /* This variable ensures that descriptors are not 2975 * Submitted when dma engine is in progress. This variable is 2976 * Added to avoid polling for a bit in the status register to 2977 * Know dma state in the driver hot path. 2978 */ 2979 chan->idle = true; 2980 2981 spin_lock_init(&chan->lock); 2982 INIT_LIST_HEAD(&chan->pending_list); 2983 INIT_LIST_HEAD(&chan->done_list); 2984 INIT_LIST_HEAD(&chan->active_list); 2985 INIT_LIST_HEAD(&chan->free_seg_list); 2986 2987 /* Retrieve the channel properties from the device tree */ 2988 has_dre = of_property_read_bool(node, "xlnx,include-dre"); 2989 2990 of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay); 2991 2992 chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); 2993 2994 err = of_property_read_u32(node, "xlnx,datawidth", &value); 2995 if (err) { 2996 dev_err(xdev->dev, "missing xlnx,datawidth property\n"); 2997 return err; 2998 } 2999 width = value >> 3; /* Convert bits to bytes */ 3000 3001 /* If data width is greater than 8 bytes, DRE is not in hw */ 3002 if (width > 8) 3003 has_dre = false; 3004 3005 if (!has_dre) 3006 xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1); 3007 3008 if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") || 3009 of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || 3010 of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { 3011 chan->direction = DMA_MEM_TO_DEV; 3012 chan->id = xdev->mm2s_chan_id++; 3013 chan->tdest = chan->id; 3014 3015 chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; 3016 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 3017 chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET; 3018 chan->config.park = 1; 3019 3020 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || 3021 xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S) 3022 chan->flush_on_fsync = true; 3023 } 3024 } else if (of_device_is_compatible(node, 3025 "xlnx,axi-vdma-s2mm-channel") || 3026 of_device_is_compatible(node, 3027 "xlnx,axi-dma-s2mm-channel")) { 3028 chan->direction = DMA_DEV_TO_MEM; 3029 chan->id = xdev->s2mm_chan_id++; 3030 chan->tdest = chan->id - xdev->dma_config->max_channels / 2; 3031 chan->has_vflip = of_property_read_bool(node, 3032 "xlnx,enable-vert-flip"); 3033 if (chan->has_vflip) { 3034 chan->config.vflip_en = dma_read(chan, 3035 XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP) & 3036 XILINX_VDMA_ENABLE_VERTICAL_FLIP; 3037 } 3038 3039 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) 3040 chan->ctrl_offset = XILINX_MCDMA_S2MM_CTRL_OFFSET; 3041 else 3042 chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET; 3043 3044 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 3045 chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET; 3046 chan->config.park = 1; 3047 3048 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || 3049 xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM) 3050 chan->flush_on_fsync = true; 3051 } 3052 } else { 3053 dev_err(xdev->dev, "Invalid channel compatible node\n"); 3054 return -EINVAL; 3055 } 3056 3057 xdev->common.directions |= BIT(chan->direction); 3058 3059 /* Request the interrupt */ 3060 chan->irq = of_irq_get(node, chan->tdest); 3061 if (chan->irq < 0) 3062 return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n"); 3063 err = request_irq(chan->irq, xdev->dma_config->irq_handler, 3064 IRQF_SHARED, "xilinx-dma-controller", chan); 3065 if (err) { 3066 dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq); 3067 return err; 3068 } 3069 3070 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 3071 chan->start_transfer = xilinx_dma_start_transfer; 3072 chan->stop_transfer = xilinx_dma_stop_transfer; 3073 } else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { 3074 chan->start_transfer = xilinx_mcdma_start_transfer; 3075 chan->stop_transfer = xilinx_dma_stop_transfer; 3076 } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 3077 chan->start_transfer = xilinx_cdma_start_transfer; 3078 chan->stop_transfer = xilinx_cdma_stop_transfer; 3079 } else { 3080 chan->start_transfer = xilinx_vdma_start_transfer; 3081 chan->stop_transfer = xilinx_dma_stop_transfer; 3082 } 3083 3084 /* check if SG is enabled (only for AXIDMA, AXIMCDMA, and CDMA) */ 3085 if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) { 3086 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA || 3087 dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & 3088 XILINX_DMA_DMASR_SG_MASK) 3089 chan->has_sg = true; 3090 dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, 3091 str_enabled_disabled(chan->has_sg)); 3092 } 3093 3094 /* Initialize the tasklet */ 3095 tasklet_setup(&chan->tasklet, xilinx_dma_do_tasklet); 3096 3097 /* 3098 * Initialize the DMA channel and add it to the DMA engine channels 3099 * list. 3100 */ 3101 chan->common.device = &xdev->common; 3102 3103 list_add_tail(&chan->common.device_node, &xdev->common.channels); 3104 xdev->chan[chan->id] = chan; 3105 3106 /* Reset the channel */ 3107 err = xilinx_dma_chan_reset(chan); 3108 if (err < 0) { 3109 dev_err(xdev->dev, "Reset channel failed\n"); 3110 return err; 3111 } 3112 3113 return 0; 3114 } 3115 3116 /** 3117 * xilinx_dma_child_probe - Per child node probe 3118 * It get number of dma-channels per child node from 3119 * device-tree and initializes all the channels. 3120 * 3121 * @xdev: Driver specific device structure 3122 * @node: Device node 3123 * 3124 * Return: '0' on success and failure value on error. 3125 */ 3126 static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, 3127 struct device_node *node) 3128 { 3129 int ret, i; 3130 u32 nr_channels = 1; 3131 3132 ret = of_property_read_u32(node, "dma-channels", &nr_channels); 3133 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0) 3134 dev_warn(xdev->dev, "missing dma-channels property\n"); 3135 3136 for (i = 0; i < nr_channels; i++) { 3137 ret = xilinx_dma_chan_probe(xdev, node); 3138 if (ret) 3139 return ret; 3140 } 3141 3142 return 0; 3143 } 3144 3145 /** 3146 * of_dma_xilinx_xlate - Translation function 3147 * @dma_spec: Pointer to DMA specifier as found in the device tree 3148 * @ofdma: Pointer to DMA controller data 3149 * 3150 * Return: DMA channel pointer on success and NULL on error 3151 */ 3152 static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, 3153 struct of_dma *ofdma) 3154 { 3155 struct xilinx_dma_device *xdev = ofdma->of_dma_data; 3156 int chan_id = dma_spec->args[0]; 3157 3158 if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id]) 3159 return NULL; 3160 3161 return dma_get_slave_channel(&xdev->chan[chan_id]->common); 3162 } 3163 3164 static const struct xilinx_dma_config axidma_config = { 3165 .dmatype = XDMA_TYPE_AXIDMA, 3166 .clk_init = axidma_clk_init, 3167 .irq_handler = xilinx_dma_irq_handler, 3168 .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, 3169 }; 3170 3171 static const struct xilinx_dma_config aximcdma_config = { 3172 .dmatype = XDMA_TYPE_AXIMCDMA, 3173 .clk_init = axidma_clk_init, 3174 .irq_handler = xilinx_mcdma_irq_handler, 3175 .max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE, 3176 }; 3177 static const struct xilinx_dma_config axicdma_config = { 3178 .dmatype = XDMA_TYPE_CDMA, 3179 .clk_init = axicdma_clk_init, 3180 .irq_handler = xilinx_dma_irq_handler, 3181 .max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE, 3182 }; 3183 3184 static const struct xilinx_dma_config axivdma_config = { 3185 .dmatype = XDMA_TYPE_VDMA, 3186 .clk_init = axivdma_clk_init, 3187 .irq_handler = xilinx_dma_irq_handler, 3188 .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, 3189 }; 3190 3191 static const struct of_device_id xilinx_dma_of_ids[] = { 3192 { .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config }, 3193 { .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config }, 3194 { .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config }, 3195 { .compatible = "xlnx,axi-mcdma-1.00.a", .data = &aximcdma_config }, 3196 {} 3197 }; 3198 MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids); 3199 3200 /** 3201 * xilinx_dma_probe - Driver probe function 3202 * @pdev: Pointer to the platform_device structure 3203 * 3204 * Return: '0' on success and failure value on error 3205 */ 3206 static int xilinx_dma_probe(struct platform_device *pdev) 3207 { 3208 int (*clk_init)(struct platform_device *, struct clk **, struct clk **, 3209 struct clk **, struct clk **, struct clk **) 3210 = axivdma_clk_init; 3211 struct device_node *node = pdev->dev.of_node; 3212 struct xilinx_dma_device *xdev; 3213 struct device_node *np = pdev->dev.of_node; 3214 u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width; 3215 int i, err; 3216 3217 /* Allocate and initialize the DMA engine structure */ 3218 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); 3219 if (!xdev) 3220 return -ENOMEM; 3221 3222 xdev->dev = &pdev->dev; 3223 if (np) { 3224 const struct of_device_id *match; 3225 3226 match = of_match_node(xilinx_dma_of_ids, np); 3227 if (match && match->data) { 3228 xdev->dma_config = match->data; 3229 clk_init = xdev->dma_config->clk_init; 3230 } 3231 } 3232 3233 err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk, 3234 &xdev->rx_clk, &xdev->rxs_clk); 3235 if (err) 3236 return err; 3237 3238 /* Request and map I/O memory */ 3239 xdev->regs = devm_platform_ioremap_resource(pdev, 0); 3240 if (IS_ERR(xdev->regs)) { 3241 err = PTR_ERR(xdev->regs); 3242 goto disable_clks; 3243 } 3244 /* Retrieve the DMA engine properties from the device tree */ 3245 xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); 3246 xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; 3247 3248 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || 3249 xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { 3250 if (!of_property_read_u32(node, "xlnx,sg-length-width", 3251 &len_width)) { 3252 if (len_width < XILINX_DMA_MAX_TRANS_LEN_MIN || 3253 len_width > XILINX_DMA_V2_MAX_TRANS_LEN_MAX) { 3254 dev_warn(xdev->dev, 3255 "invalid xlnx,sg-length-width property value. Using default width\n"); 3256 } else { 3257 if (len_width > XILINX_DMA_MAX_TRANS_LEN_MAX) 3258 dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n"); 3259 xdev->max_buffer_len = 3260 GENMASK(len_width - 1, 0); 3261 } 3262 } 3263 } 3264 3265 dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len); 3266 3267 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 3268 xdev->has_axistream_connected = 3269 of_property_read_bool(node, "xlnx,axistream-connected"); 3270 } 3271 3272 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 3273 err = of_property_read_u32(node, "xlnx,num-fstores", 3274 &num_frames); 3275 if (err < 0) { 3276 dev_err(xdev->dev, 3277 "missing xlnx,num-fstores property\n"); 3278 goto disable_clks; 3279 } 3280 3281 err = of_property_read_u32(node, "xlnx,flush-fsync", 3282 &xdev->flush_on_fsync); 3283 if (err < 0) 3284 dev_warn(xdev->dev, 3285 "missing xlnx,flush-fsync property\n"); 3286 } 3287 3288 err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width); 3289 if (err < 0) 3290 dev_warn(xdev->dev, 3291 "missing xlnx,addrwidth property, using default value %d\n", 3292 XILINX_DMA_DFAULT_ADDRWIDTH); 3293 3294 if (addr_width > 32) 3295 xdev->ext_addr = true; 3296 else 3297 xdev->ext_addr = false; 3298 3299 /* Set metadata mode */ 3300 if (xdev->has_axistream_connected) 3301 xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE; 3302 3303 /* Set the dma mask bits */ 3304 err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); 3305 if (err < 0) { 3306 dev_err(xdev->dev, "DMA mask error %d\n", err); 3307 goto disable_clks; 3308 } 3309 3310 /* Initialize the DMA engine */ 3311 xdev->common.dev = &pdev->dev; 3312 3313 INIT_LIST_HEAD(&xdev->common.channels); 3314 if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) { 3315 dma_cap_set(DMA_SLAVE, xdev->common.cap_mask); 3316 dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask); 3317 } 3318 3319 xdev->common.device_alloc_chan_resources = 3320 xilinx_dma_alloc_chan_resources; 3321 xdev->common.device_free_chan_resources = 3322 xilinx_dma_free_chan_resources; 3323 xdev->common.device_terminate_all = xilinx_dma_terminate_all; 3324 xdev->common.device_synchronize = xilinx_dma_synchronize; 3325 xdev->common.device_tx_status = xilinx_dma_tx_status; 3326 xdev->common.device_issue_pending = xilinx_dma_issue_pending; 3327 xdev->common.device_config = xilinx_dma_device_config; 3328 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 3329 dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); 3330 xdev->common.device_prep_peripheral_dma_vec = xilinx_dma_prep_peripheral_dma_vec; 3331 xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; 3332 xdev->common.device_prep_dma_cyclic = 3333 xilinx_dma_prep_dma_cyclic; 3334 /* Residue calculation is supported by only AXI DMA and CDMA */ 3335 xdev->common.residue_granularity = 3336 DMA_RESIDUE_GRANULARITY_SEGMENT; 3337 } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 3338 dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask); 3339 xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy; 3340 /* Residue calculation is supported by only AXI DMA and CDMA */ 3341 xdev->common.residue_granularity = 3342 DMA_RESIDUE_GRANULARITY_SEGMENT; 3343 } else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { 3344 xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg; 3345 } else { 3346 xdev->common.device_prep_interleaved_dma = 3347 xilinx_vdma_dma_prep_interleaved; 3348 } 3349 3350 platform_set_drvdata(pdev, xdev); 3351 3352 /* Initialize the channels */ 3353 for_each_child_of_node_scoped(node, child) { 3354 err = xilinx_dma_child_probe(xdev, child); 3355 if (err < 0) 3356 goto error; 3357 } 3358 3359 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 3360 for (i = 0; i < xdev->dma_config->max_channels; i++) 3361 if (xdev->chan[i]) 3362 xdev->chan[i]->num_frms = num_frames; 3363 } 3364 3365 /* Register the DMA engine with the core */ 3366 err = dma_async_device_register(&xdev->common); 3367 if (err) { 3368 dev_err(xdev->dev, "failed to register the dma device\n"); 3369 goto error; 3370 } 3371 3372 err = of_dma_controller_register(node, of_dma_xilinx_xlate, 3373 xdev); 3374 if (err < 0) { 3375 dev_err(&pdev->dev, "Unable to register DMA to DT\n"); 3376 dma_async_device_unregister(&xdev->common); 3377 goto error; 3378 } 3379 3380 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) 3381 dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n"); 3382 else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) 3383 dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n"); 3384 else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) 3385 dev_info(&pdev->dev, "Xilinx AXI MCDMA Engine Driver Probed!!\n"); 3386 else 3387 dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n"); 3388 3389 return 0; 3390 3391 error: 3392 for (i = 0; i < xdev->dma_config->max_channels; i++) 3393 if (xdev->chan[i]) 3394 xilinx_dma_chan_remove(xdev->chan[i]); 3395 disable_clks: 3396 xdma_disable_allclks(xdev); 3397 3398 return err; 3399 } 3400 3401 /** 3402 * xilinx_dma_remove - Driver remove function 3403 * @pdev: Pointer to the platform_device structure 3404 */ 3405 static void xilinx_dma_remove(struct platform_device *pdev) 3406 { 3407 struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); 3408 int i; 3409 3410 of_dma_controller_free(pdev->dev.of_node); 3411 3412 dma_async_device_unregister(&xdev->common); 3413 3414 for (i = 0; i < xdev->dma_config->max_channels; i++) 3415 if (xdev->chan[i]) 3416 xilinx_dma_chan_remove(xdev->chan[i]); 3417 3418 xdma_disable_allclks(xdev); 3419 } 3420 3421 static struct platform_driver xilinx_vdma_driver = { 3422 .driver = { 3423 .name = "xilinx-vdma", 3424 .of_match_table = xilinx_dma_of_ids, 3425 }, 3426 .probe = xilinx_dma_probe, 3427 .remove = xilinx_dma_remove, 3428 }; 3429 3430 module_platform_driver(xilinx_vdma_driver); 3431 3432 MODULE_AUTHOR("Xilinx, Inc."); 3433 MODULE_DESCRIPTION("Xilinx VDMA driver"); 3434 MODULE_LICENSE("GPL v2"); 3435