1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * STM32 DMA3 controller driver 4 * 5 * Copyright (C) STMicroelectronics 2024 6 * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com> 7 */ 8 9 #include <linux/bitfield.h> 10 #include <linux/clk.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dmapool.h> 14 #include <linux/init.h> 15 #include <linux/iopoll.h> 16 #include <linux/list.h> 17 #include <linux/module.h> 18 #include <linux/of_dma.h> 19 #include <linux/platform_device.h> 20 #include <linux/pm_runtime.h> 21 #include <linux/reset.h> 22 #include <linux/slab.h> 23 24 #include "../virt-dma.h" 25 26 #define STM32_DMA3_SECCFGR 0x00 27 #define STM32_DMA3_PRIVCFGR 0x04 28 #define STM32_DMA3_RCFGLOCKR 0x08 29 #define STM32_DMA3_MISR 0x0c 30 #define STM32_DMA3_SMISR 0x10 31 32 #define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x)) 33 #define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x)) 34 #define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x)) 35 #define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x)) 36 #define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x)) 37 #define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x)) 38 #define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x)) 39 #define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x)) 40 #define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x)) 41 #define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x)) 42 #define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x)) 43 #define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x)) 44 45 #define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */ 46 #define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */ 47 #define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */ 48 #define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */ 49 #define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */ 50 #define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */ 51 #define STM32_DMA3_VERR 0xff4 52 53 /* SECCFGR DMA secure configuration register */ 54 #define SECCFGR_SEC(x) BIT(x) 55 56 /* MISR DMA non-secure/secure masked interrupt status register */ 57 #define MISR_MIS(x) BIT(x) 58 59 /* CxLBAR DMA channel x linked_list base address register */ 60 #define CLBAR_LBA GENMASK(31, 16) 61 62 /* CxCIDCFGR DMA channel x CID register */ 63 #define CCIDCFGR_CFEN BIT(0) 64 #define CCIDCFGR_SEM_EN BIT(1) 65 #define CCIDCFGR_SCID GENMASK(5, 4) 66 #define CCIDCFGR_SEM_WLIST_CID0 BIT(16) 67 #define CCIDCFGR_SEM_WLIST_CID1 BIT(17) 68 #define CCIDCFGR_SEM_WLIST_CID2 BIT(18) 69 70 enum ccidcfgr_cid { 71 CCIDCFGR_CID0, 72 CCIDCFGR_CID1, 73 CCIDCFGR_CID2, 74 }; 75 76 /* CxSEMCR DMA channel x semaphore control register */ 77 #define CSEMCR_SEM_MUTEX BIT(0) 78 #define CSEMCR_SEM_CCID GENMASK(5, 4) 79 80 /* CxFCR DMA channel x flag clear register */ 81 #define CFCR_TCF BIT(8) 82 #define CFCR_HTF BIT(9) 83 #define CFCR_DTEF BIT(10) 84 #define CFCR_ULEF BIT(11) 85 #define CFCR_USEF BIT(12) 86 #define CFCR_SUSPF BIT(13) 87 88 /* CxSR DMA channel x status register */ 89 #define CSR_IDLEF BIT(0) 90 #define CSR_TCF BIT(8) 91 #define CSR_HTF BIT(9) 92 #define CSR_DTEF BIT(10) 93 #define CSR_ULEF BIT(11) 94 #define CSR_USEF BIT(12) 95 #define CSR_SUSPF BIT(13) 96 #define CSR_ALL_F GENMASK(13, 8) 97 #define CSR_FIFOL GENMASK(24, 16) 98 99 /* CxCR DMA channel x control register */ 100 #define CCR_EN BIT(0) 101 #define CCR_RESET BIT(1) 102 #define CCR_SUSP BIT(2) 103 #define CCR_TCIE BIT(8) 104 #define CCR_HTIE BIT(9) 105 #define CCR_DTEIE BIT(10) 106 #define CCR_ULEIE BIT(11) 107 #define CCR_USEIE BIT(12) 108 #define CCR_SUSPIE BIT(13) 109 #define CCR_ALLIE GENMASK(13, 8) 110 #define CCR_LSM BIT(16) 111 #define CCR_LAP BIT(17) 112 #define CCR_PRIO GENMASK(23, 22) 113 114 enum ccr_prio { 115 CCR_PRIO_LOW, 116 CCR_PRIO_MID, 117 CCR_PRIO_HIGH, 118 CCR_PRIO_VERY_HIGH, 119 }; 120 121 /* CxTR1 DMA channel x transfer register 1 */ 122 #define CTR1_SINC BIT(3) 123 #define CTR1_SBL_1 GENMASK(9, 4) 124 #define CTR1_DINC BIT(19) 125 #define CTR1_DBL_1 GENMASK(25, 20) 126 #define CTR1_SDW_LOG2 GENMASK(1, 0) 127 #define CTR1_PAM GENMASK(12, 11) 128 #define CTR1_SAP BIT(14) 129 #define CTR1_DDW_LOG2 GENMASK(17, 16) 130 #define CTR1_DAP BIT(30) 131 132 enum ctr1_dw { 133 CTR1_DW_BYTE, 134 CTR1_DW_HWORD, 135 CTR1_DW_WORD, 136 CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */ 137 }; 138 139 enum ctr1_pam { 140 CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */ 141 CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */ 142 CTR1_PAM_PACK_UNPACK, /* FIFO queued */ 143 }; 144 145 /* CxTR2 DMA channel x transfer register 2 */ 146 #define CTR2_REQSEL GENMASK(7, 0) 147 #define CTR2_SWREQ BIT(9) 148 #define CTR2_DREQ BIT(10) 149 #define CTR2_BREQ BIT(11) 150 #define CTR2_PFREQ BIT(12) 151 #define CTR2_TCEM GENMASK(31, 30) 152 153 enum ctr2_tcem { 154 CTR2_TCEM_BLOCK, 155 CTR2_TCEM_REPEAT_BLOCK, 156 CTR2_TCEM_LLI, 157 CTR2_TCEM_CHANNEL, 158 }; 159 160 /* CxBR1 DMA channel x block register 1 */ 161 #define CBR1_BNDT GENMASK(15, 0) 162 163 /* CxLLR DMA channel x linked-list address register */ 164 #define CLLR_LA GENMASK(15, 2) 165 #define CLLR_ULL BIT(16) 166 #define CLLR_UDA BIT(27) 167 #define CLLR_USA BIT(28) 168 #define CLLR_UB1 BIT(29) 169 #define CLLR_UT2 BIT(30) 170 #define CLLR_UT1 BIT(31) 171 172 /* HWCFGR13 DMA hardware configuration register 13 x=8..15 */ 173 /* HWCFGR12 DMA hardware configuration register 12 x=0..7 */ 174 #define G_PER_CTRL(x) (ULL(0x1) << (4 * (x))) 175 176 /* HWCFGR4 DMA hardware configuration register 4 x=8..15 */ 177 /* HWCFGR3 DMA hardware configuration register 3 x=0..7 */ 178 #define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x))) 179 180 #define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x))) 181 182 /* HWCFGR2 DMA hardware configuration register 2 */ 183 #define G_MAX_REQ_ID GENMASK(7, 0) 184 185 /* HWCFGR1 DMA hardware configuration register 1 */ 186 #define G_MASTER_PORTS GENMASK(2, 0) 187 #define G_NUM_CHANNELS GENMASK(12, 8) 188 #define G_M0_DATA_WIDTH_ENC GENMASK(25, 24) 189 #define G_M1_DATA_WIDTH_ENC GENMASK(29, 28) 190 191 enum stm32_dma3_master_ports { 192 AXI64, /* 1x AXI: 64-bit port 0 */ 193 AHB32, /* 1x AHB: 32-bit port 0 */ 194 AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */ 195 AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */ 196 AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */ 197 AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */ 198 }; 199 200 enum stm32_dma3_port_data_width { 201 DW_32, /* 32-bit, for AHB */ 202 DW_64, /* 64-bit, for AXI */ 203 DW_128, /* 128-bit, for AXI */ 204 DW_INVALID, 205 }; 206 207 /* VERR DMA version register */ 208 #define VERR_MINREV GENMASK(3, 0) 209 #define VERR_MAJREV GENMASK(7, 4) 210 211 /* Device tree */ 212 /* struct stm32_dma3_dt_conf */ 213 /* .ch_conf */ 214 #define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */ 215 #define STM32_DMA3_DT_FIFO GENMASK(7, 4) 216 /* .tr_conf */ 217 #define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */ 218 #define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */ 219 #define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */ 220 #define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */ 221 #define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */ 222 #define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */ 223 #define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */ 224 #define STM32_DMA3_DT_NOPACK BIT(16) /* CTR1_PAM */ 225 #define STM32_DMA3_DT_NOREFACT BIT(17) 226 227 /* struct stm32_dma3_chan .config_set bitfield */ 228 #define STM32_DMA3_CFG_SET_DT BIT(0) 229 #define STM32_DMA3_CFG_SET_DMA BIT(1) 230 #define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA) 231 232 #define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64) 233 #define STM32_DMA3_MAX_BURST_LEN (1 + min_t(u32, FIELD_MAX(CTR1_SBL_1), \ 234 FIELD_MAX(CTR1_DBL_1))) 235 #define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ 236 ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); }) 237 #define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ 238 ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); }) 239 #define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \ 240 (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \ 241 DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES) 242 243 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */ 244 struct stm32_dma3_hwdesc { 245 u32 ctr1; 246 u32 ctr2; 247 u32 cbr1; 248 u32 csar; 249 u32 cdar; 250 u32 cllr; 251 } __packed __aligned(32); 252 253 /* 254 * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed 255 * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte 256 * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents 257 * the sg size limitation. 258 */ 259 #define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE) 260 261 /* 262 * Linked-list items 263 */ 264 struct stm32_dma3_lli { 265 struct stm32_dma3_hwdesc *hwdesc; 266 dma_addr_t hwdesc_addr; 267 }; 268 269 struct stm32_dma3_swdesc { 270 struct virt_dma_desc vdesc; 271 u32 ccr; 272 bool cyclic; 273 u32 lli_size; 274 struct stm32_dma3_lli lli[] __counted_by(lli_size); 275 }; 276 277 struct stm32_dma3_dt_conf { 278 u32 ch_id; 279 u32 req_line; 280 u32 ch_conf; 281 u32 tr_conf; 282 }; 283 284 struct stm32_dma3_chan { 285 struct virt_dma_chan vchan; 286 u32 id; 287 int irq; 288 u32 fifo_size; 289 u32 max_burst; 290 bool semaphore_mode; 291 bool semaphore_taken; 292 struct stm32_dma3_dt_conf dt_config; 293 struct dma_slave_config dma_config; 294 u8 config_set; 295 struct dma_pool *lli_pool; 296 struct stm32_dma3_swdesc *swdesc; 297 enum ctr2_tcem tcem; 298 u32 dma_status; 299 }; 300 301 struct stm32_dma3_pdata { 302 u32 axi_max_burst_len; 303 }; 304 305 struct stm32_dma3_ddata { 306 struct dma_device dma_dev; 307 void __iomem *base; 308 struct clk *clk; 309 struct stm32_dma3_chan *chans; 310 u32 dma_channels; 311 u32 dma_requests; 312 enum stm32_dma3_port_data_width ports_max_dw[2]; 313 u32 axi_max_burst_len; 314 }; 315 316 static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan) 317 { 318 return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev); 319 } 320 321 static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c) 322 { 323 return container_of(c, struct stm32_dma3_chan, vchan.chan); 324 } 325 326 static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc) 327 { 328 return container_of(vdesc, struct stm32_dma3_swdesc, vdesc); 329 } 330 331 static struct device *ddata2dev(struct stm32_dma3_ddata *ddata) 332 { 333 return ddata->dma_dev.dev; 334 } 335 336 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan) 337 { 338 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 339 struct device *dev = vchan_chan_dev(&chan->vchan); 340 u32 id = chan->id, offset; 341 342 offset = STM32_DMA3_SECCFGR; 343 dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); 344 offset = STM32_DMA3_PRIVCFGR; 345 dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); 346 offset = STM32_DMA3_CCIDCFGR(id); 347 dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 348 offset = STM32_DMA3_CSEMCR(id); 349 dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 350 offset = STM32_DMA3_CSR(id); 351 dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 352 offset = STM32_DMA3_CCR(id); 353 dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 354 offset = STM32_DMA3_CTR1(id); 355 dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 356 offset = STM32_DMA3_CTR2(id); 357 dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 358 offset = STM32_DMA3_CBR1(id); 359 dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 360 offset = STM32_DMA3_CSAR(id); 361 dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 362 offset = STM32_DMA3_CDAR(id); 363 dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 364 offset = STM32_DMA3_CLLR(id); 365 dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 366 offset = STM32_DMA3_CLBAR(id); 367 dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); 368 } 369 370 static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan, 371 struct stm32_dma3_swdesc *swdesc) 372 { 373 struct stm32_dma3_hwdesc *hwdesc; 374 int i; 375 376 for (i = 0; i < swdesc->lli_size; i++) { 377 hwdesc = swdesc->lli[i].hwdesc; 378 if (i) 379 dev_dbg(vchan_chan_dev(&chan->vchan), "V\n"); 380 dev_dbg(vchan_chan_dev(&chan->vchan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); 381 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); 382 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); 383 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); 384 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); 385 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); 386 dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); 387 } 388 389 if (swdesc->cyclic) { 390 dev_dbg(vchan_chan_dev(&chan->vchan), "|\n"); 391 dev_dbg(vchan_chan_dev(&chan->vchan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); 392 } else { 393 dev_dbg(vchan_chan_dev(&chan->vchan), "X\n"); 394 } 395 } 396 397 static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count) 398 { 399 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 400 struct stm32_dma3_swdesc *swdesc; 401 int i; 402 403 /* 404 * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes 405 * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be 406 * addressed, so abort the allocation. 407 */ 408 if ((count * 32) > CLLR_LA) { 409 dev_err(vchan_chan_dev(&chan->vchan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE); 410 return NULL; 411 } 412 413 swdesc = kzalloc_flex(*swdesc, lli, count, GFP_NOWAIT); 414 if (!swdesc) 415 return NULL; 416 swdesc->lli_size = count; 417 418 for (i = 0; i < count; i++) { 419 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, 420 &swdesc->lli[i].hwdesc_addr); 421 if (!swdesc->lli[i].hwdesc) 422 goto err_pool_free; 423 } 424 swdesc->ccr = 0; 425 426 /* Set LL base address */ 427 writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA, 428 ddata->base + STM32_DMA3_CLBAR(chan->id)); 429 430 /* Set LL allocated port */ 431 swdesc->ccr &= ~CCR_LAP; 432 433 return swdesc; 434 435 err_pool_free: 436 dev_err(vchan_chan_dev(&chan->vchan), "Failed to alloc descriptors\n"); 437 while (--i >= 0) 438 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); 439 kfree(swdesc); 440 441 return NULL; 442 } 443 444 static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan, 445 struct stm32_dma3_swdesc *swdesc) 446 { 447 int i; 448 449 for (i = 0; i < swdesc->lli_size; i++) 450 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); 451 452 kfree(swdesc); 453 } 454 455 static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc) 456 { 457 struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc); 458 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan); 459 460 stm32_dma3_chan_desc_free(chan, swdesc); 461 } 462 463 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan) 464 { 465 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 466 struct device *dev = vchan_chan_dev(&chan->vchan); 467 u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id)); 468 u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); 469 u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id)); 470 u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); 471 u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); 472 u32 bndt = FIELD_GET(CBR1_BNDT, cbr1); 473 u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1); 474 u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1); 475 u32 sap = FIELD_GET(CTR1_SAP, ctr1); 476 u32 dap = FIELD_GET(CTR1_DAP, ctr1); 477 478 if (!bndt && !FIELD_GET(CLLR_UB1, cllr)) 479 dev_err(dev, "null source block size and no update of this value\n"); 480 if (bndt % sdw) 481 dev_err(dev, "source block size not multiple of src data width\n"); 482 if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw) 483 dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n"); 484 if (csar % sdw) 485 dev_err(dev, "unaligned source address not multiple of src data width\n"); 486 if (cdar % ddw) 487 dev_err(dev, "unaligned destination address not multiple of dst data width\n"); 488 if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap])) 489 dev_err(dev, "double-word source data width not supported on port %u\n", sap); 490 if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap])) 491 dev_err(dev, "double-word destination data width not supported on port %u\n", dap); 492 } 493 494 static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan, 495 struct stm32_dma3_swdesc *swdesc, 496 u32 curr, dma_addr_t src, dma_addr_t dst, u32 len, 497 u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic) 498 { 499 struct stm32_dma3_hwdesc *hwdesc; 500 dma_addr_t next_lli; 501 u32 next = curr + 1; 502 503 hwdesc = swdesc->lli[curr].hwdesc; 504 hwdesc->ctr1 = ctr1; 505 hwdesc->ctr2 = ctr2; 506 hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len); 507 hwdesc->csar = src; 508 hwdesc->cdar = dst; 509 510 if (is_last) { 511 if (is_cyclic) 512 next_lli = swdesc->lli[0].hwdesc_addr; 513 else 514 next_lli = 0; 515 } else { 516 next_lli = swdesc->lli[next].hwdesc_addr; 517 } 518 519 hwdesc->cllr = 0; 520 if (next_lli) { 521 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1; 522 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL; 523 hwdesc->cllr |= (next_lli & CLLR_LA); 524 } 525 526 /* 527 * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read 528 * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel 529 * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors 530 * are reused (DMA_CTRL_REUSE). 531 */ 532 if (is_last) 533 dma_wmb(); 534 } 535 536 static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst, 537 enum stm32_dma3_port_data_width port_max_dw, 538 u32 len, dma_addr_t addr) 539 { 540 enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst); 541 542 /* len and addr must be a multiple of dw */ 543 return 1 << __ffs(len | addr | max_dw); 544 } 545 546 static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw, 547 u32 chan_max_burst, u32 bus_max_burst) 548 { 549 u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1; 550 551 /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */ 552 if (len < chan_max_burst) 553 max_burst = len / dw; 554 555 /* 556 * HW doesn't modify the burst if burst size <= half of the fifo size. 557 * If len is not a multiple of burst size, last burst is shortened by HW. 558 * Take care of maximum burst supported on interconnect bus. 559 */ 560 return min_t(u32, max_burst, bus_max_burst); 561 } 562 563 static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir, 564 u32 *ccr, u32 *ctr1, u32 *ctr2, 565 dma_addr_t src_addr, dma_addr_t dst_addr, u32 len) 566 { 567 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 568 struct dma_device dma_device = ddata->dma_dev; 569 u32 src_max_burst = STM32_DMA3_MAX_BURST_LEN, dst_max_burst = STM32_DMA3_MAX_BURST_LEN; 570 u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max; 571 u32 _ctr1 = 0, _ctr2 = 0; 572 u32 ch_conf = chan->dt_config.ch_conf; 573 u32 tr_conf = chan->dt_config.tr_conf; 574 u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw; 575 u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw; 576 577 dev_dbg(vchan_chan_dev(&chan->vchan), "%s from %pad to %pad\n", 578 dmaengine_get_direction_text(dir), &src_addr, &dst_addr); 579 580 sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst); 581 ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst); 582 sbl_max = chan->dma_config.src_maxburst ? : 1; 583 dbl_max = chan->dma_config.dst_maxburst ? : 1; 584 585 /* Following conditions would raise User Setting Error interrupt */ 586 if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) { 587 dev_err(vchan_chan_dev(&chan->vchan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw); 588 return -EINVAL; 589 } 590 591 if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) { 592 dev_err(vchan_chan_dev(&chan->vchan), "Only one master port, port 1 is not supported\n"); 593 return -EINVAL; 594 } 595 596 sap_max_dw = ddata->ports_max_dw[sap]; 597 dap_max_dw = ddata->ports_max_dw[dap]; 598 if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) || 599 (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) { 600 dev_err(vchan_chan_dev(&chan->vchan), 601 "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n", 602 sdw, ddw, sap, dap); 603 return -EINVAL; 604 } 605 606 if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf)) 607 _ctr1 |= CTR1_SINC; 608 if (sap) 609 _ctr1 |= CTR1_SAP; 610 if (port_is_axi(sap_max_dw)) /* AXI - apply axi maximum burst limitation */ 611 src_max_burst = ddata->axi_max_burst_len; 612 if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf)) 613 _ctr1 |= CTR1_DINC; 614 if (dap) 615 _ctr1 |= CTR1_DAP; 616 if (port_is_axi(dap_max_dw)) /* AXI - apply axi maximum burst limitation */ 617 dst_max_burst = ddata->axi_max_burst_len; 618 619 _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ; 620 if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf)) 621 _ctr2 |= CTR2_BREQ; 622 if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf)) 623 _ctr2 |= CTR2_PFREQ; 624 tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf); 625 _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem); 626 627 /* Store TCEM to know on which event TC flag occurred */ 628 chan->tcem = tcem; 629 /* Store direction for residue computation */ 630 chan->dma_config.direction = dir; 631 632 switch (dir) { 633 case DMA_MEM_TO_DEV: 634 /* Set destination (device) data width and burst */ 635 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, 636 len, dst_addr)); 637 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst, 638 dst_max_burst)); 639 640 /* Set source (memory) data width and burst */ 641 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); 642 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); 643 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf)) { 644 sdw = ddw; 645 sbl_max = dbl_max; 646 } 647 648 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); 649 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); 650 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); 651 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); 652 653 if (ddw != sdw) { 654 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); 655 /* Should never reach this case as ddw is clamped down */ 656 if (len & (ddw - 1)) { 657 dev_err(vchan_chan_dev(&chan->vchan), 658 "Packing mode is enabled and len is not multiple of ddw"); 659 return -EINVAL; 660 } 661 } 662 663 /* dst = dev */ 664 _ctr2 |= CTR2_DREQ; 665 666 break; 667 668 case DMA_DEV_TO_MEM: 669 /* Set source (device) data width and burst */ 670 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, 671 len, src_addr)); 672 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst, 673 src_max_burst)); 674 675 /* Set destination (memory) data width and burst */ 676 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); 677 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); 678 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf) || 679 ((_ctr2 & CTR2_PFREQ) && ddw > sdw)) { /* Packing to wider ddw not supported */ 680 ddw = sdw; 681 dbl_max = sbl_max; 682 } 683 684 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); 685 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); 686 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); 687 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); 688 689 if (ddw != sdw) { 690 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); 691 /* Should never reach this case as ddw is clamped down */ 692 if (len & (ddw - 1)) { 693 dev_err(vchan_chan_dev(&chan->vchan), 694 "Packing mode is enabled and len is not multiple of ddw\n"); 695 return -EINVAL; 696 } 697 } 698 699 /* dst = mem */ 700 _ctr2 &= ~CTR2_DREQ; 701 702 break; 703 704 case DMA_MEM_TO_MEM: 705 /* Set source (memory) data width and burst */ 706 init_dw = sdw; 707 init_bl_max = sbl_max; 708 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); 709 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); 710 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { 711 sdw = min_t(u32, init_dw, sdw); 712 sbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, sdw, 713 chan->max_burst, 714 src_max_burst)); 715 } 716 717 /* Set destination (memory) data width and burst */ 718 init_dw = ddw; 719 init_bl_max = dbl_max; 720 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); 721 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); 722 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { 723 ddw = min_t(u32, init_dw, ddw); 724 dbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, ddw, 725 chan->max_burst, 726 dst_max_burst)); 727 } 728 729 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); 730 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); 731 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); 732 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); 733 734 if (ddw != sdw) { 735 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); 736 /* Should never reach this case as ddw is clamped down */ 737 if (len & (ddw - 1)) { 738 dev_err(vchan_chan_dev(&chan->vchan), 739 "Packing mode is enabled and len is not multiple of ddw"); 740 return -EINVAL; 741 } 742 } 743 744 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */ 745 _ctr2 |= CTR2_SWREQ; 746 747 break; 748 749 default: 750 dev_err(vchan_chan_dev(&chan->vchan), "Direction %s not supported\n", 751 dmaengine_get_direction_text(dir)); 752 return -EINVAL; 753 } 754 755 *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf)); 756 *ctr1 = _ctr1; 757 *ctr2 = _ctr2; 758 759 dev_dbg(vchan_chan_dev(&chan->vchan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n", 760 __func__, sdw, sbl_max, ddw, dbl_max); 761 762 return 0; 763 } 764 765 static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan) 766 { 767 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 768 struct virt_dma_desc *vdesc; 769 struct stm32_dma3_hwdesc *hwdesc; 770 u32 id = chan->id; 771 u32 csr, ccr; 772 773 vdesc = vchan_next_desc(&chan->vchan); 774 if (!vdesc) { 775 chan->swdesc = NULL; 776 return; 777 } 778 list_del(&vdesc->node); 779 780 chan->swdesc = to_stm32_dma3_swdesc(vdesc); 781 hwdesc = chan->swdesc->lli[0].hwdesc; 782 783 stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc); 784 785 writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id)); 786 writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id)); 787 writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id)); 788 writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id)); 789 writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id)); 790 writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id)); 791 writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id)); 792 793 /* Clear any pending interrupts */ 794 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id)); 795 if (csr & CSR_ALL_F) 796 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id)); 797 798 stm32_dma3_chan_dump_reg(chan); 799 800 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id)); 801 writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id)); 802 803 chan->dma_status = DMA_IN_PROGRESS; 804 805 dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan); 806 } 807 808 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp) 809 { 810 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 811 u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; 812 int ret = 0; 813 814 if (susp) 815 ccr |= CCR_SUSP; 816 else 817 ccr &= ~CCR_SUSP; 818 819 writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id)); 820 821 if (susp) { 822 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, 823 csr & CSR_SUSPF, 1, 10); 824 if (!ret) 825 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); 826 827 stm32_dma3_chan_dump_reg(chan); 828 } 829 830 return ret; 831 } 832 833 static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan) 834 { 835 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 836 u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; 837 838 writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id)); 839 } 840 841 static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue) 842 { 843 u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA; 844 845 /* If cllr is null, it means it is either the last or single item */ 846 if (!cllr) 847 return swdesc->lli_size - 1; 848 849 /* In cyclic mode, go fast and first check we are not on the last item */ 850 if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA)) 851 return swdesc->lli_size - 1; 852 853 /* As transfer is in progress, look backward from the last item */ 854 for (i = swdesc->lli_size - 1; i > 0; i--) { 855 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1); 856 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA; 857 if (lli_offset == next_lli_offset) 858 return i - 1; 859 } 860 861 return -EINVAL; 862 } 863 864 static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, 865 struct stm32_dma3_swdesc *swdesc, 866 struct dma_tx_state *txstate) 867 { 868 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 869 struct device *dev = vchan_chan_dev(&chan->vchan); 870 struct stm32_dma3_hwdesc *hwdesc; 871 u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol; 872 bool pack_unpack; 873 int ret; 874 875 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); 876 if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) { 877 /* Suspend current transfer to read registers for a snapshot */ 878 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id)); 879 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, 880 csr & (CSR_SUSPF | CSR_IDLEF), 1, 10); 881 882 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) { 883 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); 884 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); 885 if (ret) 886 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr); 887 } 888 } 889 890 /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */ 891 if (!(csr & CSR_IDLEF)) 892 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr); 893 894 /* 895 * If channel is not suspended, but Idle and Transfer Complete are set, 896 * linked-list is over, no residue 897 */ 898 if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF)) 899 return; 900 901 /* Read registers to have a snapshot */ 902 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); 903 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); 904 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); 905 906 /* Resume current transfer */ 907 if (csr & CSR_SUSPF) { 908 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); 909 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); 910 } 911 912 /* Add current BNDT */ 913 bndt = FIELD_GET(CBR1_BNDT, cbr1); 914 residue = bndt; 915 916 /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */ 917 ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue); 918 if (ret < 0) { 919 dev_err(vchan_chan_dev(&chan->vchan), "Can't get residue: current hwdesc not found\n"); 920 return; 921 } 922 curr_lli = ret; 923 924 /* Read current FIFO level - in units of programmed destination data width */ 925 hwdesc = swdesc->lli[curr_lli].hwdesc; 926 fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1)); 927 /* If the FIFO contains as many bytes as its size, it can't contain more */ 928 if (fifol == (1 << (chan->fifo_size + 1))) 929 goto skip_fifol_update; 930 931 /* 932 * In case of PACKING (Destination burst length > Source burst length) or UNPACKING 933 * (Source burst length > Destination burst length), bytes could be pending in the FIFO 934 * (to be packed up to Destination burst length or unpacked into Destination burst length 935 * chunks). 936 * BNDT is not reliable, as it reflects the number of bytes read from the source but not the 937 * number of bytes written to the destination. 938 * FIFOL is also not sufficient, because it reflects the number of available write beats in 939 * units of Destination data width but not the bytes not yet packed or unpacked. 940 * In case of Destination increment DINC, it is possible to compute the number of bytes in 941 * the FIFO: 942 * fifol_in_bytes = bytes_read - bytes_written. 943 */ 944 pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK); 945 if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) { 946 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt; 947 int bytes_written = cdar - hwdesc->cdar; 948 949 if (bytes_read > 0) 950 fifol = bytes_read - bytes_written; 951 } 952 953 skip_fifol_update: 954 if (fifol) { 955 dev_dbg(vchan_chan_dev(&chan->vchan), "%u byte(s) in the FIFO\n", fifol); 956 dma_set_in_flight_bytes(txstate, fifol); 957 /* 958 * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from 959 * the source memory buffer, so just need to add fifol to residue in case of 960 * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination 961 * memory buffer. 962 */ 963 if (chan->dma_config.direction == DMA_DEV_TO_MEM) 964 residue += fifol; 965 } 966 dma_set_residue(txstate, residue); 967 } 968 969 static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan) 970 { 971 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 972 u32 ccr; 973 int ret = 0; 974 975 chan->dma_status = DMA_COMPLETE; 976 977 /* Disable interrupts */ 978 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)); 979 writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id)); 980 981 if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) { 982 /* Suspend the channel */ 983 ret = stm32_dma3_chan_suspend(chan, true); 984 if (ret) 985 dev_warn(vchan_chan_dev(&chan->vchan), "%s: timeout, data might be lost\n", __func__); 986 } 987 988 /* 989 * Reset the channel: this causes the reset of the FIFO and the reset of the channel 990 * internal state, the reset of CCR_EN and CCR_SUSP bits. 991 */ 992 stm32_dma3_chan_reset(chan); 993 994 return ret; 995 } 996 997 static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan) 998 { 999 if (!chan->swdesc) 1000 return; 1001 1002 vchan_cookie_complete(&chan->swdesc->vdesc); 1003 chan->swdesc = NULL; 1004 stm32_dma3_chan_start(chan); 1005 } 1006 1007 static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid) 1008 { 1009 struct stm32_dma3_chan *chan = devid; 1010 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1011 u32 misr, csr, ccr; 1012 1013 spin_lock(&chan->vchan.lock); 1014 1015 misr = readl_relaxed(ddata->base + STM32_DMA3_MISR); 1016 if (!(misr & MISR_MIS(chan->id))) { 1017 spin_unlock(&chan->vchan.lock); 1018 return IRQ_NONE; 1019 } 1020 1021 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); 1022 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE; 1023 1024 if (csr & CSR_TCF && ccr & CCR_TCIE) { 1025 if (chan->swdesc->cyclic) 1026 vchan_cyclic_callback(&chan->swdesc->vdesc); 1027 else 1028 stm32_dma3_chan_complete(chan); 1029 } 1030 1031 if (csr & CSR_USEF && ccr & CCR_USEIE) { 1032 dev_err(vchan_chan_dev(&chan->vchan), "User setting error\n"); 1033 chan->dma_status = DMA_ERROR; 1034 /* CCR.EN automatically cleared by HW */ 1035 stm32_dma3_check_user_setting(chan); 1036 stm32_dma3_chan_reset(chan); 1037 } 1038 1039 if (csr & CSR_ULEF && ccr & CCR_ULEIE) { 1040 dev_err(vchan_chan_dev(&chan->vchan), "Update link transfer error\n"); 1041 chan->dma_status = DMA_ERROR; 1042 /* CCR.EN automatically cleared by HW */ 1043 stm32_dma3_chan_reset(chan); 1044 } 1045 1046 if (csr & CSR_DTEF && ccr & CCR_DTEIE) { 1047 dev_err(vchan_chan_dev(&chan->vchan), "Data transfer error\n"); 1048 chan->dma_status = DMA_ERROR; 1049 /* CCR.EN automatically cleared by HW */ 1050 stm32_dma3_chan_reset(chan); 1051 } 1052 1053 /* 1054 * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set, 1055 * ensure HTF flag to be cleared, with other flags. 1056 */ 1057 csr &= (ccr | CCR_HTIE); 1058 1059 if (csr) 1060 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id)); 1061 1062 spin_unlock(&chan->vchan.lock); 1063 1064 return IRQ_HANDLED; 1065 } 1066 1067 static int stm32_dma3_get_chan_sem(struct stm32_dma3_chan *chan) 1068 { 1069 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1070 u32 csemcr, ccid; 1071 1072 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); 1073 /* Make an attempt to take the channel semaphore if not already taken */ 1074 if (!(csemcr & CSEMCR_SEM_MUTEX)) { 1075 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(chan->id)); 1076 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); 1077 } 1078 1079 /* Check if channel is under CID1 control */ 1080 ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr); 1081 if (!(csemcr & CSEMCR_SEM_MUTEX) || ccid != CCIDCFGR_CID1) 1082 goto bad_cid; 1083 1084 chan->semaphore_taken = true; 1085 dev_dbg(vchan_chan_dev(&chan->vchan), "under CID1 control (semcr=0x%08x)\n", csemcr); 1086 1087 return 0; 1088 1089 bad_cid: 1090 chan->semaphore_taken = false; 1091 dev_err(vchan_chan_dev(&chan->vchan), "not under CID1 control (in-use by CID%d)\n", ccid); 1092 1093 return -EACCES; 1094 } 1095 1096 static void stm32_dma3_put_chan_sem(struct stm32_dma3_chan *chan) 1097 { 1098 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1099 1100 if (chan->semaphore_taken) { 1101 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id)); 1102 chan->semaphore_taken = false; 1103 dev_dbg(vchan_chan_dev(&chan->vchan), "no more under CID1 control\n"); 1104 } 1105 } 1106 1107 static int stm32_dma3_alloc_chan_resources(struct dma_chan *c) 1108 { 1109 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1110 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1111 int ret; 1112 1113 ret = pm_runtime_resume_and_get(ddata2dev(ddata)); 1114 if (ret < 0) 1115 return ret; 1116 1117 /* Ensure the channel is free */ 1118 if (chan->semaphore_mode && 1119 readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) { 1120 ret = -EBUSY; 1121 goto err_put_sync; 1122 } 1123 1124 chan->lli_pool = dmam_pool_create(dma_chan_name(c), c->device->dev, 1125 sizeof(struct stm32_dma3_hwdesc), 1126 __alignof__(struct stm32_dma3_hwdesc), SZ_64K); 1127 if (!chan->lli_pool) { 1128 dev_err(vchan_chan_dev(&chan->vchan), "Failed to create LLI pool\n"); 1129 ret = -ENOMEM; 1130 goto err_put_sync; 1131 } 1132 1133 /* Take the channel semaphore */ 1134 if (chan->semaphore_mode) { 1135 ret = stm32_dma3_get_chan_sem(chan); 1136 if (ret) 1137 goto err_pool_destroy; 1138 } 1139 1140 return 0; 1141 1142 err_pool_destroy: 1143 dmam_pool_destroy(chan->lli_pool); 1144 chan->lli_pool = NULL; 1145 1146 err_put_sync: 1147 pm_runtime_put_sync(ddata2dev(ddata)); 1148 1149 return ret; 1150 } 1151 1152 static void stm32_dma3_free_chan_resources(struct dma_chan *c) 1153 { 1154 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1155 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1156 unsigned long flags; 1157 1158 /* Ensure channel is in idle state */ 1159 spin_lock_irqsave(&chan->vchan.lock, flags); 1160 stm32_dma3_chan_stop(chan); 1161 chan->swdesc = NULL; 1162 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1163 1164 vchan_free_chan_resources(to_virt_chan(c)); 1165 1166 dmam_pool_destroy(chan->lli_pool); 1167 chan->lli_pool = NULL; 1168 1169 /* Release the channel semaphore */ 1170 if (chan->semaphore_mode) 1171 stm32_dma3_put_chan_sem(chan); 1172 1173 pm_runtime_put_sync(ddata2dev(ddata)); 1174 1175 /* Reset configuration */ 1176 memset(&chan->dt_config, 0, sizeof(chan->dt_config)); 1177 memset(&chan->dma_config, 0, sizeof(chan->dma_config)); 1178 chan->config_set = 0; 1179 } 1180 1181 static u32 stm32_dma3_get_ll_count(struct stm32_dma3_chan *chan, size_t len, bool prevent_refactor) 1182 { 1183 u32 count; 1184 1185 if (prevent_refactor) 1186 return DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE); 1187 1188 count = len / STM32_DMA3_MAX_BLOCK_SIZE; 1189 len -= (len / STM32_DMA3_MAX_BLOCK_SIZE) * STM32_DMA3_MAX_BLOCK_SIZE; 1190 1191 if (len >= chan->max_burst) { 1192 count += 1; /* len < STM32_DMA3_MAX_BLOCK_SIZE here, so it fits in one item */ 1193 len -= (len / chan->max_burst) * chan->max_burst; 1194 } 1195 1196 /* Unaligned remainder fits in one extra item */ 1197 if (len > 0) 1198 count += 1; 1199 1200 return count; 1201 } 1202 1203 static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan, 1204 dma_addr_t dst, dma_addr_t src) 1205 { 1206 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1207 u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */ 1208 u32 burst = chan->max_burst / dw; 1209 1210 /* Initialize dt_config if channel not pre-configured through DT */ 1211 if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) { 1212 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH); 1213 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size); 1214 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC; 1215 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL); 1216 } 1217 1218 /* Initialize dma_config if dmaengine_slave_config() not used */ 1219 if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) { 1220 chan->dma_config.src_addr_width = dw; 1221 chan->dma_config.dst_addr_width = dw; 1222 chan->dma_config.src_maxburst = burst; 1223 chan->dma_config.dst_maxburst = burst; 1224 chan->dma_config.src_addr = src; 1225 chan->dma_config.dst_addr = dst; 1226 } 1227 } 1228 1229 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c, 1230 dma_addr_t dst, dma_addr_t src, 1231 size_t len, unsigned long flags) 1232 { 1233 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1234 struct stm32_dma3_swdesc *swdesc; 1235 size_t next_size, offset; 1236 u32 count, i, ctr1, ctr2; 1237 bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || 1238 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); 1239 1240 /* Semaphore could be lost during suspend/resume */ 1241 if (chan->semaphore_mode && !chan->semaphore_taken) 1242 return NULL; 1243 1244 count = stm32_dma3_get_ll_count(chan, len, prevent_refactor); 1245 1246 swdesc = stm32_dma3_chan_desc_alloc(chan, count); 1247 if (!swdesc) 1248 return NULL; 1249 1250 if (chan->config_set != STM32_DMA3_CFG_SET_BOTH) 1251 stm32_dma3_init_chan_config_for_memcpy(chan, dst, src); 1252 1253 for (i = 0, offset = 0; offset < len; i++, offset += next_size) { 1254 size_t remaining; 1255 int ret; 1256 1257 remaining = len - offset; 1258 next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE); 1259 1260 if (!prevent_refactor && 1261 (next_size < STM32_DMA3_MAX_BLOCK_SIZE && next_size >= chan->max_burst)) 1262 next_size = chan->max_burst * (remaining / chan->max_burst); 1263 1264 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, 1265 src + offset, dst + offset, next_size); 1266 if (ret) 1267 goto err_desc_free; 1268 1269 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size, 1270 ctr1, ctr2, next_size == remaining, false); 1271 } 1272 1273 /* Enable Errors interrupts */ 1274 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; 1275 /* Enable Transfer state interrupts */ 1276 swdesc->ccr |= CCR_TCIE; 1277 1278 swdesc->cyclic = false; 1279 1280 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); 1281 1282 err_desc_free: 1283 stm32_dma3_chan_desc_free(chan, swdesc); 1284 1285 return NULL; 1286 } 1287 1288 static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c, 1289 struct scatterlist *sgl, 1290 unsigned int sg_len, 1291 enum dma_transfer_direction dir, 1292 unsigned long flags, void *context) 1293 { 1294 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1295 struct stm32_dma3_swdesc *swdesc; 1296 struct scatterlist *sg; 1297 size_t len; 1298 dma_addr_t sg_addr, dev_addr, src, dst; 1299 u32 i, j, count, ctr1, ctr2; 1300 bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || 1301 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); 1302 int ret; 1303 1304 /* Semaphore could be lost during suspend/resume */ 1305 if (chan->semaphore_mode && !chan->semaphore_taken) 1306 return NULL; 1307 1308 count = 0; 1309 for_each_sg(sgl, sg, sg_len, i) 1310 count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor); 1311 1312 swdesc = stm32_dma3_chan_desc_alloc(chan, count); 1313 if (!swdesc) 1314 return NULL; 1315 1316 /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */ 1317 j = 0; 1318 for_each_sg(sgl, sg, sg_len, i) { 1319 sg_addr = sg_dma_address(sg); 1320 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr : 1321 chan->dma_config.src_addr; 1322 len = sg_dma_len(sg); 1323 1324 do { 1325 size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE); 1326 1327 if (!prevent_refactor && 1328 (chunk < STM32_DMA3_MAX_BLOCK_SIZE && chunk >= chan->max_burst)) 1329 chunk = chan->max_burst * (len / chan->max_burst); 1330 1331 if (dir == DMA_MEM_TO_DEV) { 1332 src = sg_addr; 1333 dst = dev_addr; 1334 1335 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, 1336 src, dst, chunk); 1337 1338 if (FIELD_GET(CTR1_DINC, ctr1)) 1339 dev_addr += chunk; 1340 } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */ 1341 src = dev_addr; 1342 dst = sg_addr; 1343 1344 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, 1345 src, dst, chunk); 1346 1347 if (FIELD_GET(CTR1_SINC, ctr1)) 1348 dev_addr += chunk; 1349 } 1350 1351 if (ret) 1352 goto err_desc_free; 1353 1354 stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk, 1355 ctr1, ctr2, j == (count - 1), false); 1356 1357 sg_addr += chunk; 1358 len -= chunk; 1359 j++; 1360 } while (len); 1361 } 1362 1363 if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL) 1364 dev_warn(vchan_chan_dev(&chan->vchan), "Linked-list refactored, %d items instead of %d\n", 1365 count, sg_len); 1366 1367 /* Enable Error interrupts */ 1368 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; 1369 /* Enable Transfer state interrupts */ 1370 swdesc->ccr |= CCR_TCIE; 1371 1372 swdesc->cyclic = false; 1373 1374 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); 1375 1376 err_desc_free: 1377 stm32_dma3_chan_desc_free(chan, swdesc); 1378 1379 return NULL; 1380 } 1381 1382 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c, 1383 dma_addr_t buf_addr, 1384 size_t buf_len, size_t period_len, 1385 enum dma_transfer_direction dir, 1386 unsigned long flags) 1387 { 1388 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1389 struct stm32_dma3_swdesc *swdesc; 1390 dma_addr_t src, dst; 1391 u32 count, i, ctr1, ctr2; 1392 int ret; 1393 1394 /* Semaphore could be lost during suspend/resume */ 1395 if (chan->semaphore_mode && !chan->semaphore_taken) 1396 return NULL; 1397 1398 if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) { 1399 dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period length\n"); 1400 return NULL; 1401 } 1402 1403 if (buf_len % period_len) { 1404 dev_err(vchan_chan_dev(&chan->vchan), "Buffer length not multiple of period length\n"); 1405 return NULL; 1406 } 1407 1408 count = buf_len / period_len; 1409 swdesc = stm32_dma3_chan_desc_alloc(chan, count); 1410 if (!swdesc) 1411 return NULL; 1412 1413 if (dir == DMA_MEM_TO_DEV) { 1414 src = buf_addr; 1415 dst = chan->dma_config.dst_addr; 1416 1417 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2, 1418 src, dst, period_len); 1419 } else if (dir == DMA_DEV_TO_MEM) { 1420 src = chan->dma_config.src_addr; 1421 dst = buf_addr; 1422 1423 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, 1424 src, dst, period_len); 1425 } else { 1426 dev_err(vchan_chan_dev(&chan->vchan), "Invalid direction\n"); 1427 ret = -EINVAL; 1428 } 1429 1430 if (ret) 1431 goto err_desc_free; 1432 1433 for (i = 0; i < count; i++) { 1434 if (dir == DMA_MEM_TO_DEV) { 1435 src = buf_addr + i * period_len; 1436 dst = chan->dma_config.dst_addr; 1437 } else { /* (dir == DMA_DEV_TO_MEM) */ 1438 src = chan->dma_config.src_addr; 1439 dst = buf_addr + i * period_len; 1440 } 1441 1442 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len, 1443 ctr1, ctr2, i == (count - 1), true); 1444 } 1445 1446 /* Enable Error interrupts */ 1447 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; 1448 /* Enable Transfer state interrupts */ 1449 swdesc->ccr |= CCR_TCIE; 1450 1451 swdesc->cyclic = true; 1452 1453 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); 1454 1455 err_desc_free: 1456 stm32_dma3_chan_desc_free(chan, swdesc); 1457 1458 return NULL; 1459 } 1460 1461 static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps) 1462 { 1463 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1464 1465 if (!chan->fifo_size) { 1466 caps->max_burst = 0; 1467 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1468 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1469 } else { 1470 /* Burst transfer should not exceed half of the fifo size */ 1471 caps->max_burst = chan->max_burst; 1472 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) { 1473 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1474 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1475 } 1476 } 1477 } 1478 1479 static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config) 1480 { 1481 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1482 1483 memcpy(&chan->dma_config, config, sizeof(*config)); 1484 chan->config_set |= STM32_DMA3_CFG_SET_DMA; 1485 1486 return 0; 1487 } 1488 1489 static int stm32_dma3_pause(struct dma_chan *c) 1490 { 1491 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1492 int ret; 1493 1494 ret = stm32_dma3_chan_suspend(chan, true); 1495 if (ret) 1496 return ret; 1497 1498 chan->dma_status = DMA_PAUSED; 1499 1500 dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: paused\n", &chan->vchan); 1501 1502 return 0; 1503 } 1504 1505 static int stm32_dma3_resume(struct dma_chan *c) 1506 { 1507 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1508 1509 stm32_dma3_chan_suspend(chan, false); 1510 1511 chan->dma_status = DMA_IN_PROGRESS; 1512 1513 dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resumed\n", &chan->vchan); 1514 1515 return 0; 1516 } 1517 1518 static int stm32_dma3_terminate_all(struct dma_chan *c) 1519 { 1520 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1521 unsigned long flags; 1522 LIST_HEAD(head); 1523 1524 spin_lock_irqsave(&chan->vchan.lock, flags); 1525 1526 if (chan->swdesc) { 1527 vchan_terminate_vdesc(&chan->swdesc->vdesc); 1528 chan->swdesc = NULL; 1529 } 1530 1531 stm32_dma3_chan_stop(chan); 1532 1533 vchan_get_all_descriptors(&chan->vchan, &head); 1534 1535 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1536 vchan_dma_desc_free_list(&chan->vchan, &head); 1537 1538 dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: terminated\n", &chan->vchan); 1539 1540 return 0; 1541 } 1542 1543 static void stm32_dma3_synchronize(struct dma_chan *c) 1544 { 1545 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1546 1547 vchan_synchronize(&chan->vchan); 1548 } 1549 1550 static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie, 1551 struct dma_tx_state *txstate) 1552 { 1553 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1554 struct stm32_dma3_swdesc *swdesc = NULL; 1555 enum dma_status status; 1556 unsigned long flags; 1557 struct virt_dma_desc *vd; 1558 1559 status = dma_cookie_status(c, cookie, txstate); 1560 if (status == DMA_COMPLETE) 1561 return status; 1562 1563 if (!txstate) 1564 return chan->dma_status; 1565 1566 spin_lock_irqsave(&chan->vchan.lock, flags); 1567 1568 vd = vchan_find_desc(&chan->vchan, cookie); 1569 if (vd) 1570 swdesc = to_stm32_dma3_swdesc(vd); 1571 else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie) 1572 swdesc = chan->swdesc; 1573 1574 /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */ 1575 if (swdesc) 1576 stm32_dma3_chan_set_residue(chan, swdesc, txstate); 1577 1578 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1579 1580 return chan->dma_status; 1581 } 1582 1583 static void stm32_dma3_issue_pending(struct dma_chan *c) 1584 { 1585 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1586 unsigned long flags; 1587 1588 spin_lock_irqsave(&chan->vchan.lock, flags); 1589 1590 if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) { 1591 dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan); 1592 stm32_dma3_chan_start(chan); 1593 } 1594 1595 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1596 } 1597 1598 static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param) 1599 { 1600 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1601 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); 1602 struct stm32_dma3_dt_conf *conf = fn_param; 1603 u32 mask, semcr; 1604 int ret; 1605 1606 dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n", 1607 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf); 1608 1609 if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask)) 1610 if (!(mask & BIT(chan->id))) 1611 return false; 1612 1613 ret = pm_runtime_resume_and_get(ddata2dev(ddata)); 1614 if (ret < 0) 1615 return false; 1616 semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); 1617 pm_runtime_put_sync(ddata2dev(ddata)); 1618 1619 /* Check if chan is free */ 1620 if (semcr & CSEMCR_SEM_MUTEX) 1621 return false; 1622 1623 /* Check if chan fifo fits well */ 1624 if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size) 1625 return false; 1626 1627 return true; 1628 } 1629 1630 static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) 1631 { 1632 struct stm32_dma3_ddata *ddata = ofdma->of_dma_data; 1633 dma_cap_mask_t mask = ddata->dma_dev.cap_mask; 1634 struct stm32_dma3_dt_conf conf; 1635 struct stm32_dma3_chan *chan; 1636 struct dma_chan *c; 1637 1638 if (dma_spec->args_count < 3) { 1639 dev_err(ddata2dev(ddata), "Invalid args count\n"); 1640 return NULL; 1641 } 1642 1643 conf.req_line = dma_spec->args[0]; 1644 conf.ch_conf = dma_spec->args[1]; 1645 conf.tr_conf = dma_spec->args[2]; 1646 1647 if (conf.req_line >= ddata->dma_requests) { 1648 dev_err(ddata2dev(ddata), "Invalid request line\n"); 1649 return NULL; 1650 } 1651 1652 /* Request dma channel among the generic dma controller list */ 1653 c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf); 1654 if (!c) { 1655 dev_err(ddata2dev(ddata), "No suitable channel found\n"); 1656 return NULL; 1657 } 1658 1659 chan = to_stm32_dma3_chan(c); 1660 chan->dt_config = conf; 1661 chan->config_set |= STM32_DMA3_CFG_SET_DT; 1662 1663 return c; 1664 } 1665 1666 static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata) 1667 { 1668 struct device *dev = ddata2dev(ddata); 1669 u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0; 1670 1671 /* Reserve Secure channels */ 1672 chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR); 1673 1674 /* 1675 * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of 1676 * the processor which is configuring and using the given channel. 1677 * In case CID filtering is not configured, dma-channel-mask property can be used to 1678 * specify available DMA channels to the kernel. 1679 */ 1680 of_property_read_u32(dev->of_node, "dma-channel-mask", &mask); 1681 1682 /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */ 1683 for (i = 0; i < ddata->dma_channels; i++) { 1684 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i)); 1685 1686 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */ 1687 invalid_cid |= BIT(i); 1688 if (!(mask & BIT(i))) /* Not in dma-channel-mask */ 1689 chan_reserved |= BIT(i); 1690 } else { /* CID-filtered */ 1691 if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */ 1692 if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1) 1693 chan_reserved |= BIT(i); 1694 } else { /* Semaphore mode */ 1695 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr)) 1696 chan_reserved |= BIT(i); 1697 ddata->chans[i].semaphore_mode = true; 1698 } 1699 } 1700 dev_dbg(dev, "chan%d: %s mode, %s\n", i, 1701 !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" : 1702 ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID", 1703 (chan_reserved & BIT(i)) ? "denied" : 1704 mask & BIT(i) ? "force allowed" : "allowed"); 1705 } 1706 1707 if (invalid_cid) 1708 dev_warn(dev, "chan%*pbl have invalid CID configuration\n", 1709 ddata->dma_channels, &invalid_cid); 1710 1711 return chan_reserved; 1712 } 1713 1714 static struct stm32_dma3_pdata stm32mp25_pdata = { 1715 .axi_max_burst_len = 16, 1716 }; 1717 1718 static const struct of_device_id stm32_dma3_of_match[] = { 1719 { .compatible = "st,stm32mp25-dma3", .data = &stm32mp25_pdata, }, 1720 { /* sentinel */ }, 1721 }; 1722 MODULE_DEVICE_TABLE(of, stm32_dma3_of_match); 1723 1724 static int stm32_dma3_probe(struct platform_device *pdev) 1725 { 1726 struct device_node *np = pdev->dev.of_node; 1727 const struct stm32_dma3_pdata *pdata; 1728 struct stm32_dma3_ddata *ddata; 1729 struct reset_control *reset; 1730 struct stm32_dma3_chan *chan; 1731 struct dma_device *dma_dev; 1732 u32 master_ports, chan_reserved, i, verr; 1733 u64 hwcfgr; 1734 int ret; 1735 1736 ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); 1737 if (!ddata) 1738 return -ENOMEM; 1739 platform_set_drvdata(pdev, ddata); 1740 1741 dma_dev = &ddata->dma_dev; 1742 1743 ddata->base = devm_platform_ioremap_resource(pdev, 0); 1744 if (IS_ERR(ddata->base)) 1745 return PTR_ERR(ddata->base); 1746 1747 ddata->clk = devm_clk_get(&pdev->dev, NULL); 1748 if (IS_ERR(ddata->clk)) 1749 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n"); 1750 1751 reset = devm_reset_control_get_optional(&pdev->dev, NULL); 1752 if (IS_ERR(reset)) 1753 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n"); 1754 1755 ret = clk_prepare_enable(ddata->clk); 1756 if (ret) 1757 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n"); 1758 1759 reset_control_reset(reset); 1760 1761 INIT_LIST_HEAD(&dma_dev->channels); 1762 1763 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 1764 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); 1765 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); 1766 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1767 dma_dev->dev = &pdev->dev; 1768 /* 1769 * This controller supports up to 8-byte buswidth depending on the port used and the 1770 * channel, and can only access address at even boundaries, multiple of the buswidth. 1771 */ 1772 dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES; 1773 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1774 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1775 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1776 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1777 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1778 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1779 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1780 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1781 dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM); 1782 1783 dma_dev->descriptor_reuse = true; 1784 dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE; 1785 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1786 dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources; 1787 dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources; 1788 dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy; 1789 dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg; 1790 dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic; 1791 dma_dev->device_caps = stm32_dma3_caps; 1792 dma_dev->device_config = stm32_dma3_config; 1793 dma_dev->device_pause = stm32_dma3_pause; 1794 dma_dev->device_resume = stm32_dma3_resume; 1795 dma_dev->device_terminate_all = stm32_dma3_terminate_all; 1796 dma_dev->device_synchronize = stm32_dma3_synchronize; 1797 dma_dev->device_tx_status = stm32_dma3_tx_status; 1798 dma_dev->device_issue_pending = stm32_dma3_issue_pending; 1799 1800 /* if dma_channels is not modified, get it from hwcfgr1 */ 1801 if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) { 1802 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); 1803 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr); 1804 } 1805 1806 /* if dma_requests is not modified, get it from hwcfgr2 */ 1807 if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) { 1808 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2); 1809 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1; 1810 } 1811 1812 /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */ 1813 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); 1814 master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr); 1815 1816 ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr); 1817 if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */ 1818 ddata->ports_max_dw[1] = DW_INVALID; 1819 else /* Dual master ports */ 1820 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr); 1821 1822 /* axi_max_burst_len is optional, if not defined, use STM32_DMA3_MAX_BURST_LEN */ 1823 ddata->axi_max_burst_len = STM32_DMA3_MAX_BURST_LEN; 1824 pdata = device_get_match_data(&pdev->dev); 1825 if (pdata && pdata->axi_max_burst_len) { 1826 ddata->axi_max_burst_len = min_t(u32, pdata->axi_max_burst_len, 1827 STM32_DMA3_MAX_BURST_LEN); 1828 dev_dbg(&pdev->dev, "Burst is limited to %u beats through AXI port\n", 1829 ddata->axi_max_burst_len); 1830 } 1831 1832 ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans), 1833 GFP_KERNEL); 1834 if (!ddata->chans) { 1835 ret = -ENOMEM; 1836 goto err_clk_disable; 1837 } 1838 1839 chan_reserved = stm32_dma3_check_rif(ddata); 1840 1841 if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) { 1842 ret = -ENODEV; 1843 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n"); 1844 goto err_clk_disable; 1845 } 1846 1847 /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */ 1848 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3); 1849 hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32; 1850 1851 for (i = 0; i < ddata->dma_channels; i++) { 1852 if (chan_reserved & BIT(i)) 1853 continue; 1854 1855 chan = &ddata->chans[i]; 1856 chan->id = i; 1857 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr); 1858 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */ 1859 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0; 1860 } 1861 1862 ret = dmaenginem_async_device_register(dma_dev); 1863 if (ret) 1864 goto err_clk_disable; 1865 1866 for (i = 0; i < ddata->dma_channels; i++) { 1867 char name[12]; 1868 1869 if (chan_reserved & BIT(i)) 1870 continue; 1871 1872 chan = &ddata->chans[i]; 1873 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id); 1874 1875 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free; 1876 vchan_init(&chan->vchan, dma_dev); 1877 1878 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name); 1879 if (ret) { 1880 dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name); 1881 goto err_clk_disable; 1882 } 1883 1884 ret = platform_get_irq(pdev, i); 1885 if (ret < 0) 1886 goto err_clk_disable; 1887 chan->irq = ret; 1888 1889 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0, 1890 vchan_chan_name(&chan->vchan), chan); 1891 if (ret) { 1892 dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n", 1893 vchan_chan_name(&chan->vchan)); 1894 goto err_clk_disable; 1895 } 1896 } 1897 1898 ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata); 1899 if (ret) { 1900 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n"); 1901 goto err_clk_disable; 1902 } 1903 1904 verr = readl_relaxed(ddata->base + STM32_DMA3_VERR); 1905 1906 pm_runtime_set_active(&pdev->dev); 1907 pm_runtime_enable(&pdev->dev); 1908 pm_runtime_get_noresume(&pdev->dev); 1909 pm_runtime_put(&pdev->dev); 1910 1911 dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n", 1912 FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr)); 1913 1914 return 0; 1915 1916 err_clk_disable: 1917 clk_disable_unprepare(ddata->clk); 1918 1919 return ret; 1920 } 1921 1922 static void stm32_dma3_remove(struct platform_device *pdev) 1923 { 1924 pm_runtime_disable(&pdev->dev); 1925 } 1926 1927 static int stm32_dma3_runtime_suspend(struct device *dev) 1928 { 1929 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); 1930 1931 clk_disable_unprepare(ddata->clk); 1932 1933 return 0; 1934 } 1935 1936 static int stm32_dma3_runtime_resume(struct device *dev) 1937 { 1938 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); 1939 int ret; 1940 1941 ret = clk_prepare_enable(ddata->clk); 1942 if (ret) 1943 dev_err(dev, "Failed to enable clk: %d\n", ret); 1944 1945 return ret; 1946 } 1947 1948 static int stm32_dma3_pm_suspend(struct device *dev) 1949 { 1950 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); 1951 struct dma_device *dma_dev = &ddata->dma_dev; 1952 struct dma_chan *c; 1953 int ccr, ret; 1954 1955 ret = pm_runtime_resume_and_get(dev); 1956 if (ret < 0) 1957 return ret; 1958 1959 list_for_each_entry(c, &dma_dev->channels, device_node) { 1960 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1961 1962 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)); 1963 if (ccr & CCR_EN) { 1964 dev_warn(dev, "Suspend is prevented: %s still in use by %s\n", 1965 dma_chan_name(c), dev_name(c->slave)); 1966 pm_runtime_put_sync(dev); 1967 return -EBUSY; 1968 } 1969 } 1970 1971 pm_runtime_put_sync(dev); 1972 1973 pm_runtime_force_suspend(dev); 1974 1975 return 0; 1976 } 1977 1978 static int stm32_dma3_pm_resume(struct device *dev) 1979 { 1980 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); 1981 struct dma_device *dma_dev = &ddata->dma_dev; 1982 struct dma_chan *c; 1983 int ret; 1984 1985 ret = pm_runtime_force_resume(dev); 1986 if (ret < 0) 1987 return ret; 1988 1989 ret = pm_runtime_resume_and_get(dev); 1990 if (ret < 0) 1991 return ret; 1992 1993 /* 1994 * Channel semaphores need to be restored in case of registers reset during low power. 1995 * stm32_dma3_get_chan_sem() will prior check the semaphore status. 1996 */ 1997 list_for_each_entry(c, &dma_dev->channels, device_node) { 1998 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); 1999 2000 if (chan->semaphore_mode && chan->semaphore_taken) 2001 stm32_dma3_get_chan_sem(chan); 2002 } 2003 2004 pm_runtime_put_sync(dev); 2005 2006 return 0; 2007 } 2008 2009 static const struct dev_pm_ops stm32_dma3_pm_ops = { 2010 SYSTEM_SLEEP_PM_OPS(stm32_dma3_pm_suspend, stm32_dma3_pm_resume) 2011 RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL) 2012 }; 2013 2014 static struct platform_driver stm32_dma3_driver = { 2015 .probe = stm32_dma3_probe, 2016 .remove = stm32_dma3_remove, 2017 .driver = { 2018 .name = "stm32-dma3", 2019 .of_match_table = stm32_dma3_of_match, 2020 .pm = pm_ptr(&stm32_dma3_pm_ops), 2021 }, 2022 }; 2023 2024 module_platform_driver(stm32_dma3_driver); 2025 2026 MODULE_DESCRIPTION("STM32 DMA3 controller driver"); 2027 MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>"); 2028 MODULE_LICENSE("GPL"); 2029