1c01faacaSFabio Estevam // SPDX-License-Identifier: GPL-2.0+
2c01faacaSFabio Estevam //
3c01faacaSFabio Estevam // drivers/dma/imx-sdma.c
4c01faacaSFabio Estevam //
5c01faacaSFabio Estevam // This file contains a driver for the Freescale Smart DMA engine
6c01faacaSFabio Estevam //
7c01faacaSFabio Estevam // Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
8c01faacaSFabio Estevam //
9c01faacaSFabio Estevam // Based on code from Freescale:
10c01faacaSFabio Estevam //
11c01faacaSFabio Estevam // Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
121ec1e82fSSascha Hauer
131ec1e82fSSascha Hauer #include <linux/init.h>
141d069bfaSMichael Olbrich #include <linux/iopoll.h>
15f8de8f4cSAxel Lin #include <linux/module.h>
161ec1e82fSSascha Hauer #include <linux/types.h>
17824a0a02SSascha Hauer #include <linux/bitfield.h>
180bbc1413SRichard Zhao #include <linux/bitops.h>
191ec1e82fSSascha Hauer #include <linux/mm.h>
201ec1e82fSSascha Hauer #include <linux/interrupt.h>
211ec1e82fSSascha Hauer #include <linux/clk.h>
222ccaef05SRichard Zhao #include <linux/delay.h>
231ec1e82fSSascha Hauer #include <linux/sched.h>
241ec1e82fSSascha Hauer #include <linux/semaphore.h>
251ec1e82fSSascha Hauer #include <linux/spinlock.h>
261ec1e82fSSascha Hauer #include <linux/device.h>
27802ef223SNicolin Chen #include <linux/genalloc.h>
281ec1e82fSSascha Hauer #include <linux/dma-mapping.h>
291ec1e82fSSascha Hauer #include <linux/firmware.h>
301ec1e82fSSascha Hauer #include <linux/slab.h>
311ec1e82fSSascha Hauer #include <linux/platform_device.h>
321ec1e82fSSascha Hauer #include <linux/dmaengine.h>
33580975d7SShawn Guo #include <linux/of.h>
348391ecf4SShengjiu Wang #include <linux/of_address.h>
359479e17cSShawn Guo #include <linux/of_dma.h>
36b8603d2aSLucas Stach #include <linux/workqueue.h>
371ec1e82fSSascha Hauer
381ec1e82fSSascha Hauer #include <asm/irq.h>
39c6547c2eSSascha Hauer #include <linux/dma/imx-dma.h>
40d078cd1bSZidan Wang #include <linux/regmap.h>
41d078cd1bSZidan Wang #include <linux/mfd/syscon.h>
42d078cd1bSZidan Wang #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
431ec1e82fSSascha Hauer
44d2ebfb33SRussell King - ARM Linux #include "dmaengine.h"
4557b772b8SRobin Gong #include "virt-dma.h"
46d2ebfb33SRussell King - ARM Linux
471ec1e82fSSascha Hauer /* SDMA registers */
481ec1e82fSSascha Hauer #define SDMA_H_C0PTR 0x000
491ec1e82fSSascha Hauer #define SDMA_H_INTR 0x004
501ec1e82fSSascha Hauer #define SDMA_H_STATSTOP 0x008
511ec1e82fSSascha Hauer #define SDMA_H_START 0x00c
521ec1e82fSSascha Hauer #define SDMA_H_EVTOVR 0x010
531ec1e82fSSascha Hauer #define SDMA_H_DSPOVR 0x014
541ec1e82fSSascha Hauer #define SDMA_H_HOSTOVR 0x018
551ec1e82fSSascha Hauer #define SDMA_H_EVTPEND 0x01c
561ec1e82fSSascha Hauer #define SDMA_H_DSPENBL 0x020
571ec1e82fSSascha Hauer #define SDMA_H_RESET 0x024
581ec1e82fSSascha Hauer #define SDMA_H_EVTERR 0x028
591ec1e82fSSascha Hauer #define SDMA_H_INTRMSK 0x02c
601ec1e82fSSascha Hauer #define SDMA_H_PSW 0x030
611ec1e82fSSascha Hauer #define SDMA_H_EVTERRDBG 0x034
621ec1e82fSSascha Hauer #define SDMA_H_CONFIG 0x038
631ec1e82fSSascha Hauer #define SDMA_ONCE_ENB 0x040
641ec1e82fSSascha Hauer #define SDMA_ONCE_DATA 0x044
651ec1e82fSSascha Hauer #define SDMA_ONCE_INSTR 0x048
661ec1e82fSSascha Hauer #define SDMA_ONCE_STAT 0x04c
671ec1e82fSSascha Hauer #define SDMA_ONCE_CMD 0x050
681ec1e82fSSascha Hauer #define SDMA_EVT_MIRROR 0x054
691ec1e82fSSascha Hauer #define SDMA_ILLINSTADDR 0x058
701ec1e82fSSascha Hauer #define SDMA_CHN0ADDR 0x05c
711ec1e82fSSascha Hauer #define SDMA_ONCE_RTB 0x060
721ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF1 0x070
731ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF2 0x074
7462550cd7SShawn Guo #define SDMA_CHNENBL0_IMX35 0x200
7562550cd7SShawn Guo #define SDMA_CHNENBL0_IMX31 0x080
761ec1e82fSSascha Hauer #define SDMA_CHNPRI_0 0x100
77824a0a02SSascha Hauer #define SDMA_DONE0_CONFIG 0x1000
781ec1e82fSSascha Hauer
791ec1e82fSSascha Hauer /*
801ec1e82fSSascha Hauer * Buffer descriptor status values.
811ec1e82fSSascha Hauer */
821ec1e82fSSascha Hauer #define BD_DONE 0x01
831ec1e82fSSascha Hauer #define BD_WRAP 0x02
841ec1e82fSSascha Hauer #define BD_CONT 0x04
851ec1e82fSSascha Hauer #define BD_INTR 0x08
861ec1e82fSSascha Hauer #define BD_RROR 0x10
871ec1e82fSSascha Hauer #define BD_LAST 0x20
881ec1e82fSSascha Hauer #define BD_EXTD 0x80
891ec1e82fSSascha Hauer
901ec1e82fSSascha Hauer /*
911ec1e82fSSascha Hauer * Data Node descriptor status values.
921ec1e82fSSascha Hauer */
931ec1e82fSSascha Hauer #define DND_END_OF_FRAME 0x80
941ec1e82fSSascha Hauer #define DND_END_OF_XFER 0x40
951ec1e82fSSascha Hauer #define DND_DONE 0x20
961ec1e82fSSascha Hauer #define DND_UNUSED 0x01
971ec1e82fSSascha Hauer
981ec1e82fSSascha Hauer /*
991ec1e82fSSascha Hauer * IPCV2 descriptor status values.
1001ec1e82fSSascha Hauer */
1011ec1e82fSSascha Hauer #define BD_IPCV2_END_OF_FRAME 0x40
1021ec1e82fSSascha Hauer
1031ec1e82fSSascha Hauer #define IPCV2_MAX_NODES 50
1041ec1e82fSSascha Hauer /*
1051ec1e82fSSascha Hauer * Error bit set in the CCB status field by the SDMA,
1061ec1e82fSSascha Hauer * in setbd routine, in case of a transfer error
1071ec1e82fSSascha Hauer */
1081ec1e82fSSascha Hauer #define DATA_ERROR 0x10000000
1091ec1e82fSSascha Hauer
1101ec1e82fSSascha Hauer /*
1111ec1e82fSSascha Hauer * Buffer descriptor commands.
1121ec1e82fSSascha Hauer */
1131ec1e82fSSascha Hauer #define C0_ADDR 0x01
1141ec1e82fSSascha Hauer #define C0_LOAD 0x02
1151ec1e82fSSascha Hauer #define C0_DUMP 0x03
1161ec1e82fSSascha Hauer #define C0_SETCTX 0x07
1171ec1e82fSSascha Hauer #define C0_GETCTX 0x03
1181ec1e82fSSascha Hauer #define C0_SETDM 0x01
1191ec1e82fSSascha Hauer #define C0_SETPM 0x04
1201ec1e82fSSascha Hauer #define C0_GETDM 0x02
1211ec1e82fSSascha Hauer #define C0_GETPM 0x08
1221ec1e82fSSascha Hauer /*
1231ec1e82fSSascha Hauer * Change endianness indicator in the BD command field
1241ec1e82fSSascha Hauer */
1251ec1e82fSSascha Hauer #define CHANGE_ENDIANNESS 0x80
1261ec1e82fSSascha Hauer
1271ec1e82fSSascha Hauer /*
1288391ecf4SShengjiu Wang * p_2_p watermark_level description
1298391ecf4SShengjiu Wang * Bits Name Description
1308391ecf4SShengjiu Wang * 0-7 Lower WML Lower watermark level
1318391ecf4SShengjiu Wang * 8 PS 1: Pad Swallowing
1328391ecf4SShengjiu Wang * 0: No Pad Swallowing
1338391ecf4SShengjiu Wang * 9 PA 1: Pad Adding
1348391ecf4SShengjiu Wang * 0: No Pad Adding
1358391ecf4SShengjiu Wang * 10 SPDIF If this bit is set both source
1368391ecf4SShengjiu Wang * and destination are on SPBA
1378391ecf4SShengjiu Wang * 11 Source Bit(SP) 1: Source on SPBA
1388391ecf4SShengjiu Wang * 0: Source on AIPS
1398391ecf4SShengjiu Wang * 12 Destination Bit(DP) 1: Destination on SPBA
1408391ecf4SShengjiu Wang * 0: Destination on AIPS
141a20f10d6SShengjiu Wang * 13 Source FIFO 1: Source is dual FIFO
142a20f10d6SShengjiu Wang * 0: Source is single FIFO
143a20f10d6SShengjiu Wang * 14 Destination FIFO 1: Destination is dual FIFO
144a20f10d6SShengjiu Wang * 0: Destination is single FIFO
145a20f10d6SShengjiu Wang * 15 --------- MUST BE 0
1468391ecf4SShengjiu Wang * 16-23 Higher WML HWML
1478391ecf4SShengjiu Wang * 24-27 N Total number of samples after
1488391ecf4SShengjiu Wang * which Pad adding/Swallowing
1498391ecf4SShengjiu Wang * must be done. It must be odd.
1508391ecf4SShengjiu Wang * 28 Lower WML Event(LWE) SDMA events reg to check for
1518391ecf4SShengjiu Wang * LWML event mask
1528391ecf4SShengjiu Wang * 0: LWE in EVENTS register
1538391ecf4SShengjiu Wang * 1: LWE in EVENTS2 register
1548391ecf4SShengjiu Wang * 29 Higher WML Event(HWE) SDMA events reg to check for
1558391ecf4SShengjiu Wang * HWML event mask
1568391ecf4SShengjiu Wang * 0: HWE in EVENTS register
1578391ecf4SShengjiu Wang * 1: HWE in EVENTS2 register
1588391ecf4SShengjiu Wang * 30 --------- MUST BE 0
1598391ecf4SShengjiu Wang * 31 CONT 1: Amount of samples to be
1608391ecf4SShengjiu Wang * transferred is unknown and
1618391ecf4SShengjiu Wang * script will keep on
1628391ecf4SShengjiu Wang * transferring samples as long as
1638391ecf4SShengjiu Wang * both events are detected and
1648391ecf4SShengjiu Wang * script must be manually stopped
1658391ecf4SShengjiu Wang * by the application
1668391ecf4SShengjiu Wang * 0: The amount of samples to be
1678391ecf4SShengjiu Wang * transferred is equal to the
1688391ecf4SShengjiu Wang * count field of mode word
1698391ecf4SShengjiu Wang */
1708391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_LWML 0xFF
1718391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_PS BIT(8)
1728391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_PA BIT(9)
1738391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
1748391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_SP BIT(11)
1758391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_DP BIT(12)
176a20f10d6SShengjiu Wang #define SDMA_WATERMARK_LEVEL_SD BIT(13)
177a20f10d6SShengjiu Wang #define SDMA_WATERMARK_LEVEL_DD BIT(14)
1788391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
1798391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_LWE BIT(28)
1808391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_HWE BIT(29)
1818391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_CONT BIT(31)
1828391ecf4SShengjiu Wang
183f9d4a398SNicolin Chen #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
184f9d4a398SNicolin Chen BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
18528810938SShengjiu Wang BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
186f9d4a398SNicolin Chen BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
187f9d4a398SNicolin Chen
188f9d4a398SNicolin Chen #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
189f9d4a398SNicolin Chen BIT(DMA_MEM_TO_DEV) | \
190f9d4a398SNicolin Chen BIT(DMA_DEV_TO_DEV))
191f9d4a398SNicolin Chen
192824a0a02SSascha Hauer #define SDMA_WATERMARK_LEVEL_N_FIFOS GENMASK(15, 12)
193e0c7ea83SShengjiu Wang #define SDMA_WATERMARK_LEVEL_OFF_FIFOS GENMASK(19, 16)
194e0c7ea83SShengjiu Wang #define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO GENMASK(31, 28)
195824a0a02SSascha Hauer #define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23)
196824a0a02SSascha Hauer
197824a0a02SSascha Hauer #define SDMA_DONE0_CONFIG_DONE_SEL BIT(7)
198824a0a02SSascha Hauer #define SDMA_DONE0_CONFIG_DONE_DIS BIT(6)
199824a0a02SSascha Hauer
20001eafd4bSShengjiu Wang /*
2018d11cfb0SVladimir Zapolskiy * struct sdma_script_start_addrs - SDMA script start pointers
2028d11cfb0SVladimir Zapolskiy *
2038d11cfb0SVladimir Zapolskiy * start addresses of the different functions in the physical
2048d11cfb0SVladimir Zapolskiy * address space of the SDMA engine.
2058d11cfb0SVladimir Zapolskiy */
2068d11cfb0SVladimir Zapolskiy struct sdma_script_start_addrs {
2078d11cfb0SVladimir Zapolskiy s32 ap_2_ap_addr;
2088d11cfb0SVladimir Zapolskiy s32 ap_2_bp_addr;
2098d11cfb0SVladimir Zapolskiy s32 ap_2_ap_fixed_addr;
2108d11cfb0SVladimir Zapolskiy s32 bp_2_ap_addr;
2118d11cfb0SVladimir Zapolskiy s32 loopback_on_dsp_side_addr;
2128d11cfb0SVladimir Zapolskiy s32 mcu_interrupt_only_addr;
2138d11cfb0SVladimir Zapolskiy s32 firi_2_per_addr;
2148d11cfb0SVladimir Zapolskiy s32 firi_2_mcu_addr;
2158d11cfb0SVladimir Zapolskiy s32 per_2_firi_addr;
2168d11cfb0SVladimir Zapolskiy s32 mcu_2_firi_addr;
2178d11cfb0SVladimir Zapolskiy s32 uart_2_per_addr;
218a3ae97f4SKevin Groeneveld s32 uart_2_mcu_addr;
2198d11cfb0SVladimir Zapolskiy s32 per_2_app_addr;
2208d11cfb0SVladimir Zapolskiy s32 mcu_2_app_addr;
2218d11cfb0SVladimir Zapolskiy s32 per_2_per_addr;
2228d11cfb0SVladimir Zapolskiy s32 uartsh_2_per_addr;
223a3ae97f4SKevin Groeneveld s32 uartsh_2_mcu_addr;
2248d11cfb0SVladimir Zapolskiy s32 per_2_shp_addr;
2258d11cfb0SVladimir Zapolskiy s32 mcu_2_shp_addr;
2268d11cfb0SVladimir Zapolskiy s32 ata_2_mcu_addr;
2278d11cfb0SVladimir Zapolskiy s32 mcu_2_ata_addr;
2288d11cfb0SVladimir Zapolskiy s32 app_2_per_addr;
2298d11cfb0SVladimir Zapolskiy s32 app_2_mcu_addr;
2308d11cfb0SVladimir Zapolskiy s32 shp_2_per_addr;
2318d11cfb0SVladimir Zapolskiy s32 shp_2_mcu_addr;
2328d11cfb0SVladimir Zapolskiy s32 mshc_2_mcu_addr;
2338d11cfb0SVladimir Zapolskiy s32 mcu_2_mshc_addr;
2348d11cfb0SVladimir Zapolskiy s32 spdif_2_mcu_addr;
2358d11cfb0SVladimir Zapolskiy s32 mcu_2_spdif_addr;
2368d11cfb0SVladimir Zapolskiy s32 asrc_2_mcu_addr;
2378d11cfb0SVladimir Zapolskiy s32 ext_mem_2_ipu_addr;
2388d11cfb0SVladimir Zapolskiy s32 descrambler_addr;
2398d11cfb0SVladimir Zapolskiy s32 dptc_dvfs_addr;
2408d11cfb0SVladimir Zapolskiy s32 utra_addr;
2418d11cfb0SVladimir Zapolskiy s32 ram_code_start_addr;
2428d11cfb0SVladimir Zapolskiy /* End of v1 array */
2431cb49f38SFrank Li union { s32 v1_end; s32 mcu_2_ssish_addr; };
2448d11cfb0SVladimir Zapolskiy s32 ssish_2_mcu_addr;
2458d11cfb0SVladimir Zapolskiy s32 hdmi_dma_addr;
2468d11cfb0SVladimir Zapolskiy /* End of v2 array */
2471cb49f38SFrank Li union { s32 v2_end; s32 zcanfd_2_mcu_addr; };
2488d11cfb0SVladimir Zapolskiy s32 zqspi_2_mcu_addr;
2498d11cfb0SVladimir Zapolskiy s32 mcu_2_ecspi_addr;
250b98ce2f4SRobin Gong s32 mcu_2_sai_addr;
251b98ce2f4SRobin Gong s32 sai_2_mcu_addr;
252a3ae97f4SKevin Groeneveld s32 uart_2_mcu_rom_addr;
253a3ae97f4SKevin Groeneveld s32 uartsh_2_mcu_rom_addr;
254d850b5baSRobin Gong s32 i2c_2_mcu_addr;
255d850b5baSRobin Gong s32 mcu_2_i2c_addr;
2568d11cfb0SVladimir Zapolskiy /* End of v3 array */
2571cb49f38SFrank Li union { s32 v3_end; s32 mcu_2_zqspi_addr; };
2588d11cfb0SVladimir Zapolskiy /* End of v4 array */
2591cb49f38SFrank Li s32 v4_end[0];
2608d11cfb0SVladimir Zapolskiy };
2618d11cfb0SVladimir Zapolskiy
2628391ecf4SShengjiu Wang /*
2631ec1e82fSSascha Hauer * Mode/Count of data node descriptors - IPCv2
2641ec1e82fSSascha Hauer */
2651ec1e82fSSascha Hauer struct sdma_mode_count {
2664a6b2e8aSRobin Gong #define SDMA_BD_MAX_CNT 0xffff
2671ec1e82fSSascha Hauer u32 count : 16; /* size of the buffer pointed by this BD */
2681ec1e82fSSascha Hauer u32 status : 8; /* E,R,I,C,W,D status bits stored here */
269e4b75760SMartin Kaiser u32 command : 8; /* command mostly used for channel 0 */
2701ec1e82fSSascha Hauer };
2711ec1e82fSSascha Hauer
2721ec1e82fSSascha Hauer /*
2731ec1e82fSSascha Hauer * Buffer descriptor
2741ec1e82fSSascha Hauer */
2751ec1e82fSSascha Hauer struct sdma_buffer_descriptor {
2761ec1e82fSSascha Hauer struct sdma_mode_count mode;
2771ec1e82fSSascha Hauer u32 buffer_addr; /* address of the buffer described */
2781ec1e82fSSascha Hauer u32 ext_buffer_addr; /* extended buffer address */
2791ec1e82fSSascha Hauer } __attribute__ ((packed));
2801ec1e82fSSascha Hauer
2811ec1e82fSSascha Hauer /**
2821ec1e82fSSascha Hauer * struct sdma_channel_control - Channel control Block
2831ec1e82fSSascha Hauer *
28424ca312dSRobin Gong * @current_bd_ptr: current buffer descriptor processed
28524ca312dSRobin Gong * @base_bd_ptr: first element of buffer descriptor array
28624ca312dSRobin Gong * @unused: padding. The SDMA engine expects an array of 128 byte
2871ec1e82fSSascha Hauer * control blocks
2881ec1e82fSSascha Hauer */
2891ec1e82fSSascha Hauer struct sdma_channel_control {
2901ec1e82fSSascha Hauer u32 current_bd_ptr;
2911ec1e82fSSascha Hauer u32 base_bd_ptr;
2921ec1e82fSSascha Hauer u32 unused[2];
2931ec1e82fSSascha Hauer } __attribute__ ((packed));
2941ec1e82fSSascha Hauer
2951ec1e82fSSascha Hauer /**
2961ec1e82fSSascha Hauer * struct sdma_state_registers - SDMA context for a channel
2971ec1e82fSSascha Hauer *
2981ec1e82fSSascha Hauer * @pc: program counter
29924ca312dSRobin Gong * @unused1: unused
3001ec1e82fSSascha Hauer * @t: test bit: status of arithmetic & test instruction
3011ec1e82fSSascha Hauer * @rpc: return program counter
30224ca312dSRobin Gong * @unused0: unused
3031ec1e82fSSascha Hauer * @sf: source fault while loading data
3041ec1e82fSSascha Hauer * @spc: loop start program counter
30524ca312dSRobin Gong * @unused2: unused
3061ec1e82fSSascha Hauer * @df: destination fault while storing data
3071ec1e82fSSascha Hauer * @epc: loop end program counter
3081ec1e82fSSascha Hauer * @lm: loop mode
3091ec1e82fSSascha Hauer */
3101ec1e82fSSascha Hauer struct sdma_state_registers {
3111ec1e82fSSascha Hauer u32 pc :14;
3121ec1e82fSSascha Hauer u32 unused1: 1;
3131ec1e82fSSascha Hauer u32 t : 1;
3141ec1e82fSSascha Hauer u32 rpc :14;
3151ec1e82fSSascha Hauer u32 unused0: 1;
3161ec1e82fSSascha Hauer u32 sf : 1;
3171ec1e82fSSascha Hauer u32 spc :14;
3181ec1e82fSSascha Hauer u32 unused2: 1;
3191ec1e82fSSascha Hauer u32 df : 1;
3201ec1e82fSSascha Hauer u32 epc :14;
3211ec1e82fSSascha Hauer u32 lm : 2;
3221ec1e82fSSascha Hauer } __attribute__ ((packed));
3231ec1e82fSSascha Hauer
3241ec1e82fSSascha Hauer /**
3251ec1e82fSSascha Hauer * struct sdma_context_data - sdma context specific to a channel
3261ec1e82fSSascha Hauer *
3271ec1e82fSSascha Hauer * @channel_state: channel state bits
3281ec1e82fSSascha Hauer * @gReg: general registers
3291ec1e82fSSascha Hauer * @mda: burst dma destination address register
3301ec1e82fSSascha Hauer * @msa: burst dma source address register
3311ec1e82fSSascha Hauer * @ms: burst dma status register
3321ec1e82fSSascha Hauer * @md: burst dma data register
3331ec1e82fSSascha Hauer * @pda: peripheral dma destination address register
3341ec1e82fSSascha Hauer * @psa: peripheral dma source address register
3351ec1e82fSSascha Hauer * @ps: peripheral dma status register
3361ec1e82fSSascha Hauer * @pd: peripheral dma data register
3371ec1e82fSSascha Hauer * @ca: CRC polynomial register
3381ec1e82fSSascha Hauer * @cs: CRC accumulator register
3391ec1e82fSSascha Hauer * @dda: dedicated core destination address register
3401ec1e82fSSascha Hauer * @dsa: dedicated core source address register
3411ec1e82fSSascha Hauer * @ds: dedicated core status register
3421ec1e82fSSascha Hauer * @dd: dedicated core data register
34324ca312dSRobin Gong * @scratch0: 1st word of dedicated ram for context switch
34424ca312dSRobin Gong * @scratch1: 2nd word of dedicated ram for context switch
34524ca312dSRobin Gong * @scratch2: 3rd word of dedicated ram for context switch
34624ca312dSRobin Gong * @scratch3: 4th word of dedicated ram for context switch
34724ca312dSRobin Gong * @scratch4: 5th word of dedicated ram for context switch
34824ca312dSRobin Gong * @scratch5: 6th word of dedicated ram for context switch
34924ca312dSRobin Gong * @scratch6: 7th word of dedicated ram for context switch
35024ca312dSRobin Gong * @scratch7: 8th word of dedicated ram for context switch
3511ec1e82fSSascha Hauer */
3521ec1e82fSSascha Hauer struct sdma_context_data {
3531ec1e82fSSascha Hauer struct sdma_state_registers channel_state;
3541ec1e82fSSascha Hauer u32 gReg[8];
3551ec1e82fSSascha Hauer u32 mda;
3561ec1e82fSSascha Hauer u32 msa;
3571ec1e82fSSascha Hauer u32 ms;
3581ec1e82fSSascha Hauer u32 md;
3591ec1e82fSSascha Hauer u32 pda;
3601ec1e82fSSascha Hauer u32 psa;
3611ec1e82fSSascha Hauer u32 ps;
3621ec1e82fSSascha Hauer u32 pd;
3631ec1e82fSSascha Hauer u32 ca;
3641ec1e82fSSascha Hauer u32 cs;
3651ec1e82fSSascha Hauer u32 dda;
3661ec1e82fSSascha Hauer u32 dsa;
3671ec1e82fSSascha Hauer u32 ds;
3681ec1e82fSSascha Hauer u32 dd;
3691ec1e82fSSascha Hauer u32 scratch0;
3701ec1e82fSSascha Hauer u32 scratch1;
3711ec1e82fSSascha Hauer u32 scratch2;
3721ec1e82fSSascha Hauer u32 scratch3;
3731ec1e82fSSascha Hauer u32 scratch4;
3741ec1e82fSSascha Hauer u32 scratch5;
3751ec1e82fSSascha Hauer u32 scratch6;
3761ec1e82fSSascha Hauer u32 scratch7;
3771ec1e82fSSascha Hauer } __attribute__ ((packed));
3781ec1e82fSSascha Hauer
3791ec1e82fSSascha Hauer
3801ec1e82fSSascha Hauer struct sdma_engine;
3811ec1e82fSSascha Hauer
3821ec1e82fSSascha Hauer /**
38376c33d27SSascha Hauer * struct sdma_desc - descriptor structor for one transfer
38424ca312dSRobin Gong * @vd: descriptor for virt dma
38524ca312dSRobin Gong * @num_bd: number of descriptors currently handling
38624ca312dSRobin Gong * @bd_phys: physical address of bd
38724ca312dSRobin Gong * @buf_tail: ID of the buffer that was processed
38824ca312dSRobin Gong * @buf_ptail: ID of the previous buffer that was processed
38924ca312dSRobin Gong * @period_len: period length, used in cyclic.
39024ca312dSRobin Gong * @chn_real_count: the real count updated from bd->mode.count
39124ca312dSRobin Gong * @chn_count: the transfer count set
39224ca312dSRobin Gong * @sdmac: sdma_channel pointer
39324ca312dSRobin Gong * @bd: pointer of allocate bd
39476c33d27SSascha Hauer */
39576c33d27SSascha Hauer struct sdma_desc {
39657b772b8SRobin Gong struct virt_dma_desc vd;
39776c33d27SSascha Hauer unsigned int num_bd;
39876c33d27SSascha Hauer dma_addr_t bd_phys;
39976c33d27SSascha Hauer unsigned int buf_tail;
40076c33d27SSascha Hauer unsigned int buf_ptail;
40176c33d27SSascha Hauer unsigned int period_len;
40276c33d27SSascha Hauer unsigned int chn_real_count;
40376c33d27SSascha Hauer unsigned int chn_count;
40476c33d27SSascha Hauer struct sdma_channel *sdmac;
40576c33d27SSascha Hauer struct sdma_buffer_descriptor *bd;
40676c33d27SSascha Hauer };
40776c33d27SSascha Hauer
40876c33d27SSascha Hauer /**
4091ec1e82fSSascha Hauer * struct sdma_channel - housekeeping for a SDMA channel
4101ec1e82fSSascha Hauer *
41124ca312dSRobin Gong * @vc: virt_dma base structure
41224ca312dSRobin Gong * @desc: sdma description including vd and other special member
41324ca312dSRobin Gong * @sdma: pointer to the SDMA engine for this channel
41424ca312dSRobin Gong * @channel: the channel number, matches dmaengine chan_id + 1
41524ca312dSRobin Gong * @direction: transfer type. Needed for setting SDMA script
416d0c4a149SLee Jones * @slave_config: Slave configuration
41724ca312dSRobin Gong * @peripheral_type: Peripheral type. Needed for setting SDMA script
41824ca312dSRobin Gong * @event_id0: aka dma request line
41924ca312dSRobin Gong * @event_id1: for channels that use 2 events
42024ca312dSRobin Gong * @word_size: peripheral access size
42124ca312dSRobin Gong * @pc_from_device: script address for those device_2_memory
42224ca312dSRobin Gong * @pc_to_device: script address for those memory_2_device
42324ca312dSRobin Gong * @device_to_device: script address for those device_2_device
4240f06c027SRobin Gong * @pc_to_pc: script address for those memory_2_memory
42524ca312dSRobin Gong * @flags: loop mode or not
42624ca312dSRobin Gong * @per_address: peripheral source or destination address in common case
42724ca312dSRobin Gong * destination address in p_2_p case
42824ca312dSRobin Gong * @per_address2: peripheral source address in p_2_p case
42924ca312dSRobin Gong * @event_mask: event mask used in p_2_p script
43024ca312dSRobin Gong * @watermark_level: value for gReg[7], some script will extend it from
43124ca312dSRobin Gong * basic watermark such as p_2_p
43224ca312dSRobin Gong * @shp_addr: value for gReg[6]
43324ca312dSRobin Gong * @per_addr: value for gReg[2]
43424ca312dSRobin Gong * @status: status of dma channel
43524ca312dSRobin Gong * @data: specific sdma interface structure
436d0c4a149SLee Jones * @terminate_worker: used to call back into terminate work function
43701eafd4bSShengjiu Wang * @terminated: terminated list
43801eafd4bSShengjiu Wang * @is_ram_script: flag for script in ram
43901eafd4bSShengjiu Wang * @n_fifos_src: number of source device fifos
44001eafd4bSShengjiu Wang * @n_fifos_dst: number of destination device fifos
44101eafd4bSShengjiu Wang * @sw_done: software done flag
442e0c7ea83SShengjiu Wang * @stride_fifos_src: stride for source device FIFOs
443e0c7ea83SShengjiu Wang * @stride_fifos_dst: stride for destination device FIFOs
444e0c7ea83SShengjiu Wang * @words_per_fifo: copy number of words one time for one FIFO
4451ec1e82fSSascha Hauer */
4461ec1e82fSSascha Hauer struct sdma_channel {
44757b772b8SRobin Gong struct virt_dma_chan vc;
44876c33d27SSascha Hauer struct sdma_desc *desc;
4491ec1e82fSSascha Hauer struct sdma_engine *sdma;
4501ec1e82fSSascha Hauer unsigned int channel;
451db8196dfSVinod Koul enum dma_transfer_direction direction;
452107d0644SVinod Koul struct dma_slave_config slave_config;
4531ec1e82fSSascha Hauer enum sdma_peripheral_type peripheral_type;
4541ec1e82fSSascha Hauer unsigned int event_id0;
4551ec1e82fSSascha Hauer unsigned int event_id1;
4561ec1e82fSSascha Hauer enum dma_slave_buswidth word_size;
4571ec1e82fSSascha Hauer unsigned int pc_from_device, pc_to_device;
4588391ecf4SShengjiu Wang unsigned int device_to_device;
4590f06c027SRobin Gong unsigned int pc_to_pc;
4601ec1e82fSSascha Hauer unsigned long flags;
4618391ecf4SShengjiu Wang dma_addr_t per_address, per_address2;
4620bbc1413SRichard Zhao unsigned long event_mask[2];
4630bbc1413SRichard Zhao unsigned long watermark_level;
4641ec1e82fSSascha Hauer u32 shp_addr, per_addr;
4651ec1e82fSSascha Hauer enum dma_status status;
4660b351865SNicolin Chen struct imx_dma_data data;
467b8603d2aSLucas Stach struct work_struct terminate_worker;
4684e2b10beSRobin Gong struct list_head terminated;
469e8fafa50SRobin Gong bool is_ram_script;
470824a0a02SSascha Hauer unsigned int n_fifos_src;
471824a0a02SSascha Hauer unsigned int n_fifos_dst;
472e0c7ea83SShengjiu Wang unsigned int stride_fifos_src;
473e0c7ea83SShengjiu Wang unsigned int stride_fifos_dst;
474e0c7ea83SShengjiu Wang unsigned int words_per_fifo;
475824a0a02SSascha Hauer bool sw_done;
4761ec1e82fSSascha Hauer };
4771ec1e82fSSascha Hauer
4780bbc1413SRichard Zhao #define IMX_DMA_SG_LOOP BIT(0)
4791ec1e82fSSascha Hauer
4801ec1e82fSSascha Hauer #define MAX_DMA_CHANNELS 32
4811ec1e82fSSascha Hauer #define MXC_SDMA_DEFAULT_PRIORITY 1
4821ec1e82fSSascha Hauer #define MXC_SDMA_MIN_PRIORITY 1
4831ec1e82fSSascha Hauer #define MXC_SDMA_MAX_PRIORITY 7
4841ec1e82fSSascha Hauer
4851ec1e82fSSascha Hauer #define SDMA_FIRMWARE_MAGIC 0x414d4453
4861ec1e82fSSascha Hauer
4871ec1e82fSSascha Hauer /**
4881ec1e82fSSascha Hauer * struct sdma_firmware_header - Layout of the firmware image
4891ec1e82fSSascha Hauer *
49024ca312dSRobin Gong * @magic: "SDMA"
49124ca312dSRobin Gong * @version_major: increased whenever layout of struct
49224ca312dSRobin Gong * sdma_script_start_addrs changes.
49324ca312dSRobin Gong * @version_minor: firmware minor version (for binary compatible changes)
49424ca312dSRobin Gong * @script_addrs_start: offset of struct sdma_script_start_addrs in this image
49524ca312dSRobin Gong * @num_script_addrs: Number of script addresses in this image
49624ca312dSRobin Gong * @ram_code_start: offset of SDMA ram image in this firmware image
49724ca312dSRobin Gong * @ram_code_size: size of SDMA ram image
4981ec1e82fSSascha Hauer */
4991ec1e82fSSascha Hauer struct sdma_firmware_header {
5001ec1e82fSSascha Hauer u32 magic;
5011ec1e82fSSascha Hauer u32 version_major;
5021ec1e82fSSascha Hauer u32 version_minor;
5031ec1e82fSSascha Hauer u32 script_addrs_start;
5041ec1e82fSSascha Hauer u32 num_script_addrs;
5051ec1e82fSSascha Hauer u32 ram_code_start;
5061ec1e82fSSascha Hauer u32 ram_code_size;
5071ec1e82fSSascha Hauer };
5081ec1e82fSSascha Hauer
50917bba72fSSascha Hauer struct sdma_driver_data {
51017bba72fSSascha Hauer int chnenbl0;
51117bba72fSSascha Hauer int num_events;
512dcfec3c0SSascha Hauer struct sdma_script_start_addrs *script_addrs;
513941acd56SAngus Ainslie (Purism) bool check_ratio;
5144852e9a2SRobin Gong /*
5154852e9a2SRobin Gong * ecspi ERR009165 fixed should be done in sdma script
5164852e9a2SRobin Gong * and it has been fixed in soc from i.mx6ul.
5174852e9a2SRobin Gong * please get more information from the below link:
5184852e9a2SRobin Gong * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
5194852e9a2SRobin Gong */
5204852e9a2SRobin Gong bool ecspi_fixed;
52162550cd7SShawn Guo };
52262550cd7SShawn Guo
5231ec1e82fSSascha Hauer struct sdma_engine {
5241ec1e82fSSascha Hauer struct device *dev;
5251ec1e82fSSascha Hauer struct sdma_channel channel[MAX_DMA_CHANNELS];
5261ec1e82fSSascha Hauer struct sdma_channel_control *channel_control;
5271ec1e82fSSascha Hauer void __iomem *regs;
5281ec1e82fSSascha Hauer struct sdma_context_data *context;
5291ec1e82fSSascha Hauer dma_addr_t context_phys;
5301ec1e82fSSascha Hauer struct dma_device dma_device;
5317560e3f3SSascha Hauer struct clk *clk_ipg;
5327560e3f3SSascha Hauer struct clk *clk_ahb;
5332ccaef05SRichard Zhao spinlock_t channel_0_lock;
534cd72b846SNicolin Chen u32 script_number;
5351ec1e82fSSascha Hauer struct sdma_script_start_addrs *script_addrs;
53617bba72fSSascha Hauer const struct sdma_driver_data *drvdata;
5378391ecf4SShengjiu Wang u32 spba_start_addr;
5388391ecf4SShengjiu Wang u32 spba_end_addr;
5395bb9dbb5SVinod Koul unsigned int irq;
54076c33d27SSascha Hauer dma_addr_t bd0_phys;
54176c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0;
54225aaa75dSAngus Ainslie (Purism) /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
54325aaa75dSAngus Ainslie (Purism) bool clk_ratio;
544e8fafa50SRobin Gong bool fw_loaded;
545802ef223SNicolin Chen struct gen_pool *iram_pool;
54617bba72fSSascha Hauer };
54717bba72fSSascha Hauer
548107d0644SVinod Koul static int sdma_config_write(struct dma_chan *chan,
549107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg,
550107d0644SVinod Koul enum dma_transfer_direction direction);
551107d0644SVinod Koul
552e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx31 = {
55317bba72fSSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX31,
55417bba72fSSascha Hauer .num_events = 32,
55517bba72fSSascha Hauer };
55617bba72fSSascha Hauer
557dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx25 = {
558dcfec3c0SSascha Hauer .ap_2_ap_addr = 729,
559dcfec3c0SSascha Hauer .uart_2_mcu_addr = 904,
560dcfec3c0SSascha Hauer .per_2_app_addr = 1255,
561dcfec3c0SSascha Hauer .mcu_2_app_addr = 834,
562dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1120,
563dcfec3c0SSascha Hauer .per_2_shp_addr = 1329,
564dcfec3c0SSascha Hauer .mcu_2_shp_addr = 1048,
565dcfec3c0SSascha Hauer .ata_2_mcu_addr = 1560,
566dcfec3c0SSascha Hauer .mcu_2_ata_addr = 1479,
567dcfec3c0SSascha Hauer .app_2_per_addr = 1189,
568dcfec3c0SSascha Hauer .app_2_mcu_addr = 770,
569dcfec3c0SSascha Hauer .shp_2_per_addr = 1407,
570dcfec3c0SSascha Hauer .shp_2_mcu_addr = 979,
571dcfec3c0SSascha Hauer };
572dcfec3c0SSascha Hauer
573e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx25 = {
574dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35,
575dcfec3c0SSascha Hauer .num_events = 48,
576dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx25,
577dcfec3c0SSascha Hauer };
578dcfec3c0SSascha Hauer
579e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx35 = {
58017bba72fSSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35,
58117bba72fSSascha Hauer .num_events = 48,
5821ec1e82fSSascha Hauer };
5831ec1e82fSSascha Hauer
584dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx51 = {
585dcfec3c0SSascha Hauer .ap_2_ap_addr = 642,
586dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817,
587dcfec3c0SSascha Hauer .mcu_2_app_addr = 747,
588dcfec3c0SSascha Hauer .mcu_2_shp_addr = 961,
589dcfec3c0SSascha Hauer .ata_2_mcu_addr = 1473,
590dcfec3c0SSascha Hauer .mcu_2_ata_addr = 1392,
591dcfec3c0SSascha Hauer .app_2_per_addr = 1033,
592dcfec3c0SSascha Hauer .app_2_mcu_addr = 683,
593dcfec3c0SSascha Hauer .shp_2_per_addr = 1251,
594dcfec3c0SSascha Hauer .shp_2_mcu_addr = 892,
595dcfec3c0SSascha Hauer };
596dcfec3c0SSascha Hauer
597e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx51 = {
598dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35,
599dcfec3c0SSascha Hauer .num_events = 48,
600dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx51,
601dcfec3c0SSascha Hauer };
602dcfec3c0SSascha Hauer
603dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx53 = {
604dcfec3c0SSascha Hauer .ap_2_ap_addr = 642,
605dcfec3c0SSascha Hauer .app_2_mcu_addr = 683,
606dcfec3c0SSascha Hauer .mcu_2_app_addr = 747,
607dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817,
608dcfec3c0SSascha Hauer .shp_2_mcu_addr = 891,
609dcfec3c0SSascha Hauer .mcu_2_shp_addr = 960,
610dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1032,
611dcfec3c0SSascha Hauer .spdif_2_mcu_addr = 1100,
612dcfec3c0SSascha Hauer .mcu_2_spdif_addr = 1134,
613dcfec3c0SSascha Hauer .firi_2_mcu_addr = 1193,
614dcfec3c0SSascha Hauer .mcu_2_firi_addr = 1290,
615dcfec3c0SSascha Hauer };
616dcfec3c0SSascha Hauer
617e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx53 = {
618dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35,
619dcfec3c0SSascha Hauer .num_events = 48,
620dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx53,
621dcfec3c0SSascha Hauer };
622dcfec3c0SSascha Hauer
623dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx6q = {
624dcfec3c0SSascha Hauer .ap_2_ap_addr = 642,
625dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817,
626dcfec3c0SSascha Hauer .mcu_2_app_addr = 747,
627dcfec3c0SSascha Hauer .per_2_per_addr = 6331,
628dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1032,
629dcfec3c0SSascha Hauer .mcu_2_shp_addr = 960,
630dcfec3c0SSascha Hauer .app_2_mcu_addr = 683,
631dcfec3c0SSascha Hauer .shp_2_mcu_addr = 891,
632dcfec3c0SSascha Hauer .spdif_2_mcu_addr = 1100,
633dcfec3c0SSascha Hauer .mcu_2_spdif_addr = 1134,
634dcfec3c0SSascha Hauer };
635dcfec3c0SSascha Hauer
636e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx6q = {
637dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35,
638dcfec3c0SSascha Hauer .num_events = 48,
639dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx6q,
640dcfec3c0SSascha Hauer };
641dcfec3c0SSascha Hauer
6424852e9a2SRobin Gong static struct sdma_driver_data sdma_imx6ul = {
6434852e9a2SRobin Gong .chnenbl0 = SDMA_CHNENBL0_IMX35,
6444852e9a2SRobin Gong .num_events = 48,
6454852e9a2SRobin Gong .script_addrs = &sdma_script_imx6q,
6464852e9a2SRobin Gong .ecspi_fixed = true,
6474852e9a2SRobin Gong };
6484852e9a2SRobin Gong
649b7d2648aSFabio Estevam static struct sdma_script_start_addrs sdma_script_imx7d = {
650b7d2648aSFabio Estevam .ap_2_ap_addr = 644,
651b7d2648aSFabio Estevam .uart_2_mcu_addr = 819,
652b7d2648aSFabio Estevam .mcu_2_app_addr = 749,
653b7d2648aSFabio Estevam .uartsh_2_mcu_addr = 1034,
654b7d2648aSFabio Estevam .mcu_2_shp_addr = 962,
655b7d2648aSFabio Estevam .app_2_mcu_addr = 685,
656b7d2648aSFabio Estevam .shp_2_mcu_addr = 893,
657b7d2648aSFabio Estevam .spdif_2_mcu_addr = 1102,
658b7d2648aSFabio Estevam .mcu_2_spdif_addr = 1136,
659b7d2648aSFabio Estevam };
660b7d2648aSFabio Estevam
661b7d2648aSFabio Estevam static struct sdma_driver_data sdma_imx7d = {
662b7d2648aSFabio Estevam .chnenbl0 = SDMA_CHNENBL0_IMX35,
663b7d2648aSFabio Estevam .num_events = 48,
664b7d2648aSFabio Estevam .script_addrs = &sdma_script_imx7d,
665b7d2648aSFabio Estevam };
666b7d2648aSFabio Estevam
667941acd56SAngus Ainslie (Purism) static struct sdma_driver_data sdma_imx8mq = {
668941acd56SAngus Ainslie (Purism) .chnenbl0 = SDMA_CHNENBL0_IMX35,
669941acd56SAngus Ainslie (Purism) .num_events = 48,
670941acd56SAngus Ainslie (Purism) .script_addrs = &sdma_script_imx7d,
671941acd56SAngus Ainslie (Purism) .check_ratio = 1,
672941acd56SAngus Ainslie (Purism) };
673941acd56SAngus Ainslie (Purism)
674580975d7SShawn Guo static const struct of_device_id sdma_dt_ids[] = {
675dcfec3c0SSascha Hauer { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
676dcfec3c0SSascha Hauer { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
677dcfec3c0SSascha Hauer { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
67817bba72fSSascha Hauer { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
679dcfec3c0SSascha Hauer { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
68063edea16SMarkus Pargmann { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
681b7d2648aSFabio Estevam { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
6824852e9a2SRobin Gong { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, },
683941acd56SAngus Ainslie (Purism) { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
684580975d7SShawn Guo { /* sentinel */ }
685580975d7SShawn Guo };
686580975d7SShawn Guo MODULE_DEVICE_TABLE(of, sdma_dt_ids);
687580975d7SShawn Guo
6880bbc1413SRichard Zhao #define SDMA_H_CONFIG_DSPDMA BIT(12) /* indicates if the DSPDMA is used */
6890bbc1413SRichard Zhao #define SDMA_H_CONFIG_RTD_PINS BIT(11) /* indicates if Real-Time Debug pins are enabled */
6900bbc1413SRichard Zhao #define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */
6911ec1e82fSSascha Hauer #define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/
6921ec1e82fSSascha Hauer
chnenbl_ofs(struct sdma_engine * sdma,unsigned int event)6931ec1e82fSSascha Hauer static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
6941ec1e82fSSascha Hauer {
69517bba72fSSascha Hauer u32 chnenbl0 = sdma->drvdata->chnenbl0;
6961ec1e82fSSascha Hauer return chnenbl0 + event * 4;
6971ec1e82fSSascha Hauer }
6981ec1e82fSSascha Hauer
sdma_config_ownership(struct sdma_channel * sdmac,bool event_override,bool mcu_override,bool dsp_override)6991ec1e82fSSascha Hauer static int sdma_config_ownership(struct sdma_channel *sdmac,
7001ec1e82fSSascha Hauer bool event_override, bool mcu_override, bool dsp_override)
7011ec1e82fSSascha Hauer {
7021ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
7031ec1e82fSSascha Hauer int channel = sdmac->channel;
7040bbc1413SRichard Zhao unsigned long evt, mcu, dsp;
7051ec1e82fSSascha Hauer
7061ec1e82fSSascha Hauer if (event_override && mcu_override && dsp_override)
7071ec1e82fSSascha Hauer return -EINVAL;
7081ec1e82fSSascha Hauer
709c4b56857SRichard Zhao evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
710c4b56857SRichard Zhao mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
711c4b56857SRichard Zhao dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
7121ec1e82fSSascha Hauer
7131ec1e82fSSascha Hauer if (dsp_override)
7140bbc1413SRichard Zhao __clear_bit(channel, &dsp);
7151ec1e82fSSascha Hauer else
7160bbc1413SRichard Zhao __set_bit(channel, &dsp);
7171ec1e82fSSascha Hauer
7181ec1e82fSSascha Hauer if (event_override)
7190bbc1413SRichard Zhao __clear_bit(channel, &evt);
7201ec1e82fSSascha Hauer else
7210bbc1413SRichard Zhao __set_bit(channel, &evt);
7221ec1e82fSSascha Hauer
7231ec1e82fSSascha Hauer if (mcu_override)
7240bbc1413SRichard Zhao __clear_bit(channel, &mcu);
7251ec1e82fSSascha Hauer else
7260bbc1413SRichard Zhao __set_bit(channel, &mcu);
7271ec1e82fSSascha Hauer
728c4b56857SRichard Zhao writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
729c4b56857SRichard Zhao writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
730c4b56857SRichard Zhao writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
7311ec1e82fSSascha Hauer
7321ec1e82fSSascha Hauer return 0;
7331ec1e82fSSascha Hauer }
7341ec1e82fSSascha Hauer
is_sdma_channel_enabled(struct sdma_engine * sdma,int channel)7355b215c28STomasz Moń static int is_sdma_channel_enabled(struct sdma_engine *sdma, int channel)
7365b215c28STomasz Moń {
7375b215c28STomasz Moń return !!(readl(sdma->regs + SDMA_H_STATSTOP) & BIT(channel));
7385b215c28STomasz Moń }
7395b215c28STomasz Moń
sdma_enable_channel(struct sdma_engine * sdma,int channel)740b9a59166SRichard Zhao static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
741b9a59166SRichard Zhao {
7420bbc1413SRichard Zhao writel(BIT(channel), sdma->regs + SDMA_H_START);
743b9a59166SRichard Zhao }
744b9a59166SRichard Zhao
7451ec1e82fSSascha Hauer /*
7462ccaef05SRichard Zhao * sdma_run_channel0 - run a channel and wait till it's done
7471ec1e82fSSascha Hauer */
sdma_run_channel0(struct sdma_engine * sdma)7482ccaef05SRichard Zhao static int sdma_run_channel0(struct sdma_engine *sdma)
7491ec1e82fSSascha Hauer {
7501ec1e82fSSascha Hauer int ret;
7511d069bfaSMichael Olbrich u32 reg;
7521ec1e82fSSascha Hauer
7532ccaef05SRichard Zhao sdma_enable_channel(sdma, 0);
7541ec1e82fSSascha Hauer
7551d069bfaSMichael Olbrich ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
7561d069bfaSMichael Olbrich reg, !(reg & 1), 1, 500);
7571d069bfaSMichael Olbrich if (ret)
7582ccaef05SRichard Zhao dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
7591ec1e82fSSascha Hauer
760855832e4SRobin Gong /* Set bits of CONFIG register with dynamic context switching */
76125aaa75dSAngus Ainslie (Purism) reg = readl(sdma->regs + SDMA_H_CONFIG);
76225aaa75dSAngus Ainslie (Purism) if ((reg & SDMA_H_CONFIG_CSM) == 0) {
76325aaa75dSAngus Ainslie (Purism) reg |= SDMA_H_CONFIG_CSM;
76425aaa75dSAngus Ainslie (Purism) writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
76525aaa75dSAngus Ainslie (Purism) }
766855832e4SRobin Gong
7671d069bfaSMichael Olbrich return ret;
7681ec1e82fSSascha Hauer }
7691ec1e82fSSascha Hauer
sdma_load_script(struct sdma_engine * sdma,void * buf,int size,u32 address)7701ec1e82fSSascha Hauer static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
7711ec1e82fSSascha Hauer u32 address)
7721ec1e82fSSascha Hauer {
77376c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0 = sdma->bd0;
7741ec1e82fSSascha Hauer void *buf_virt;
7751ec1e82fSSascha Hauer dma_addr_t buf_phys;
7761ec1e82fSSascha Hauer int ret;
7772ccaef05SRichard Zhao unsigned long flags;
77873eab978SSascha Hauer
779ceaf5226SAndy Duan buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
780ef6c1dadSFlavio Suligoi if (!buf_virt)
7812ccaef05SRichard Zhao return -ENOMEM;
7821ec1e82fSSascha Hauer
7832ccaef05SRichard Zhao spin_lock_irqsave(&sdma->channel_0_lock, flags);
7842ccaef05SRichard Zhao
7851ec1e82fSSascha Hauer bd0->mode.command = C0_SETPM;
7863f93a4f2SRobin Gong bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
7871ec1e82fSSascha Hauer bd0->mode.count = size / 2;
7881ec1e82fSSascha Hauer bd0->buffer_addr = buf_phys;
7891ec1e82fSSascha Hauer bd0->ext_buffer_addr = address;
7901ec1e82fSSascha Hauer
7911ec1e82fSSascha Hauer memcpy(buf_virt, buf, size);
7921ec1e82fSSascha Hauer
7932ccaef05SRichard Zhao ret = sdma_run_channel0(sdma);
7942ccaef05SRichard Zhao
7952ccaef05SRichard Zhao spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
7961ec1e82fSSascha Hauer
797ceaf5226SAndy Duan dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);
7981ec1e82fSSascha Hauer
7991ec1e82fSSascha Hauer return ret;
8001ec1e82fSSascha Hauer }
8011ec1e82fSSascha Hauer
sdma_event_enable(struct sdma_channel * sdmac,unsigned int event)8021ec1e82fSSascha Hauer static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
8031ec1e82fSSascha Hauer {
8041ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
8051ec1e82fSSascha Hauer int channel = sdmac->channel;
8060bbc1413SRichard Zhao unsigned long val;
8071ec1e82fSSascha Hauer u32 chnenbl = chnenbl_ofs(sdma, event);
8081ec1e82fSSascha Hauer
809c4b56857SRichard Zhao val = readl_relaxed(sdma->regs + chnenbl);
8100bbc1413SRichard Zhao __set_bit(channel, &val);
811c4b56857SRichard Zhao writel_relaxed(val, sdma->regs + chnenbl);
812824a0a02SSascha Hauer
813824a0a02SSascha Hauer /* Set SDMA_DONEx_CONFIG is sw_done enabled */
814824a0a02SSascha Hauer if (sdmac->sw_done) {
815824a0a02SSascha Hauer val = readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG);
816824a0a02SSascha Hauer val |= SDMA_DONE0_CONFIG_DONE_SEL;
817824a0a02SSascha Hauer val &= ~SDMA_DONE0_CONFIG_DONE_DIS;
818824a0a02SSascha Hauer writel_relaxed(val, sdma->regs + SDMA_DONE0_CONFIG);
819824a0a02SSascha Hauer }
8201ec1e82fSSascha Hauer }
8211ec1e82fSSascha Hauer
sdma_event_disable(struct sdma_channel * sdmac,unsigned int event)8221ec1e82fSSascha Hauer static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
8231ec1e82fSSascha Hauer {
8241ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
8251ec1e82fSSascha Hauer int channel = sdmac->channel;
8261ec1e82fSSascha Hauer u32 chnenbl = chnenbl_ofs(sdma, event);
8270bbc1413SRichard Zhao unsigned long val;
8281ec1e82fSSascha Hauer
829c4b56857SRichard Zhao val = readl_relaxed(sdma->regs + chnenbl);
8300bbc1413SRichard Zhao __clear_bit(channel, &val);
831c4b56857SRichard Zhao writel_relaxed(val, sdma->regs + chnenbl);
8321ec1e82fSSascha Hauer }
8331ec1e82fSSascha Hauer
to_sdma_desc(struct dma_async_tx_descriptor * t)83457b772b8SRobin Gong static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
83557b772b8SRobin Gong {
83657b772b8SRobin Gong return container_of(t, struct sdma_desc, vd.tx);
83757b772b8SRobin Gong }
83857b772b8SRobin Gong
sdma_start_desc(struct sdma_channel * sdmac)83957b772b8SRobin Gong static void sdma_start_desc(struct sdma_channel *sdmac)
84057b772b8SRobin Gong {
84157b772b8SRobin Gong struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
84257b772b8SRobin Gong struct sdma_desc *desc;
84357b772b8SRobin Gong struct sdma_engine *sdma = sdmac->sdma;
84457b772b8SRobin Gong int channel = sdmac->channel;
84557b772b8SRobin Gong
84657b772b8SRobin Gong if (!vd) {
84757b772b8SRobin Gong sdmac->desc = NULL;
84857b772b8SRobin Gong return;
84957b772b8SRobin Gong }
85057b772b8SRobin Gong sdmac->desc = desc = to_sdma_desc(&vd->tx);
85102939cd1SSascha Hauer
85257b772b8SRobin Gong list_del(&vd->node);
85357b772b8SRobin Gong
85457b772b8SRobin Gong sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
85557b772b8SRobin Gong sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
85657b772b8SRobin Gong sdma_enable_channel(sdma, sdmac->channel);
85757b772b8SRobin Gong }
85857b772b8SRobin Gong
sdma_update_channel_loop(struct sdma_channel * sdmac)859d1a792f3SRussell King - ARM Linux static void sdma_update_channel_loop(struct sdma_channel *sdmac)
860d1a792f3SRussell King - ARM Linux {
8611ec1e82fSSascha Hauer struct sdma_buffer_descriptor *bd;
8625881826dSNandor Han int error = 0;
8635881826dSNandor Han enum dma_status old_status = sdmac->status;
8641ec1e82fSSascha Hauer
8651ec1e82fSSascha Hauer /*
8661ec1e82fSSascha Hauer * loop mode. Iterate over descriptors, re-setup them and
8671ec1e82fSSascha Hauer * call callback function.
8681ec1e82fSSascha Hauer */
86957b772b8SRobin Gong while (sdmac->desc) {
87076c33d27SSascha Hauer struct sdma_desc *desc = sdmac->desc;
87176c33d27SSascha Hauer
87276c33d27SSascha Hauer bd = &desc->bd[desc->buf_tail];
8731ec1e82fSSascha Hauer
8741ec1e82fSSascha Hauer if (bd->mode.status & BD_DONE)
8751ec1e82fSSascha Hauer break;
8761ec1e82fSSascha Hauer
8775881826dSNandor Han if (bd->mode.status & BD_RROR) {
8785881826dSNandor Han bd->mode.status &= ~BD_RROR;
8791ec1e82fSSascha Hauer sdmac->status = DMA_ERROR;
8805881826dSNandor Han error = -EIO;
8815881826dSNandor Han }
8821ec1e82fSSascha Hauer
8835881826dSNandor Han /*
8845881826dSNandor Han * We use bd->mode.count to calculate the residue, since contains
8855881826dSNandor Han * the number of bytes present in the current buffer descriptor.
8865881826dSNandor Han */
8875881826dSNandor Han
88876c33d27SSascha Hauer desc->chn_real_count = bd->mode.count;
88976c33d27SSascha Hauer bd->mode.count = desc->period_len;
89076c33d27SSascha Hauer desc->buf_ptail = desc->buf_tail;
89176c33d27SSascha Hauer desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
89215f30f51SNandor Han
89315f30f51SNandor Han /*
89415f30f51SNandor Han * The callback is called from the interrupt context in order
89515f30f51SNandor Han * to reduce latency and to avoid the risk of altering the
89615f30f51SNandor Han * SDMA transaction status by the time the client tasklet is
89715f30f51SNandor Han * executed.
89815f30f51SNandor Han */
89957b772b8SRobin Gong spin_unlock(&sdmac->vc.lock);
90057b772b8SRobin Gong dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
90157b772b8SRobin Gong spin_lock(&sdmac->vc.lock);
90215f30f51SNandor Han
903177360e0STomasz Moń /* Assign buffer ownership to SDMA */
904177360e0STomasz Moń bd->mode.status |= BD_DONE;
905177360e0STomasz Moń
9065881826dSNandor Han if (error)
9075881826dSNandor Han sdmac->status = old_status;
9081ec1e82fSSascha Hauer }
9095b215c28STomasz Moń
9105b215c28STomasz Moń /*
9115b215c28STomasz Moń * SDMA stops cyclic channel when DMA request triggers a channel and no SDMA
9125b215c28STomasz Moń * owned buffer is available (i.e. BD_DONE was set too late).
9135b215c28STomasz Moń */
91409f7b80fSSascha Hauer if (sdmac->desc && !is_sdma_channel_enabled(sdmac->sdma, sdmac->channel)) {
9155b215c28STomasz Moń dev_warn(sdmac->sdma->dev, "restart cyclic channel %d\n", sdmac->channel);
9165b215c28STomasz Moń sdma_enable_channel(sdmac->sdma, sdmac->channel);
9175b215c28STomasz Moń }
9181ec1e82fSSascha Hauer }
9191ec1e82fSSascha Hauer
mxc_sdma_handle_channel_normal(struct sdma_channel * data)92057b772b8SRobin Gong static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
9211ec1e82fSSascha Hauer {
92215f30f51SNandor Han struct sdma_channel *sdmac = (struct sdma_channel *) data;
9231ec1e82fSSascha Hauer struct sdma_buffer_descriptor *bd;
9241ec1e82fSSascha Hauer int i, error = 0;
9251ec1e82fSSascha Hauer
92676c33d27SSascha Hauer sdmac->desc->chn_real_count = 0;
9271ec1e82fSSascha Hauer /*
9281ec1e82fSSascha Hauer * non loop mode. Iterate over all descriptors, collect
9291ec1e82fSSascha Hauer * errors and call callback function
9301ec1e82fSSascha Hauer */
93176c33d27SSascha Hauer for (i = 0; i < sdmac->desc->num_bd; i++) {
93276c33d27SSascha Hauer bd = &sdmac->desc->bd[i];
9331ec1e82fSSascha Hauer
9341ec1e82fSSascha Hauer if (bd->mode.status & (BD_DONE | BD_RROR))
9351ec1e82fSSascha Hauer error = -EIO;
93676c33d27SSascha Hauer sdmac->desc->chn_real_count += bd->mode.count;
9371ec1e82fSSascha Hauer }
9381ec1e82fSSascha Hauer
9391ec1e82fSSascha Hauer if (error)
9401ec1e82fSSascha Hauer sdmac->status = DMA_ERROR;
9411ec1e82fSSascha Hauer else
942409bff6aSVinod Koul sdmac->status = DMA_COMPLETE;
9431ec1e82fSSascha Hauer }
9441ec1e82fSSascha Hauer
sdma_int_handler(int irq,void * dev_id)9451ec1e82fSSascha Hauer static irqreturn_t sdma_int_handler(int irq, void *dev_id)
9461ec1e82fSSascha Hauer {
9471ec1e82fSSascha Hauer struct sdma_engine *sdma = dev_id;
9480bbc1413SRichard Zhao unsigned long stat;
9491ec1e82fSSascha Hauer
950c4b56857SRichard Zhao stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
951c4b56857SRichard Zhao writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
9521d069bfaSMichael Olbrich /* channel 0 is special and not handled here, see run_channel0() */
9531d069bfaSMichael Olbrich stat &= ~1;
9541ec1e82fSSascha Hauer
9551ec1e82fSSascha Hauer while (stat) {
9561ec1e82fSSascha Hauer int channel = fls(stat) - 1;
9571ec1e82fSSascha Hauer struct sdma_channel *sdmac = &sdma->channel[channel];
95857b772b8SRobin Gong struct sdma_desc *desc;
9591ec1e82fSSascha Hauer
96057b772b8SRobin Gong spin_lock(&sdmac->vc.lock);
96157b772b8SRobin Gong desc = sdmac->desc;
96257b772b8SRobin Gong if (desc) {
96357b772b8SRobin Gong if (sdmac->flags & IMX_DMA_SG_LOOP) {
964e873d432SJoy Zou if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
965d1a792f3SRussell King - ARM Linux sdma_update_channel_loop(sdmac);
966e873d432SJoy Zou else
967e873d432SJoy Zou vchan_cyclic_callback(&desc->vd);
96857b772b8SRobin Gong } else {
96957b772b8SRobin Gong mxc_sdma_handle_channel_normal(sdmac);
97057b772b8SRobin Gong vchan_cookie_complete(&desc->vd);
97157b772b8SRobin Gong sdma_start_desc(sdmac);
97257b772b8SRobin Gong }
97357b772b8SRobin Gong }
9741ec1e82fSSascha Hauer
97557b772b8SRobin Gong spin_unlock(&sdmac->vc.lock);
9760bbc1413SRichard Zhao __clear_bit(channel, &stat);
9771ec1e82fSSascha Hauer }
9781ec1e82fSSascha Hauer
9791ec1e82fSSascha Hauer return IRQ_HANDLED;
9801ec1e82fSSascha Hauer }
9811ec1e82fSSascha Hauer
9821ec1e82fSSascha Hauer /*
9831ec1e82fSSascha Hauer * sets the pc of SDMA script according to the peripheral type
9841ec1e82fSSascha Hauer */
sdma_get_pc(struct sdma_channel * sdmac,enum sdma_peripheral_type peripheral_type)985625d8936SSascha Hauer static int sdma_get_pc(struct sdma_channel *sdmac,
9861ec1e82fSSascha Hauer enum sdma_peripheral_type peripheral_type)
9871ec1e82fSSascha Hauer {
9881ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
9891ec1e82fSSascha Hauer int per_2_emi = 0, emi_2_per = 0;
9901ec1e82fSSascha Hauer /*
9911ec1e82fSSascha Hauer * These are needed once we start to support transfers between
9921ec1e82fSSascha Hauer * two peripherals or memory-to-memory transfers
9931ec1e82fSSascha Hauer */
9940f06c027SRobin Gong int per_2_per = 0, emi_2_emi = 0;
9951ec1e82fSSascha Hauer
9961ec1e82fSSascha Hauer sdmac->pc_from_device = 0;
9971ec1e82fSSascha Hauer sdmac->pc_to_device = 0;
9988391ecf4SShengjiu Wang sdmac->device_to_device = 0;
9990f06c027SRobin Gong sdmac->pc_to_pc = 0;
1000e8fafa50SRobin Gong sdmac->is_ram_script = false;
10011ec1e82fSSascha Hauer
10021ec1e82fSSascha Hauer switch (peripheral_type) {
10031ec1e82fSSascha Hauer case IMX_DMATYPE_MEMORY:
10040f06c027SRobin Gong emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
10051ec1e82fSSascha Hauer break;
10061ec1e82fSSascha Hauer case IMX_DMATYPE_DSP:
10071ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->bp_2_ap_addr;
10081ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->ap_2_bp_addr;
10091ec1e82fSSascha Hauer break;
10101ec1e82fSSascha Hauer case IMX_DMATYPE_FIRI:
10111ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
10121ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
10131ec1e82fSSascha Hauer break;
10141ec1e82fSSascha Hauer case IMX_DMATYPE_UART:
10151ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
10161ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_app_addr;
10171ec1e82fSSascha Hauer break;
10181ec1e82fSSascha Hauer case IMX_DMATYPE_UART_SP:
10191ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
10201ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
10211ec1e82fSSascha Hauer break;
10221ec1e82fSSascha Hauer case IMX_DMATYPE_ATA:
10231ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
10241ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
10251ec1e82fSSascha Hauer break;
10261ec1e82fSSascha Hauer case IMX_DMATYPE_CSPI:
1027a4965888SRobin Gong per_2_emi = sdma->script_addrs->app_2_mcu_addr;
10284852e9a2SRobin Gong
10294852e9a2SRobin Gong /* Use rom script mcu_2_app if ERR009165 fixed */
10304852e9a2SRobin Gong if (sdmac->sdma->drvdata->ecspi_fixed) {
10314852e9a2SRobin Gong emi_2_per = sdma->script_addrs->mcu_2_app_addr;
10324852e9a2SRobin Gong } else {
1033a4965888SRobin Gong emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr;
1034a4965888SRobin Gong sdmac->is_ram_script = true;
10354852e9a2SRobin Gong }
10364852e9a2SRobin Gong
1037a4965888SRobin Gong break;
10381ec1e82fSSascha Hauer case IMX_DMATYPE_EXT:
10391ec1e82fSSascha Hauer case IMX_DMATYPE_SSI:
104029aebfdeSNicolin Chen case IMX_DMATYPE_SAI:
10411ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->app_2_mcu_addr;
10421ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_app_addr;
10431ec1e82fSSascha Hauer break;
10441a895578SNicolin Chen case IMX_DMATYPE_SSI_DUAL:
10451a895578SNicolin Chen per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
10461a895578SNicolin Chen emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
1047e8fafa50SRobin Gong sdmac->is_ram_script = true;
10481a895578SNicolin Chen break;
10491ec1e82fSSascha Hauer case IMX_DMATYPE_SSI_SP:
10501ec1e82fSSascha Hauer case IMX_DMATYPE_MMC:
10511ec1e82fSSascha Hauer case IMX_DMATYPE_SDHC:
10521ec1e82fSSascha Hauer case IMX_DMATYPE_CSPI_SP:
10531ec1e82fSSascha Hauer case IMX_DMATYPE_ESAI:
10541ec1e82fSSascha Hauer case IMX_DMATYPE_MSHC_SP:
10551ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
10561ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
10571ec1e82fSSascha Hauer break;
10581ec1e82fSSascha Hauer case IMX_DMATYPE_ASRC:
10591ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
10601ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
10611ec1e82fSSascha Hauer per_2_per = sdma->script_addrs->per_2_per_addr;
1062e8fafa50SRobin Gong sdmac->is_ram_script = true;
10631ec1e82fSSascha Hauer break;
1064f892afb0SNicolin Chen case IMX_DMATYPE_ASRC_SP:
1065f892afb0SNicolin Chen per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
1066f892afb0SNicolin Chen emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1067f892afb0SNicolin Chen per_2_per = sdma->script_addrs->per_2_per_addr;
1068f892afb0SNicolin Chen break;
10691ec1e82fSSascha Hauer case IMX_DMATYPE_MSHC:
10701ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
10711ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
10721ec1e82fSSascha Hauer break;
10731ec1e82fSSascha Hauer case IMX_DMATYPE_CCM:
10741ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
10751ec1e82fSSascha Hauer break;
10761ec1e82fSSascha Hauer case IMX_DMATYPE_SPDIF:
10771ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
10781ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
10791ec1e82fSSascha Hauer break;
10801ec1e82fSSascha Hauer case IMX_DMATYPE_IPU_MEMORY:
10811ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
10821ec1e82fSSascha Hauer break;
1083824a0a02SSascha Hauer case IMX_DMATYPE_MULTI_SAI:
1084824a0a02SSascha Hauer per_2_emi = sdma->script_addrs->sai_2_mcu_addr;
1085824a0a02SSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_sai_addr;
10861ec1e82fSSascha Hauer break;
1087d850b5baSRobin Gong case IMX_DMATYPE_I2C:
1088d850b5baSRobin Gong per_2_emi = sdma->script_addrs->i2c_2_mcu_addr;
1089d850b5baSRobin Gong emi_2_per = sdma->script_addrs->mcu_2_i2c_addr;
1090d850b5baSRobin Gong sdmac->is_ram_script = true;
1091d850b5baSRobin Gong break;
1092e873d432SJoy Zou case IMX_DMATYPE_HDMI:
1093e873d432SJoy Zou emi_2_per = sdma->script_addrs->hdmi_dma_addr;
1094e873d432SJoy Zou sdmac->is_ram_script = true;
1095e873d432SJoy Zou break;
10961ec1e82fSSascha Hauer default:
1097625d8936SSascha Hauer dev_err(sdma->dev, "Unsupported transfer type %d\n",
1098625d8936SSascha Hauer peripheral_type);
1099625d8936SSascha Hauer return -EINVAL;
11001ec1e82fSSascha Hauer }
11011ec1e82fSSascha Hauer
11021ec1e82fSSascha Hauer sdmac->pc_from_device = per_2_emi;
11031ec1e82fSSascha Hauer sdmac->pc_to_device = emi_2_per;
11048391ecf4SShengjiu Wang sdmac->device_to_device = per_2_per;
11050f06c027SRobin Gong sdmac->pc_to_pc = emi_2_emi;
1106625d8936SSascha Hauer
1107625d8936SSascha Hauer return 0;
11081ec1e82fSSascha Hauer }
11091ec1e82fSSascha Hauer
sdma_load_context(struct sdma_channel * sdmac)11101ec1e82fSSascha Hauer static int sdma_load_context(struct sdma_channel *sdmac)
11111ec1e82fSSascha Hauer {
11121ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
11131ec1e82fSSascha Hauer int channel = sdmac->channel;
11141ec1e82fSSascha Hauer int load_address;
11151ec1e82fSSascha Hauer struct sdma_context_data *context = sdma->context;
111676c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0 = sdma->bd0;
11171ec1e82fSSascha Hauer int ret;
11182ccaef05SRichard Zhao unsigned long flags;
11191ec1e82fSSascha Hauer
11208391ecf4SShengjiu Wang if (sdmac->direction == DMA_DEV_TO_MEM)
11211ec1e82fSSascha Hauer load_address = sdmac->pc_from_device;
11228391ecf4SShengjiu Wang else if (sdmac->direction == DMA_DEV_TO_DEV)
11238391ecf4SShengjiu Wang load_address = sdmac->device_to_device;
11240f06c027SRobin Gong else if (sdmac->direction == DMA_MEM_TO_MEM)
11250f06c027SRobin Gong load_address = sdmac->pc_to_pc;
11268391ecf4SShengjiu Wang else
11271ec1e82fSSascha Hauer load_address = sdmac->pc_to_device;
11281ec1e82fSSascha Hauer
11291ec1e82fSSascha Hauer if (load_address < 0)
11301ec1e82fSSascha Hauer return load_address;
11311ec1e82fSSascha Hauer
11321ec1e82fSSascha Hauer dev_dbg(sdma->dev, "load_address = %d\n", load_address);
11330bbc1413SRichard Zhao dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
11341ec1e82fSSascha Hauer dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
11351ec1e82fSSascha Hauer dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
11360bbc1413SRichard Zhao dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
11370bbc1413SRichard Zhao dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
11381ec1e82fSSascha Hauer
11392ccaef05SRichard Zhao spin_lock_irqsave(&sdma->channel_0_lock, flags);
114073eab978SSascha Hauer
11411ec1e82fSSascha Hauer memset(context, 0, sizeof(*context));
11421ec1e82fSSascha Hauer context->channel_state.pc = load_address;
11431ec1e82fSSascha Hauer
11441ec1e82fSSascha Hauer /* Send by context the event mask,base address for peripheral
11451ec1e82fSSascha Hauer * and watermark level
11461ec1e82fSSascha Hauer */
1147e873d432SJoy Zou if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1148e873d432SJoy Zou context->gReg[4] = sdmac->per_addr;
1149e873d432SJoy Zou context->gReg[6] = sdmac->shp_addr;
1150e873d432SJoy Zou } else {
11510bbc1413SRichard Zhao context->gReg[0] = sdmac->event_mask[1];
11520bbc1413SRichard Zhao context->gReg[1] = sdmac->event_mask[0];
11531ec1e82fSSascha Hauer context->gReg[2] = sdmac->per_addr;
11541ec1e82fSSascha Hauer context->gReg[6] = sdmac->shp_addr;
11551ec1e82fSSascha Hauer context->gReg[7] = sdmac->watermark_level;
1156e873d432SJoy Zou }
11571ec1e82fSSascha Hauer
11581ec1e82fSSascha Hauer bd0->mode.command = C0_SETDM;
11593f93a4f2SRobin Gong bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
11601ec1e82fSSascha Hauer bd0->mode.count = sizeof(*context) / 4;
11611ec1e82fSSascha Hauer bd0->buffer_addr = sdma->context_phys;
11621ec1e82fSSascha Hauer bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
11632ccaef05SRichard Zhao ret = sdma_run_channel0(sdma);
11641ec1e82fSSascha Hauer
11652ccaef05SRichard Zhao spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
116673eab978SSascha Hauer
11671ec1e82fSSascha Hauer return ret;
11681ec1e82fSSascha Hauer }
11691ec1e82fSSascha Hauer
to_sdma_chan(struct dma_chan * chan)11707b350ab0SMaxime Ripard static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
11711ec1e82fSSascha Hauer {
117257b772b8SRobin Gong return container_of(chan, struct sdma_channel, vc.chan);
11737b350ab0SMaxime Ripard }
11747b350ab0SMaxime Ripard
sdma_disable_channel(struct dma_chan * chan)11757b350ab0SMaxime Ripard static int sdma_disable_channel(struct dma_chan *chan)
11767b350ab0SMaxime Ripard {
11777b350ab0SMaxime Ripard struct sdma_channel *sdmac = to_sdma_chan(chan);
11781ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
11791ec1e82fSSascha Hauer int channel = sdmac->channel;
11801ec1e82fSSascha Hauer
11810bbc1413SRichard Zhao writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
11821ec1e82fSSascha Hauer sdmac->status = DMA_ERROR;
11837b350ab0SMaxime Ripard
11847b350ab0SMaxime Ripard return 0;
11851ec1e82fSSascha Hauer }
sdma_channel_terminate_work(struct work_struct * work)1186b8603d2aSLucas Stach static void sdma_channel_terminate_work(struct work_struct *work)
11877f3ff14bSJiada Wang {
1188b8603d2aSLucas Stach struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
1189b8603d2aSLucas Stach terminate_worker);
11907f3ff14bSJiada Wang /*
11917f3ff14bSJiada Wang * According to NXP R&D team a delay of one BD SDMA cost time
11927f3ff14bSJiada Wang * (maximum is 1ms) should be added after disable of the channel
11937f3ff14bSJiada Wang * bit, to ensure SDMA core has really been stopped after SDMA
11947f3ff14bSJiada Wang * clients call .device_terminate_all.
11957f3ff14bSJiada Wang */
1196b8603d2aSLucas Stach usleep_range(1000, 2000);
1197b8603d2aSLucas Stach
11984e2b10beSRobin Gong vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated);
1199b8603d2aSLucas Stach }
1200b8603d2aSLucas Stach
sdma_terminate_all(struct dma_chan * chan)1201a80f2787SSascha Hauer static int sdma_terminate_all(struct dma_chan *chan)
1202b8603d2aSLucas Stach {
1203b8603d2aSLucas Stach struct sdma_channel *sdmac = to_sdma_chan(chan);
120402939cd1SSascha Hauer unsigned long flags;
120502939cd1SSascha Hauer
120602939cd1SSascha Hauer spin_lock_irqsave(&sdmac->vc.lock, flags);
1207b8603d2aSLucas Stach
1208b8603d2aSLucas Stach sdma_disable_channel(chan);
1209b8603d2aSLucas Stach
121002939cd1SSascha Hauer if (sdmac->desc) {
121102939cd1SSascha Hauer vchan_terminate_vdesc(&sdmac->desc->vd);
12124e2b10beSRobin Gong /*
12134e2b10beSRobin Gong * move out current descriptor into terminated list so that
12144e2b10beSRobin Gong * it could be free in sdma_channel_terminate_work alone
12154e2b10beSRobin Gong * later without potential involving next descriptor raised
12164e2b10beSRobin Gong * up before the last descriptor terminated.
12174e2b10beSRobin Gong */
12184e2b10beSRobin Gong vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated);
121902939cd1SSascha Hauer sdmac->desc = NULL;
1220b8603d2aSLucas Stach schedule_work(&sdmac->terminate_worker);
122102939cd1SSascha Hauer }
122202939cd1SSascha Hauer
122302939cd1SSascha Hauer spin_unlock_irqrestore(&sdmac->vc.lock, flags);
12247f3ff14bSJiada Wang
12257f3ff14bSJiada Wang return 0;
12267f3ff14bSJiada Wang }
12277f3ff14bSJiada Wang
sdma_channel_synchronize(struct dma_chan * chan)1228b8603d2aSLucas Stach static void sdma_channel_synchronize(struct dma_chan *chan)
1229b8603d2aSLucas Stach {
1230b8603d2aSLucas Stach struct sdma_channel *sdmac = to_sdma_chan(chan);
1231b8603d2aSLucas Stach
1232b8603d2aSLucas Stach vchan_synchronize(&sdmac->vc);
1233b8603d2aSLucas Stach
1234b8603d2aSLucas Stach flush_work(&sdmac->terminate_worker);
1235b8603d2aSLucas Stach }
1236b8603d2aSLucas Stach
sdma_set_watermarklevel_for_p2p(struct sdma_channel * sdmac)12378391ecf4SShengjiu Wang static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
12388391ecf4SShengjiu Wang {
12398391ecf4SShengjiu Wang struct sdma_engine *sdma = sdmac->sdma;
12408391ecf4SShengjiu Wang
12418391ecf4SShengjiu Wang int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
12428391ecf4SShengjiu Wang int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
12438391ecf4SShengjiu Wang
12448391ecf4SShengjiu Wang set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
12458391ecf4SShengjiu Wang set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
12468391ecf4SShengjiu Wang
12478391ecf4SShengjiu Wang if (sdmac->event_id0 > 31)
12488391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
12498391ecf4SShengjiu Wang
12508391ecf4SShengjiu Wang if (sdmac->event_id1 > 31)
12518391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
12528391ecf4SShengjiu Wang
12538391ecf4SShengjiu Wang /*
12548391ecf4SShengjiu Wang * If LWML(src_maxburst) > HWML(dst_maxburst), we need
12558391ecf4SShengjiu Wang * swap LWML and HWML of INFO(A.3.2.5.1), also need swap
12568391ecf4SShengjiu Wang * r0(event_mask[1]) and r1(event_mask[0]).
12578391ecf4SShengjiu Wang */
12588391ecf4SShengjiu Wang if (lwml > hwml) {
12598391ecf4SShengjiu Wang sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
12608391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_HWML);
12618391ecf4SShengjiu Wang sdmac->watermark_level |= hwml;
12628391ecf4SShengjiu Wang sdmac->watermark_level |= lwml << 16;
12638391ecf4SShengjiu Wang swap(sdmac->event_mask[0], sdmac->event_mask[1]);
12648391ecf4SShengjiu Wang }
12658391ecf4SShengjiu Wang
12668391ecf4SShengjiu Wang if (sdmac->per_address2 >= sdma->spba_start_addr &&
12678391ecf4SShengjiu Wang sdmac->per_address2 <= sdma->spba_end_addr)
12688391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
12698391ecf4SShengjiu Wang
12708391ecf4SShengjiu Wang if (sdmac->per_address >= sdma->spba_start_addr &&
12718391ecf4SShengjiu Wang sdmac->per_address <= sdma->spba_end_addr)
12728391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
12738391ecf4SShengjiu Wang
12748391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
1275a20f10d6SShengjiu Wang
1276a20f10d6SShengjiu Wang /*
1277a20f10d6SShengjiu Wang * Limitation: The p2p script support dual fifos in maximum,
1278a20f10d6SShengjiu Wang * So when fifo number is larger than 1, force enable dual
1279a20f10d6SShengjiu Wang * fifos.
1280a20f10d6SShengjiu Wang */
1281a20f10d6SShengjiu Wang if (sdmac->n_fifos_src > 1)
1282a20f10d6SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD;
1283a20f10d6SShengjiu Wang if (sdmac->n_fifos_dst > 1)
1284a20f10d6SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD;
12858391ecf4SShengjiu Wang }
12868391ecf4SShengjiu Wang
sdma_set_watermarklevel_for_sais(struct sdma_channel * sdmac)1287824a0a02SSascha Hauer static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac)
1288824a0a02SSascha Hauer {
1289824a0a02SSascha Hauer unsigned int n_fifos;
1290e0c7ea83SShengjiu Wang unsigned int stride_fifos;
1291e0c7ea83SShengjiu Wang unsigned int words_per_fifo;
1292824a0a02SSascha Hauer
1293824a0a02SSascha Hauer if (sdmac->sw_done)
1294824a0a02SSascha Hauer sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE;
1295824a0a02SSascha Hauer
1296e0c7ea83SShengjiu Wang if (sdmac->direction == DMA_DEV_TO_MEM) {
1297824a0a02SSascha Hauer n_fifos = sdmac->n_fifos_src;
1298e0c7ea83SShengjiu Wang stride_fifos = sdmac->stride_fifos_src;
1299e0c7ea83SShengjiu Wang } else {
1300824a0a02SSascha Hauer n_fifos = sdmac->n_fifos_dst;
1301e0c7ea83SShengjiu Wang stride_fifos = sdmac->stride_fifos_dst;
1302e0c7ea83SShengjiu Wang }
1303e0c7ea83SShengjiu Wang
1304e0c7ea83SShengjiu Wang words_per_fifo = sdmac->words_per_fifo;
1305824a0a02SSascha Hauer
1306824a0a02SSascha Hauer sdmac->watermark_level |=
1307824a0a02SSascha Hauer FIELD_PREP(SDMA_WATERMARK_LEVEL_N_FIFOS, n_fifos);
1308e0c7ea83SShengjiu Wang sdmac->watermark_level |=
1309e0c7ea83SShengjiu Wang FIELD_PREP(SDMA_WATERMARK_LEVEL_OFF_FIFOS, stride_fifos);
1310e0c7ea83SShengjiu Wang if (words_per_fifo)
1311e0c7ea83SShengjiu Wang sdmac->watermark_level |=
1312e0c7ea83SShengjiu Wang FIELD_PREP(SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO, (words_per_fifo - 1));
1313824a0a02SSascha Hauer }
1314824a0a02SSascha Hauer
sdma_config_channel(struct dma_chan * chan)13157b350ab0SMaxime Ripard static int sdma_config_channel(struct dma_chan *chan)
13161ec1e82fSSascha Hauer {
13177b350ab0SMaxime Ripard struct sdma_channel *sdmac = to_sdma_chan(chan);
1318625d8936SSascha Hauer int ret;
13191ec1e82fSSascha Hauer
13207b350ab0SMaxime Ripard sdma_disable_channel(chan);
13211ec1e82fSSascha Hauer
13220bbc1413SRichard Zhao sdmac->event_mask[0] = 0;
13230bbc1413SRichard Zhao sdmac->event_mask[1] = 0;
13241ec1e82fSSascha Hauer sdmac->shp_addr = 0;
13251ec1e82fSSascha Hauer sdmac->per_addr = 0;
13261ec1e82fSSascha Hauer
13271ec1e82fSSascha Hauer switch (sdmac->peripheral_type) {
13281ec1e82fSSascha Hauer case IMX_DMATYPE_DSP:
13291ec1e82fSSascha Hauer sdma_config_ownership(sdmac, false, true, true);
13301ec1e82fSSascha Hauer break;
13311ec1e82fSSascha Hauer case IMX_DMATYPE_MEMORY:
13321ec1e82fSSascha Hauer sdma_config_ownership(sdmac, false, true, false);
13331ec1e82fSSascha Hauer break;
13341ec1e82fSSascha Hauer default:
13351ec1e82fSSascha Hauer sdma_config_ownership(sdmac, true, true, false);
13361ec1e82fSSascha Hauer break;
13371ec1e82fSSascha Hauer }
13381ec1e82fSSascha Hauer
1339625d8936SSascha Hauer ret = sdma_get_pc(sdmac, sdmac->peripheral_type);
1340625d8936SSascha Hauer if (ret)
1341625d8936SSascha Hauer return ret;
13421ec1e82fSSascha Hauer
13431ec1e82fSSascha Hauer if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
13441ec1e82fSSascha Hauer (sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
13451ec1e82fSSascha Hauer /* Handle multiple event channels differently */
13461ec1e82fSSascha Hauer if (sdmac->event_id1) {
13478391ecf4SShengjiu Wang if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
13488391ecf4SShengjiu Wang sdmac->peripheral_type == IMX_DMATYPE_ASRC)
13498391ecf4SShengjiu Wang sdma_set_watermarklevel_for_p2p(sdmac);
13501f8595efSFlavio Suligoi } else {
1351824a0a02SSascha Hauer if (sdmac->peripheral_type ==
1352824a0a02SSascha Hauer IMX_DMATYPE_MULTI_SAI)
1353824a0a02SSascha Hauer sdma_set_watermarklevel_for_sais(sdmac);
1354824a0a02SSascha Hauer
13550bbc1413SRichard Zhao __set_bit(sdmac->event_id0, sdmac->event_mask);
13561f8595efSFlavio Suligoi }
13578391ecf4SShengjiu Wang
13581ec1e82fSSascha Hauer /* Address */
13591ec1e82fSSascha Hauer sdmac->shp_addr = sdmac->per_address;
13608391ecf4SShengjiu Wang sdmac->per_addr = sdmac->per_address2;
13611ec1e82fSSascha Hauer } else {
13621ec1e82fSSascha Hauer sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
13631ec1e82fSSascha Hauer }
13641ec1e82fSSascha Hauer
1365e555a03bSRobin Gong return 0;
13661ec1e82fSSascha Hauer }
13671ec1e82fSSascha Hauer
sdma_set_channel_priority(struct sdma_channel * sdmac,unsigned int priority)13681ec1e82fSSascha Hauer static int sdma_set_channel_priority(struct sdma_channel *sdmac,
13691ec1e82fSSascha Hauer unsigned int priority)
13701ec1e82fSSascha Hauer {
13711ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
13721ec1e82fSSascha Hauer int channel = sdmac->channel;
13731ec1e82fSSascha Hauer
13741ec1e82fSSascha Hauer if (priority < MXC_SDMA_MIN_PRIORITY
13751ec1e82fSSascha Hauer || priority > MXC_SDMA_MAX_PRIORITY) {
13761ec1e82fSSascha Hauer return -EINVAL;
13771ec1e82fSSascha Hauer }
13781ec1e82fSSascha Hauer
1379c4b56857SRichard Zhao writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
13801ec1e82fSSascha Hauer
13811ec1e82fSSascha Hauer return 0;
13821ec1e82fSSascha Hauer }
13831ec1e82fSSascha Hauer
sdma_request_channel0(struct sdma_engine * sdma)138457b772b8SRobin Gong static int sdma_request_channel0(struct sdma_engine *sdma)
13851ec1e82fSSascha Hauer {
13861ec1e82fSSascha Hauer int ret = -EBUSY;
13871ec1e82fSSascha Hauer
1388802ef223SNicolin Chen if (sdma->iram_pool)
1389802ef223SNicolin Chen sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool,
1390802ef223SNicolin Chen sizeof(struct sdma_buffer_descriptor),
1391802ef223SNicolin Chen &sdma->bd0_phys);
1392802ef223SNicolin Chen else
1393802ef223SNicolin Chen sdma->bd0 = dma_alloc_coherent(sdma->dev,
1394802ef223SNicolin Chen sizeof(struct sdma_buffer_descriptor),
1395802ef223SNicolin Chen &sdma->bd0_phys, GFP_NOWAIT);
139657b772b8SRobin Gong if (!sdma->bd0) {
13971ec1e82fSSascha Hauer ret = -ENOMEM;
13981ec1e82fSSascha Hauer goto out;
13991ec1e82fSSascha Hauer }
14001ec1e82fSSascha Hauer
140157b772b8SRobin Gong sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
140257b772b8SRobin Gong sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
14031ec1e82fSSascha Hauer
140457b772b8SRobin Gong sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
14051ec1e82fSSascha Hauer return 0;
14061ec1e82fSSascha Hauer out:
14071ec1e82fSSascha Hauer
14081ec1e82fSSascha Hauer return ret;
14091ec1e82fSSascha Hauer }
14101ec1e82fSSascha Hauer
141157b772b8SRobin Gong
sdma_alloc_bd(struct sdma_desc * desc)141257b772b8SRobin Gong static int sdma_alloc_bd(struct sdma_desc *desc)
14131ec1e82fSSascha Hauer {
1414ebb853b1SLucas Stach u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1415802ef223SNicolin Chen struct sdma_engine *sdma = desc->sdmac->sdma;
141657b772b8SRobin Gong int ret = 0;
14171ec1e82fSSascha Hauer
1418802ef223SNicolin Chen if (sdma->iram_pool)
1419802ef223SNicolin Chen desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, &desc->bd_phys);
1420802ef223SNicolin Chen else
1421802ef223SNicolin Chen desc->bd = dma_alloc_coherent(sdma->dev, bd_size, &desc->bd_phys, GFP_NOWAIT);
1422802ef223SNicolin Chen
142357b772b8SRobin Gong if (!desc->bd) {
142457b772b8SRobin Gong ret = -ENOMEM;
142557b772b8SRobin Gong goto out;
142657b772b8SRobin Gong }
142757b772b8SRobin Gong out:
142857b772b8SRobin Gong return ret;
142957b772b8SRobin Gong }
14301ec1e82fSSascha Hauer
sdma_free_bd(struct sdma_desc * desc)143157b772b8SRobin Gong static void sdma_free_bd(struct sdma_desc *desc)
143257b772b8SRobin Gong {
1433ebb853b1SLucas Stach u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1434802ef223SNicolin Chen struct sdma_engine *sdma = desc->sdmac->sdma;
1435ebb853b1SLucas Stach
1436802ef223SNicolin Chen if (sdma->iram_pool)
1437802ef223SNicolin Chen gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, bd_size);
1438802ef223SNicolin Chen else
1439802ef223SNicolin Chen dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys);
144057b772b8SRobin Gong }
14411ec1e82fSSascha Hauer
sdma_desc_free(struct virt_dma_desc * vd)144257b772b8SRobin Gong static void sdma_desc_free(struct virt_dma_desc *vd)
144357b772b8SRobin Gong {
144457b772b8SRobin Gong struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
144557b772b8SRobin Gong
144657b772b8SRobin Gong sdma_free_bd(desc);
144757b772b8SRobin Gong kfree(desc);
14481ec1e82fSSascha Hauer }
14491ec1e82fSSascha Hauer
sdma_alloc_chan_resources(struct dma_chan * chan)14501ec1e82fSSascha Hauer static int sdma_alloc_chan_resources(struct dma_chan *chan)
14511ec1e82fSSascha Hauer {
14521ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
14531ec1e82fSSascha Hauer struct imx_dma_data *data = chan->private;
14540f06c027SRobin Gong struct imx_dma_data mem_data;
14551ec1e82fSSascha Hauer int prio, ret;
14561ec1e82fSSascha Hauer
14570f06c027SRobin Gong /*
14580f06c027SRobin Gong * MEMCPY may never setup chan->private by filter function such as
14590f06c027SRobin Gong * dmatest, thus create 'struct imx_dma_data mem_data' for this case.
14600f06c027SRobin Gong * Please note in any other slave case, you have to setup chan->private
14610f06c027SRobin Gong * with 'struct imx_dma_data' in your own filter function if you want to
14620f06c027SRobin Gong * request dma channel by dma_request_channel() rather than
14630f06c027SRobin Gong * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
14640f06c027SRobin Gong * to warn you to correct your filter function.
14650f06c027SRobin Gong */
14660f06c027SRobin Gong if (!data) {
14670f06c027SRobin Gong dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
14680f06c027SRobin Gong mem_data.priority = 2;
14690f06c027SRobin Gong mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
14700f06c027SRobin Gong mem_data.dma_request = 0;
14710f06c027SRobin Gong mem_data.dma_request2 = 0;
14720f06c027SRobin Gong data = &mem_data;
14730f06c027SRobin Gong
1474625d8936SSascha Hauer ret = sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
1475625d8936SSascha Hauer if (ret)
1476625d8936SSascha Hauer return ret;
14770f06c027SRobin Gong }
14781ec1e82fSSascha Hauer
14791ec1e82fSSascha Hauer switch (data->priority) {
14801ec1e82fSSascha Hauer case DMA_PRIO_HIGH:
14811ec1e82fSSascha Hauer prio = 3;
14821ec1e82fSSascha Hauer break;
14831ec1e82fSSascha Hauer case DMA_PRIO_MEDIUM:
14841ec1e82fSSascha Hauer prio = 2;
14851ec1e82fSSascha Hauer break;
14861ec1e82fSSascha Hauer case DMA_PRIO_LOW:
14871ec1e82fSSascha Hauer default:
14881ec1e82fSSascha Hauer prio = 1;
14891ec1e82fSSascha Hauer break;
14901ec1e82fSSascha Hauer }
14911ec1e82fSSascha Hauer
14921ec1e82fSSascha Hauer sdmac->peripheral_type = data->peripheral_type;
14931ec1e82fSSascha Hauer sdmac->event_id0 = data->dma_request;
14948391ecf4SShengjiu Wang sdmac->event_id1 = data->dma_request2;
1495c2c744d3SRichard Zhao
1496b93edcddSFabio Estevam ret = clk_enable(sdmac->sdma->clk_ipg);
1497b93edcddSFabio Estevam if (ret)
1498b93edcddSFabio Estevam return ret;
1499b93edcddSFabio Estevam ret = clk_enable(sdmac->sdma->clk_ahb);
1500b93edcddSFabio Estevam if (ret)
1501b93edcddSFabio Estevam goto disable_clk_ipg;
1502c2c744d3SRichard Zhao
15033bb5e7caSRichard Zhao ret = sdma_set_channel_priority(sdmac, prio);
15041ec1e82fSSascha Hauer if (ret)
1505b93edcddSFabio Estevam goto disable_clk_ahb;
15061ec1e82fSSascha Hauer
15071ec1e82fSSascha Hauer return 0;
1508b93edcddSFabio Estevam
1509b93edcddSFabio Estevam disable_clk_ahb:
1510b93edcddSFabio Estevam clk_disable(sdmac->sdma->clk_ahb);
1511b93edcddSFabio Estevam disable_clk_ipg:
1512b93edcddSFabio Estevam clk_disable(sdmac->sdma->clk_ipg);
1513b93edcddSFabio Estevam return ret;
15141ec1e82fSSascha Hauer }
15151ec1e82fSSascha Hauer
sdma_free_chan_resources(struct dma_chan * chan)15161ec1e82fSSascha Hauer static void sdma_free_chan_resources(struct dma_chan *chan)
15171ec1e82fSSascha Hauer {
15181ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
15191ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
15201ec1e82fSSascha Hauer
1521a80f2787SSascha Hauer sdma_terminate_all(chan);
1522b8603d2aSLucas Stach
1523b8603d2aSLucas Stach sdma_channel_synchronize(chan);
15241ec1e82fSSascha Hauer
15251ec1e82fSSascha Hauer sdma_event_disable(sdmac, sdmac->event_id0);
15261ec1e82fSSascha Hauer if (sdmac->event_id1)
15271ec1e82fSSascha Hauer sdma_event_disable(sdmac, sdmac->event_id1);
15281ec1e82fSSascha Hauer
15291ec1e82fSSascha Hauer sdmac->event_id0 = 0;
15301ec1e82fSSascha Hauer sdmac->event_id1 = 0;
15311ec1e82fSSascha Hauer
15321ec1e82fSSascha Hauer sdma_set_channel_priority(sdmac, 0);
15331ec1e82fSSascha Hauer
15347560e3f3SSascha Hauer clk_disable(sdma->clk_ipg);
15357560e3f3SSascha Hauer clk_disable(sdma->clk_ahb);
15361ec1e82fSSascha Hauer }
15371ec1e82fSSascha Hauer
sdma_transfer_init(struct sdma_channel * sdmac,enum dma_transfer_direction direction,u32 bds)153821420841SRobin Gong static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
153921420841SRobin Gong enum dma_transfer_direction direction, u32 bds)
154021420841SRobin Gong {
154121420841SRobin Gong struct sdma_desc *desc;
154221420841SRobin Gong
1543e8fafa50SRobin Gong if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) {
1544e8fafa50SRobin Gong dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n");
1545e8fafa50SRobin Gong goto err_out;
1546e8fafa50SRobin Gong }
1547e8fafa50SRobin Gong
154821420841SRobin Gong desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
154921420841SRobin Gong if (!desc)
155021420841SRobin Gong goto err_out;
155121420841SRobin Gong
155221420841SRobin Gong sdmac->status = DMA_IN_PROGRESS;
155321420841SRobin Gong sdmac->direction = direction;
155421420841SRobin Gong sdmac->flags = 0;
155521420841SRobin Gong
155621420841SRobin Gong desc->chn_count = 0;
155721420841SRobin Gong desc->chn_real_count = 0;
155821420841SRobin Gong desc->buf_tail = 0;
155921420841SRobin Gong desc->buf_ptail = 0;
156021420841SRobin Gong desc->sdmac = sdmac;
156121420841SRobin Gong desc->num_bd = bds;
156221420841SRobin Gong
1563e873d432SJoy Zou if (bds && sdma_alloc_bd(desc))
156421420841SRobin Gong goto err_desc_out;
156521420841SRobin Gong
15660f06c027SRobin Gong /* No slave_config called in MEMCPY case, so do here */
15670f06c027SRobin Gong if (direction == DMA_MEM_TO_MEM)
15680f06c027SRobin Gong sdma_config_ownership(sdmac, false, true, false);
15690f06c027SRobin Gong
157021420841SRobin Gong if (sdma_load_context(sdmac))
15711417f59aSHui Wang goto err_bd_out;
157221420841SRobin Gong
157321420841SRobin Gong return desc;
157421420841SRobin Gong
15751417f59aSHui Wang err_bd_out:
15761417f59aSHui Wang sdma_free_bd(desc);
157721420841SRobin Gong err_desc_out:
157821420841SRobin Gong kfree(desc);
157921420841SRobin Gong err_out:
158021420841SRobin Gong return NULL;
158121420841SRobin Gong }
158221420841SRobin Gong
sdma_prep_memcpy(struct dma_chan * chan,dma_addr_t dma_dst,dma_addr_t dma_src,size_t len,unsigned long flags)15830f06c027SRobin Gong static struct dma_async_tx_descriptor *sdma_prep_memcpy(
15840f06c027SRobin Gong struct dma_chan *chan, dma_addr_t dma_dst,
15850f06c027SRobin Gong dma_addr_t dma_src, size_t len, unsigned long flags)
15860f06c027SRobin Gong {
15870f06c027SRobin Gong struct sdma_channel *sdmac = to_sdma_chan(chan);
15880f06c027SRobin Gong struct sdma_engine *sdma = sdmac->sdma;
15890f06c027SRobin Gong int channel = sdmac->channel;
15900f06c027SRobin Gong size_t count;
15910f06c027SRobin Gong int i = 0, param;
15920f06c027SRobin Gong struct sdma_buffer_descriptor *bd;
15930f06c027SRobin Gong struct sdma_desc *desc;
15940f06c027SRobin Gong
15950f06c027SRobin Gong if (!chan || !len)
15960f06c027SRobin Gong return NULL;
15970f06c027SRobin Gong
15980f06c027SRobin Gong dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
15990f06c027SRobin Gong &dma_src, &dma_dst, len, channel);
16000f06c027SRobin Gong
16010f06c027SRobin Gong desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
16020f06c027SRobin Gong len / SDMA_BD_MAX_CNT + 1);
16030f06c027SRobin Gong if (!desc)
16040f06c027SRobin Gong return NULL;
16050f06c027SRobin Gong
16060f06c027SRobin Gong do {
16070f06c027SRobin Gong count = min_t(size_t, len, SDMA_BD_MAX_CNT);
16080f06c027SRobin Gong bd = &desc->bd[i];
16090f06c027SRobin Gong bd->buffer_addr = dma_src;
16100f06c027SRobin Gong bd->ext_buffer_addr = dma_dst;
16110f06c027SRobin Gong bd->mode.count = count;
16120f06c027SRobin Gong desc->chn_count += count;
16130f06c027SRobin Gong bd->mode.command = 0;
16140f06c027SRobin Gong
16150f06c027SRobin Gong dma_src += count;
16160f06c027SRobin Gong dma_dst += count;
16170f06c027SRobin Gong len -= count;
16180f06c027SRobin Gong i++;
16190f06c027SRobin Gong
16200f06c027SRobin Gong param = BD_DONE | BD_EXTD | BD_CONT;
16210f06c027SRobin Gong /* last bd */
16220f06c027SRobin Gong if (!len) {
16230f06c027SRobin Gong param |= BD_INTR;
16240f06c027SRobin Gong param |= BD_LAST;
16250f06c027SRobin Gong param &= ~BD_CONT;
16260f06c027SRobin Gong }
16270f06c027SRobin Gong
16280f06c027SRobin Gong dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
16290f06c027SRobin Gong i, count, bd->buffer_addr,
16300f06c027SRobin Gong param & BD_WRAP ? "wrap" : "",
16310f06c027SRobin Gong param & BD_INTR ? " intr" : "");
16320f06c027SRobin Gong
16330f06c027SRobin Gong bd->mode.status = param;
16340f06c027SRobin Gong } while (len);
16350f06c027SRobin Gong
16360f06c027SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
16370f06c027SRobin Gong }
16380f06c027SRobin Gong
sdma_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)16391ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
16401ec1e82fSSascha Hauer struct dma_chan *chan, struct scatterlist *sgl,
1641db8196dfSVinod Koul unsigned int sg_len, enum dma_transfer_direction direction,
1642185ecb5fSAlexandre Bounine unsigned long flags, void *context)
16431ec1e82fSSascha Hauer {
16441ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
16451ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
1646ad78b000SVinod Koul int i, count;
164723889c63SSascha Hauer int channel = sdmac->channel;
16481ec1e82fSSascha Hauer struct scatterlist *sg;
164957b772b8SRobin Gong struct sdma_desc *desc;
16501ec1e82fSSascha Hauer
1651107d0644SVinod Koul sdma_config_write(chan, &sdmac->slave_config, direction);
1652107d0644SVinod Koul
165321420841SRobin Gong desc = sdma_transfer_init(sdmac, direction, sg_len);
165457b772b8SRobin Gong if (!desc)
165557b772b8SRobin Gong goto err_out;
165657b772b8SRobin Gong
16571ec1e82fSSascha Hauer dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
16581ec1e82fSSascha Hauer sg_len, channel);
16591ec1e82fSSascha Hauer
16601ec1e82fSSascha Hauer for_each_sg(sgl, sg, sg_len, i) {
166176c33d27SSascha Hauer struct sdma_buffer_descriptor *bd = &desc->bd[i];
16621ec1e82fSSascha Hauer int param;
16631ec1e82fSSascha Hauer
1664d2f5c276SAnatolij Gustschin bd->buffer_addr = sg->dma_address;
16651ec1e82fSSascha Hauer
1666fdaf9c4bSLars-Peter Clausen count = sg_dma_len(sg);
16671ec1e82fSSascha Hauer
16684a6b2e8aSRobin Gong if (count > SDMA_BD_MAX_CNT) {
16691ec1e82fSSascha Hauer dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
16704a6b2e8aSRobin Gong channel, count, SDMA_BD_MAX_CNT);
167157b772b8SRobin Gong goto err_bd_out;
16721ec1e82fSSascha Hauer }
16731ec1e82fSSascha Hauer
16741ec1e82fSSascha Hauer bd->mode.count = count;
167576c33d27SSascha Hauer desc->chn_count += count;
16761ec1e82fSSascha Hauer
1677ad78b000SVinod Koul if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
167857b772b8SRobin Gong goto err_bd_out;
16791fa81c27SSascha Hauer
16801fa81c27SSascha Hauer switch (sdmac->word_size) {
16811fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_4_BYTES:
16821ec1e82fSSascha Hauer bd->mode.command = 0;
16831fa81c27SSascha Hauer if (count & 3 || sg->dma_address & 3)
168457b772b8SRobin Gong goto err_bd_out;
16851fa81c27SSascha Hauer break;
168628810938SShengjiu Wang case DMA_SLAVE_BUSWIDTH_3_BYTES:
168728810938SShengjiu Wang bd->mode.command = 3;
168828810938SShengjiu Wang break;
16891fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_2_BYTES:
16901fa81c27SSascha Hauer bd->mode.command = 2;
16911fa81c27SSascha Hauer if (count & 1 || sg->dma_address & 1)
169257b772b8SRobin Gong goto err_bd_out;
16931fa81c27SSascha Hauer break;
16941fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_1_BYTE:
16951fa81c27SSascha Hauer bd->mode.command = 1;
16961fa81c27SSascha Hauer break;
16971fa81c27SSascha Hauer default:
169857b772b8SRobin Gong goto err_bd_out;
16991fa81c27SSascha Hauer }
17001ec1e82fSSascha Hauer
17011ec1e82fSSascha Hauer param = BD_DONE | BD_EXTD | BD_CONT;
17021ec1e82fSSascha Hauer
1703341b9419SShawn Guo if (i + 1 == sg_len) {
17041ec1e82fSSascha Hauer param |= BD_INTR;
1705341b9419SShawn Guo param |= BD_LAST;
1706341b9419SShawn Guo param &= ~BD_CONT;
17071ec1e82fSSascha Hauer }
17081ec1e82fSSascha Hauer
1709c3cc74b2SOlof Johansson dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1710c3cc74b2SOlof Johansson i, count, (u64)sg->dma_address,
17111ec1e82fSSascha Hauer param & BD_WRAP ? "wrap" : "",
17121ec1e82fSSascha Hauer param & BD_INTR ? " intr" : "");
17131ec1e82fSSascha Hauer
17141ec1e82fSSascha Hauer bd->mode.status = param;
17151ec1e82fSSascha Hauer }
17161ec1e82fSSascha Hauer
171757b772b8SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
171857b772b8SRobin Gong err_bd_out:
171957b772b8SRobin Gong sdma_free_bd(desc);
172057b772b8SRobin Gong kfree(desc);
17211ec1e82fSSascha Hauer err_out:
17224b2ce9ddSShawn Guo sdmac->status = DMA_ERROR;
17231ec1e82fSSascha Hauer return NULL;
17241ec1e82fSSascha Hauer }
17251ec1e82fSSascha Hauer
sdma_prep_dma_cyclic(struct dma_chan * chan,dma_addr_t dma_addr,size_t buf_len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)17261ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
17271ec1e82fSSascha Hauer struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1728185ecb5fSAlexandre Bounine size_t period_len, enum dma_transfer_direction direction,
172931c1e5a1SLaurent Pinchart unsigned long flags)
17301ec1e82fSSascha Hauer {
17311ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
17321ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
1733e873d432SJoy Zou int num_periods = 0;
173423889c63SSascha Hauer int channel = sdmac->channel;
173521420841SRobin Gong int i = 0, buf = 0;
173657b772b8SRobin Gong struct sdma_desc *desc;
17371ec1e82fSSascha Hauer
17381ec1e82fSSascha Hauer dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
17391ec1e82fSSascha Hauer
1740e873d432SJoy Zou if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
1741e873d432SJoy Zou num_periods = buf_len / period_len;
1742e873d432SJoy Zou
1743107d0644SVinod Koul sdma_config_write(chan, &sdmac->slave_config, direction);
1744107d0644SVinod Koul
174521420841SRobin Gong desc = sdma_transfer_init(sdmac, direction, num_periods);
174657b772b8SRobin Gong if (!desc)
174757b772b8SRobin Gong goto err_out;
174857b772b8SRobin Gong
174976c33d27SSascha Hauer desc->period_len = period_len;
17508e2e27c7SRichard Zhao
17511ec1e82fSSascha Hauer sdmac->flags |= IMX_DMA_SG_LOOP;
17521ec1e82fSSascha Hauer
17534a6b2e8aSRobin Gong if (period_len > SDMA_BD_MAX_CNT) {
1754ba6ab3b3SArvind Yadav dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
17554a6b2e8aSRobin Gong channel, period_len, SDMA_BD_MAX_CNT);
175657b772b8SRobin Gong goto err_bd_out;
17571ec1e82fSSascha Hauer }
17581ec1e82fSSascha Hauer
1759e873d432SJoy Zou if (sdmac->peripheral_type == IMX_DMATYPE_HDMI)
1760e873d432SJoy Zou return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1761e873d432SJoy Zou
17621ec1e82fSSascha Hauer while (buf < buf_len) {
176376c33d27SSascha Hauer struct sdma_buffer_descriptor *bd = &desc->bd[i];
17641ec1e82fSSascha Hauer int param;
17651ec1e82fSSascha Hauer
17661ec1e82fSSascha Hauer bd->buffer_addr = dma_addr;
17671ec1e82fSSascha Hauer
17681ec1e82fSSascha Hauer bd->mode.count = period_len;
17691ec1e82fSSascha Hauer
17701ec1e82fSSascha Hauer if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
177157b772b8SRobin Gong goto err_bd_out;
17721ec1e82fSSascha Hauer if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
17731ec1e82fSSascha Hauer bd->mode.command = 0;
17741ec1e82fSSascha Hauer else
17751ec1e82fSSascha Hauer bd->mode.command = sdmac->word_size;
17761ec1e82fSSascha Hauer
17771ec1e82fSSascha Hauer param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
17781ec1e82fSSascha Hauer if (i + 1 == num_periods)
17791ec1e82fSSascha Hauer param |= BD_WRAP;
17801ec1e82fSSascha Hauer
1781ba6ab3b3SArvind Yadav dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
1782c3cc74b2SOlof Johansson i, period_len, (u64)dma_addr,
17831ec1e82fSSascha Hauer param & BD_WRAP ? "wrap" : "",
17841ec1e82fSSascha Hauer param & BD_INTR ? " intr" : "");
17851ec1e82fSSascha Hauer
17861ec1e82fSSascha Hauer bd->mode.status = param;
17871ec1e82fSSascha Hauer
17881ec1e82fSSascha Hauer dma_addr += period_len;
17891ec1e82fSSascha Hauer buf += period_len;
17901ec1e82fSSascha Hauer
17911ec1e82fSSascha Hauer i++;
17921ec1e82fSSascha Hauer }
17931ec1e82fSSascha Hauer
179457b772b8SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
179557b772b8SRobin Gong err_bd_out:
179657b772b8SRobin Gong sdma_free_bd(desc);
179757b772b8SRobin Gong kfree(desc);
17981ec1e82fSSascha Hauer err_out:
17991ec1e82fSSascha Hauer sdmac->status = DMA_ERROR;
18001ec1e82fSSascha Hauer return NULL;
18011ec1e82fSSascha Hauer }
18021ec1e82fSSascha Hauer
sdma_config_write(struct dma_chan * chan,struct dma_slave_config * dmaengine_cfg,enum dma_transfer_direction direction)1803107d0644SVinod Koul static int sdma_config_write(struct dma_chan *chan,
1804107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg,
1805107d0644SVinod Koul enum dma_transfer_direction direction)
18061ec1e82fSSascha Hauer {
18071ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
18081ec1e82fSSascha Hauer
1809107d0644SVinod Koul if (direction == DMA_DEV_TO_MEM) {
18101ec1e82fSSascha Hauer sdmac->per_address = dmaengine_cfg->src_addr;
181194ac27a5SPhilippe Rétornaz sdmac->watermark_level = dmaengine_cfg->src_maxburst *
181294ac27a5SPhilippe Rétornaz dmaengine_cfg->src_addr_width;
18131ec1e82fSSascha Hauer sdmac->word_size = dmaengine_cfg->src_addr_width;
1814107d0644SVinod Koul } else if (direction == DMA_DEV_TO_DEV) {
18158391ecf4SShengjiu Wang sdmac->per_address2 = dmaengine_cfg->src_addr;
18168391ecf4SShengjiu Wang sdmac->per_address = dmaengine_cfg->dst_addr;
18178391ecf4SShengjiu Wang sdmac->watermark_level = dmaengine_cfg->src_maxburst &
18188391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_LWML;
18198391ecf4SShengjiu Wang sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
18208391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_HWML;
18218391ecf4SShengjiu Wang sdmac->word_size = dmaengine_cfg->dst_addr_width;
1822e873d432SJoy Zou } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1823e873d432SJoy Zou sdmac->per_address = dmaengine_cfg->dst_addr;
1824e873d432SJoy Zou sdmac->per_address2 = dmaengine_cfg->src_addr;
1825e873d432SJoy Zou sdmac->watermark_level = 0;
18261ec1e82fSSascha Hauer } else {
18271ec1e82fSSascha Hauer sdmac->per_address = dmaengine_cfg->dst_addr;
182894ac27a5SPhilippe Rétornaz sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
182994ac27a5SPhilippe Rétornaz dmaengine_cfg->dst_addr_width;
18301ec1e82fSSascha Hauer sdmac->word_size = dmaengine_cfg->dst_addr_width;
18311ec1e82fSSascha Hauer }
1832107d0644SVinod Koul sdmac->direction = direction;
18337b350ab0SMaxime Ripard return sdma_config_channel(chan);
18341ec1e82fSSascha Hauer }
18351ec1e82fSSascha Hauer
sdma_config(struct dma_chan * chan,struct dma_slave_config * dmaengine_cfg)1836107d0644SVinod Koul static int sdma_config(struct dma_chan *chan,
1837107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg)
1838107d0644SVinod Koul {
1839107d0644SVinod Koul struct sdma_channel *sdmac = to_sdma_chan(chan);
1840824a0a02SSascha Hauer struct sdma_engine *sdma = sdmac->sdma;
1841107d0644SVinod Koul
1842107d0644SVinod Koul memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
1843107d0644SVinod Koul
1844824a0a02SSascha Hauer if (dmaengine_cfg->peripheral_config) {
1845824a0a02SSascha Hauer struct sdma_peripheral_config *sdmacfg = dmaengine_cfg->peripheral_config;
1846824a0a02SSascha Hauer if (dmaengine_cfg->peripheral_size != sizeof(struct sdma_peripheral_config)) {
1847824a0a02SSascha Hauer dev_err(sdma->dev, "Invalid peripheral size %zu, expected %zu\n",
1848824a0a02SSascha Hauer dmaengine_cfg->peripheral_size,
1849824a0a02SSascha Hauer sizeof(struct sdma_peripheral_config));
1850824a0a02SSascha Hauer return -EINVAL;
1851824a0a02SSascha Hauer }
1852824a0a02SSascha Hauer sdmac->n_fifos_src = sdmacfg->n_fifos_src;
1853824a0a02SSascha Hauer sdmac->n_fifos_dst = sdmacfg->n_fifos_dst;
1854e0c7ea83SShengjiu Wang sdmac->stride_fifos_src = sdmacfg->stride_fifos_src;
1855e0c7ea83SShengjiu Wang sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst;
1856e0c7ea83SShengjiu Wang sdmac->words_per_fifo = sdmacfg->words_per_fifo;
1857824a0a02SSascha Hauer sdmac->sw_done = sdmacfg->sw_done;
1858824a0a02SSascha Hauer }
1859824a0a02SSascha Hauer
1860107d0644SVinod Koul /* Set ENBLn earlier to make sure dma request triggered after that */
1861107d0644SVinod Koul if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
1862107d0644SVinod Koul return -EINVAL;
1863107d0644SVinod Koul sdma_event_enable(sdmac, sdmac->event_id0);
1864107d0644SVinod Koul
1865107d0644SVinod Koul if (sdmac->event_id1) {
1866107d0644SVinod Koul if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
1867107d0644SVinod Koul return -EINVAL;
1868107d0644SVinod Koul sdma_event_enable(sdmac, sdmac->event_id1);
1869107d0644SVinod Koul }
1870107d0644SVinod Koul
1871107d0644SVinod Koul return 0;
1872107d0644SVinod Koul }
1873107d0644SVinod Koul
sdma_tx_status(struct dma_chan * chan,dma_cookie_t cookie,struct dma_tx_state * txstate)18741ec1e82fSSascha Hauer static enum dma_status sdma_tx_status(struct dma_chan *chan,
18751ec1e82fSSascha Hauer dma_cookie_t cookie,
18761ec1e82fSSascha Hauer struct dma_tx_state *txstate)
18771ec1e82fSSascha Hauer {
18781ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
1879a1ff6a07SSascha Hauer struct sdma_desc *desc = NULL;
1880d1a792f3SRussell King - ARM Linux u32 residue;
188157b772b8SRobin Gong struct virt_dma_desc *vd;
188257b772b8SRobin Gong enum dma_status ret;
188357b772b8SRobin Gong unsigned long flags;
1884d1a792f3SRussell King - ARM Linux
188557b772b8SRobin Gong ret = dma_cookie_status(chan, cookie, txstate);
188657b772b8SRobin Gong if (ret == DMA_COMPLETE || !txstate)
188757b772b8SRobin Gong return ret;
188857b772b8SRobin Gong
188957b772b8SRobin Gong spin_lock_irqsave(&sdmac->vc.lock, flags);
1890a1ff6a07SSascha Hauer
189157b772b8SRobin Gong vd = vchan_find_desc(&sdmac->vc, cookie);
1892a1ff6a07SSascha Hauer if (vd)
189357b772b8SRobin Gong desc = to_sdma_desc(&vd->tx);
1894a1ff6a07SSascha Hauer else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
1895a1ff6a07SSascha Hauer desc = sdmac->desc;
1896a1ff6a07SSascha Hauer
1897a1ff6a07SSascha Hauer if (desc) {
1898d1a792f3SRussell King - ARM Linux if (sdmac->flags & IMX_DMA_SG_LOOP)
189976c33d27SSascha Hauer residue = (desc->num_bd - desc->buf_ptail) *
190076c33d27SSascha Hauer desc->period_len - desc->chn_real_count;
1901d1a792f3SRussell King - ARM Linux else
190276c33d27SSascha Hauer residue = desc->chn_count - desc->chn_real_count;
190357b772b8SRobin Gong } else {
190457b772b8SRobin Gong residue = 0;
190557b772b8SRobin Gong }
1906a1ff6a07SSascha Hauer
190757b772b8SRobin Gong spin_unlock_irqrestore(&sdmac->vc.lock, flags);
19081ec1e82fSSascha Hauer
1909e8e3a790SAndy Shevchenko dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
1910d1a792f3SRussell King - ARM Linux residue);
19111ec1e82fSSascha Hauer
19128a965911SShawn Guo return sdmac->status;
19131ec1e82fSSascha Hauer }
19141ec1e82fSSascha Hauer
sdma_issue_pending(struct dma_chan * chan)19151ec1e82fSSascha Hauer static void sdma_issue_pending(struct dma_chan *chan)
19161ec1e82fSSascha Hauer {
19172b4f130eSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan);
191857b772b8SRobin Gong unsigned long flags;
19192b4f130eSSascha Hauer
192057b772b8SRobin Gong spin_lock_irqsave(&sdmac->vc.lock, flags);
192157b772b8SRobin Gong if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
192257b772b8SRobin Gong sdma_start_desc(sdmac);
192357b772b8SRobin Gong spin_unlock_irqrestore(&sdmac->vc.lock, flags);
19241ec1e82fSSascha Hauer }
19251ec1e82fSSascha Hauer
19261cb49f38SFrank Li #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 \
19271cb49f38SFrank Li (offsetof(struct sdma_script_start_addrs, v1_end) / sizeof(s32))
19281cb49f38SFrank Li
19291cb49f38SFrank Li #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 \
19301cb49f38SFrank Li (offsetof(struct sdma_script_start_addrs, v2_end) / sizeof(s32))
19311cb49f38SFrank Li
19321cb49f38SFrank Li #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 \
19331cb49f38SFrank Li (offsetof(struct sdma_script_start_addrs, v3_end) / sizeof(s32))
19341cb49f38SFrank Li
19351cb49f38SFrank Li #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 \
19361cb49f38SFrank Li (offsetof(struct sdma_script_start_addrs, v4_end) / sizeof(s32))
19375b28aa31SSascha Hauer
sdma_add_scripts(struct sdma_engine * sdma,const struct sdma_script_start_addrs * addr)19385b28aa31SSascha Hauer static void sdma_add_scripts(struct sdma_engine *sdma,
19395b28aa31SSascha Hauer const struct sdma_script_start_addrs *addr)
19405b28aa31SSascha Hauer {
19415b28aa31SSascha Hauer s32 *addr_arr = (u32 *)addr;
19425b28aa31SSascha Hauer s32 *saddr_arr = (u32 *)sdma->script_addrs;
19435b28aa31SSascha Hauer int i;
19445b28aa31SSascha Hauer
194570dabaedSNicolin Chen /* use the default firmware in ROM if missing external firmware */
194670dabaedSNicolin Chen if (!sdma->script_number)
194770dabaedSNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
194870dabaedSNicolin Chen
1949bd73dfabSRobin Gong if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
1950bd73dfabSRobin Gong / sizeof(s32)) {
1951bd73dfabSRobin Gong dev_err(sdma->dev,
1952bd73dfabSRobin Gong "SDMA script number %d not match with firmware.\n",
1953bd73dfabSRobin Gong sdma->script_number);
1954bd73dfabSRobin Gong return;
1955bd73dfabSRobin Gong }
1956bd73dfabSRobin Gong
1957cd72b846SNicolin Chen for (i = 0; i < sdma->script_number; i++)
19585b28aa31SSascha Hauer if (addr_arr[i] > 0)
19595b28aa31SSascha Hauer saddr_arr[i] = addr_arr[i];
1960b98ce2f4SRobin Gong
1961b98ce2f4SRobin Gong /*
1962a3ae97f4SKevin Groeneveld * For compatibility with NXP internal legacy kernel before 4.19 which
1963a3ae97f4SKevin Groeneveld * is based on uart ram script and mainline kernel based on uart rom
1964a3ae97f4SKevin Groeneveld * script, both uart ram/rom scripts are present in newer sdma
1965a3ae97f4SKevin Groeneveld * firmware. Use the rom versions if they are present (V3 or newer).
1966b98ce2f4SRobin Gong */
1967a3ae97f4SKevin Groeneveld if (sdma->script_number >= SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3) {
1968a3ae97f4SKevin Groeneveld if (addr->uart_2_mcu_rom_addr)
1969a3ae97f4SKevin Groeneveld sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_rom_addr;
1970a3ae97f4SKevin Groeneveld if (addr->uartsh_2_mcu_rom_addr)
1971a3ae97f4SKevin Groeneveld sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_rom_addr;
1972a3ae97f4SKevin Groeneveld }
19735b28aa31SSascha Hauer }
19745b28aa31SSascha Hauer
sdma_load_firmware(const struct firmware * fw,void * context)19757b4b88e0SSascha Hauer static void sdma_load_firmware(const struct firmware *fw, void *context)
19765b28aa31SSascha Hauer {
19777b4b88e0SSascha Hauer struct sdma_engine *sdma = context;
19785b28aa31SSascha Hauer const struct sdma_firmware_header *header;
19795b28aa31SSascha Hauer const struct sdma_script_start_addrs *addr;
19805b28aa31SSascha Hauer unsigned short *ram_code;
19815b28aa31SSascha Hauer
19827b4b88e0SSascha Hauer if (!fw) {
19830f927a11SSascha Hauer dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
19840f927a11SSascha Hauer /* In this case we just use the ROM firmware. */
19857b4b88e0SSascha Hauer return;
19867b4b88e0SSascha Hauer }
19875b28aa31SSascha Hauer
19885b28aa31SSascha Hauer if (fw->size < sizeof(*header))
19895b28aa31SSascha Hauer goto err_firmware;
19905b28aa31SSascha Hauer
19915b28aa31SSascha Hauer header = (struct sdma_firmware_header *)fw->data;
19925b28aa31SSascha Hauer
19935b28aa31SSascha Hauer if (header->magic != SDMA_FIRMWARE_MAGIC)
19945b28aa31SSascha Hauer goto err_firmware;
19955b28aa31SSascha Hauer if (header->ram_code_start + header->ram_code_size > fw->size)
19965b28aa31SSascha Hauer goto err_firmware;
1997cd72b846SNicolin Chen switch (header->version_major) {
1998cd72b846SNicolin Chen case 1:
1999cd72b846SNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
2000cd72b846SNicolin Chen break;
2001cd72b846SNicolin Chen case 2:
2002cd72b846SNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
2003cd72b846SNicolin Chen break;
2004a572460bSFabio Estevam case 3:
2005a572460bSFabio Estevam sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
2006a572460bSFabio Estevam break;
2007b7d2648aSFabio Estevam case 4:
2008b7d2648aSFabio Estevam sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
2009b7d2648aSFabio Estevam break;
2010cd72b846SNicolin Chen default:
2011cd72b846SNicolin Chen dev_err(sdma->dev, "unknown firmware version\n");
2012cd72b846SNicolin Chen goto err_firmware;
2013cd72b846SNicolin Chen }
20145b28aa31SSascha Hauer
20155b28aa31SSascha Hauer addr = (void *)header + header->script_addrs_start;
20165b28aa31SSascha Hauer ram_code = (void *)header + header->ram_code_start;
20175b28aa31SSascha Hauer
20187560e3f3SSascha Hauer clk_enable(sdma->clk_ipg);
20197560e3f3SSascha Hauer clk_enable(sdma->clk_ahb);
20205b28aa31SSascha Hauer /* download the RAM image for SDMA */
20215b28aa31SSascha Hauer sdma_load_script(sdma, ram_code,
20225b28aa31SSascha Hauer header->ram_code_size,
20236866fd3bSSascha Hauer addr->ram_code_start_addr);
20247560e3f3SSascha Hauer clk_disable(sdma->clk_ipg);
20257560e3f3SSascha Hauer clk_disable(sdma->clk_ahb);
20265b28aa31SSascha Hauer
20275b28aa31SSascha Hauer sdma_add_scripts(sdma, addr);
20285b28aa31SSascha Hauer
2029e8fafa50SRobin Gong sdma->fw_loaded = true;
2030e8fafa50SRobin Gong
20315b28aa31SSascha Hauer dev_info(sdma->dev, "loaded firmware %d.%d\n",
20325b28aa31SSascha Hauer header->version_major,
20335b28aa31SSascha Hauer header->version_minor);
20345b28aa31SSascha Hauer
20355b28aa31SSascha Hauer err_firmware:
20365b28aa31SSascha Hauer release_firmware(fw);
20377b4b88e0SSascha Hauer }
20387b4b88e0SSascha Hauer
2039d078cd1bSZidan Wang #define EVENT_REMAP_CELLS 3
2040d078cd1bSZidan Wang
sdma_event_remap(struct sdma_engine * sdma)204129f493daSJason Liu static int sdma_event_remap(struct sdma_engine *sdma)
2042d078cd1bSZidan Wang {
2043d078cd1bSZidan Wang struct device_node *np = sdma->dev->of_node;
2044d078cd1bSZidan Wang struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
2045d078cd1bSZidan Wang struct property *event_remap;
2046d078cd1bSZidan Wang struct regmap *gpr;
2047d078cd1bSZidan Wang char propname[] = "fsl,sdma-event-remap";
2048d078cd1bSZidan Wang u32 reg, val, shift, num_map, i;
2049d078cd1bSZidan Wang int ret = 0;
2050d078cd1bSZidan Wang
20517104b9cbSMiaoqian Lin if (IS_ERR(np) || !gpr_np)
2052d078cd1bSZidan Wang goto out;
2053d078cd1bSZidan Wang
2054d078cd1bSZidan Wang event_remap = of_find_property(np, propname, NULL);
2055d078cd1bSZidan Wang num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
2056d078cd1bSZidan Wang if (!num_map) {
2057ce078af7SFabio Estevam dev_dbg(sdma->dev, "no event needs to be remapped\n");
2058d078cd1bSZidan Wang goto out;
2059d078cd1bSZidan Wang } else if (num_map % EVENT_REMAP_CELLS) {
2060d078cd1bSZidan Wang dev_err(sdma->dev, "the property %s must modulo %d\n",
2061d078cd1bSZidan Wang propname, EVENT_REMAP_CELLS);
2062d078cd1bSZidan Wang ret = -EINVAL;
2063d078cd1bSZidan Wang goto out;
2064d078cd1bSZidan Wang }
2065d078cd1bSZidan Wang
2066d078cd1bSZidan Wang gpr = syscon_node_to_regmap(gpr_np);
2067d078cd1bSZidan Wang if (IS_ERR(gpr)) {
2068d078cd1bSZidan Wang dev_err(sdma->dev, "failed to get gpr regmap\n");
2069d078cd1bSZidan Wang ret = PTR_ERR(gpr);
2070d078cd1bSZidan Wang goto out;
2071d078cd1bSZidan Wang }
2072d078cd1bSZidan Wang
2073d078cd1bSZidan Wang for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
2074d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i, ®);
2075d078cd1bSZidan Wang if (ret) {
2076d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n",
2077d078cd1bSZidan Wang propname, i);
2078d078cd1bSZidan Wang goto out;
2079d078cd1bSZidan Wang }
2080d078cd1bSZidan Wang
2081d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i + 1, &shift);
2082d078cd1bSZidan Wang if (ret) {
2083d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n",
2084d078cd1bSZidan Wang propname, i + 1);
2085d078cd1bSZidan Wang goto out;
2086d078cd1bSZidan Wang }
2087d078cd1bSZidan Wang
2088d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i + 2, &val);
2089d078cd1bSZidan Wang if (ret) {
2090d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n",
2091d078cd1bSZidan Wang propname, i + 2);
2092d078cd1bSZidan Wang goto out;
2093d078cd1bSZidan Wang }
2094d078cd1bSZidan Wang
2095d078cd1bSZidan Wang regmap_update_bits(gpr, reg, BIT(shift), val << shift);
2096d078cd1bSZidan Wang }
2097d078cd1bSZidan Wang
2098d078cd1bSZidan Wang out:
20997104b9cbSMiaoqian Lin if (gpr_np)
2100d078cd1bSZidan Wang of_node_put(gpr_np);
2101d078cd1bSZidan Wang
2102d078cd1bSZidan Wang return ret;
2103d078cd1bSZidan Wang }
2104d078cd1bSZidan Wang
sdma_get_firmware(struct sdma_engine * sdma,const char * fw_name)2105fe6cf289SArnd Bergmann static int sdma_get_firmware(struct sdma_engine *sdma,
21067b4b88e0SSascha Hauer const char *fw_name)
21077b4b88e0SSascha Hauer {
21087b4b88e0SSascha Hauer int ret;
21097b4b88e0SSascha Hauer
2110*ea00def5SLucas Stach ret = firmware_request_nowait_nowarn(THIS_MODULE, fw_name, sdma->dev,
21117b4b88e0SSascha Hauer GFP_KERNEL, sdma, sdma_load_firmware);
21125b28aa31SSascha Hauer
21135b28aa31SSascha Hauer return ret;
21145b28aa31SSascha Hauer }
21155b28aa31SSascha Hauer
sdma_init(struct sdma_engine * sdma)211619bfc772SJingoo Han static int sdma_init(struct sdma_engine *sdma)
21171ec1e82fSSascha Hauer {
21181ec1e82fSSascha Hauer int i, ret;
21191ec1e82fSSascha Hauer dma_addr_t ccb_phys;
2120802ef223SNicolin Chen int ccbsize;
21211ec1e82fSSascha Hauer
2122b93edcddSFabio Estevam ret = clk_enable(sdma->clk_ipg);
2123b93edcddSFabio Estevam if (ret)
2124b93edcddSFabio Estevam return ret;
2125b93edcddSFabio Estevam ret = clk_enable(sdma->clk_ahb);
2126b93edcddSFabio Estevam if (ret)
2127b93edcddSFabio Estevam goto disable_clk_ipg;
21281ec1e82fSSascha Hauer
2129941acd56SAngus Ainslie (Purism) if (sdma->drvdata->check_ratio &&
2130941acd56SAngus Ainslie (Purism) (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
213125aaa75dSAngus Ainslie (Purism) sdma->clk_ratio = 1;
213225aaa75dSAngus Ainslie (Purism)
21331ec1e82fSSascha Hauer /* Be sure SDMA has not started yet */
2134c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
21351ec1e82fSSascha Hauer
2136802ef223SNicolin Chen ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control)
2137802ef223SNicolin Chen + sizeof(struct sdma_context_data));
2138802ef223SNicolin Chen
2139802ef223SNicolin Chen if (sdma->iram_pool)
2140802ef223SNicolin Chen sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys);
2141802ef223SNicolin Chen else
2142802ef223SNicolin Chen sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, &ccb_phys,
2143802ef223SNicolin Chen GFP_KERNEL);
21441ec1e82fSSascha Hauer
21451ec1e82fSSascha Hauer if (!sdma->channel_control) {
21461ec1e82fSSascha Hauer ret = -ENOMEM;
21471ec1e82fSSascha Hauer goto err_dma_alloc;
21481ec1e82fSSascha Hauer }
21491ec1e82fSSascha Hauer
21501ec1e82fSSascha Hauer sdma->context = (void *)sdma->channel_control +
21511ec1e82fSSascha Hauer MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
21521ec1e82fSSascha Hauer sdma->context_phys = ccb_phys +
21531ec1e82fSSascha Hauer MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
21541ec1e82fSSascha Hauer
21551ec1e82fSSascha Hauer /* disable all channels */
215617bba72fSSascha Hauer for (i = 0; i < sdma->drvdata->num_events; i++)
2157c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
21581ec1e82fSSascha Hauer
21591ec1e82fSSascha Hauer /* All channels have priority 0 */
21601ec1e82fSSascha Hauer for (i = 0; i < MAX_DMA_CHANNELS; i++)
2161c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
21621ec1e82fSSascha Hauer
216357b772b8SRobin Gong ret = sdma_request_channel0(sdma);
21641ec1e82fSSascha Hauer if (ret)
21651ec1e82fSSascha Hauer goto err_dma_alloc;
21661ec1e82fSSascha Hauer
21671ec1e82fSSascha Hauer sdma_config_ownership(&sdma->channel[0], false, true, false);
21681ec1e82fSSascha Hauer
21691ec1e82fSSascha Hauer /* Set Command Channel (Channel Zero) */
2170c4b56857SRichard Zhao writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
21711ec1e82fSSascha Hauer
21721ec1e82fSSascha Hauer /* Set bits of CONFIG register but with static context switching */
217325aaa75dSAngus Ainslie (Purism) if (sdma->clk_ratio)
217425aaa75dSAngus Ainslie (Purism) writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
217525aaa75dSAngus Ainslie (Purism) else
2176c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
21771ec1e82fSSascha Hauer
2178c4b56857SRichard Zhao writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
21791ec1e82fSSascha Hauer
21801ec1e82fSSascha Hauer /* Initializes channel's priorities */
21811ec1e82fSSascha Hauer sdma_set_channel_priority(&sdma->channel[0], 7);
21821ec1e82fSSascha Hauer
21837560e3f3SSascha Hauer clk_disable(sdma->clk_ipg);
21847560e3f3SSascha Hauer clk_disable(sdma->clk_ahb);
21851ec1e82fSSascha Hauer
21861ec1e82fSSascha Hauer return 0;
21871ec1e82fSSascha Hauer
21881ec1e82fSSascha Hauer err_dma_alloc:
21897560e3f3SSascha Hauer clk_disable(sdma->clk_ahb);
2190b93edcddSFabio Estevam disable_clk_ipg:
2191b93edcddSFabio Estevam clk_disable(sdma->clk_ipg);
21921ec1e82fSSascha Hauer dev_err(sdma->dev, "initialisation failed with %d\n", ret);
21931ec1e82fSSascha Hauer return ret;
21941ec1e82fSSascha Hauer }
21951ec1e82fSSascha Hauer
sdma_filter_fn(struct dma_chan * chan,void * fn_param)21969479e17cSShawn Guo static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
21979479e17cSShawn Guo {
21980b351865SNicolin Chen struct sdma_channel *sdmac = to_sdma_chan(chan);
21999479e17cSShawn Guo struct imx_dma_data *data = fn_param;
22009479e17cSShawn Guo
22019479e17cSShawn Guo if (!imx_dma_is_general_purpose(chan))
22029479e17cSShawn Guo return false;
22039479e17cSShawn Guo
22040b351865SNicolin Chen sdmac->data = *data;
22050b351865SNicolin Chen chan->private = &sdmac->data;
22069479e17cSShawn Guo
22079479e17cSShawn Guo return true;
22089479e17cSShawn Guo }
22099479e17cSShawn Guo
sdma_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)22109479e17cSShawn Guo static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
22119479e17cSShawn Guo struct of_dma *ofdma)
22129479e17cSShawn Guo {
22139479e17cSShawn Guo struct sdma_engine *sdma = ofdma->of_dma_data;
22149479e17cSShawn Guo dma_cap_mask_t mask = sdma->dma_device.cap_mask;
22159479e17cSShawn Guo struct imx_dma_data data;
22169479e17cSShawn Guo
22179479e17cSShawn Guo if (dma_spec->args_count != 3)
22189479e17cSShawn Guo return NULL;
22199479e17cSShawn Guo
22209479e17cSShawn Guo data.dma_request = dma_spec->args[0];
22219479e17cSShawn Guo data.peripheral_type = dma_spec->args[1];
22229479e17cSShawn Guo data.priority = dma_spec->args[2];
22238391ecf4SShengjiu Wang /*
22248391ecf4SShengjiu Wang * init dma_request2 to zero, which is not used by the dts.
22258391ecf4SShengjiu Wang * For P2P, dma_request2 is init from dma_request_channel(),
22268391ecf4SShengjiu Wang * chan->private will point to the imx_dma_data, and in
22278391ecf4SShengjiu Wang * device_alloc_chan_resources(), imx_dma_data.dma_request2 will
22288391ecf4SShengjiu Wang * be set to sdmac->event_id1.
22298391ecf4SShengjiu Wang */
22308391ecf4SShengjiu Wang data.dma_request2 = 0;
22319479e17cSShawn Guo
2232990c0b53SBaolin Wang return __dma_request_channel(&mask, sdma_filter_fn, &data,
2233990c0b53SBaolin Wang ofdma->of_node);
22349479e17cSShawn Guo }
22359479e17cSShawn Guo
sdma_probe(struct platform_device * pdev)2236e34b731fSMark Brown static int sdma_probe(struct platform_device *pdev)
22371ec1e82fSSascha Hauer {
2238580975d7SShawn Guo struct device_node *np = pdev->dev.of_node;
22398391ecf4SShengjiu Wang struct device_node *spba_bus;
2240580975d7SShawn Guo const char *fw_name;
22411ec1e82fSSascha Hauer int ret;
22421ec1e82fSSascha Hauer int irq;
22438391ecf4SShengjiu Wang struct resource spba_res;
22441ec1e82fSSascha Hauer int i;
22451ec1e82fSSascha Hauer struct sdma_engine *sdma;
224636e2f21aSSascha Hauer s32 *saddr_arr;
22471ec1e82fSSascha Hauer
224842536b9fSPhilippe Retornaz ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
224942536b9fSPhilippe Retornaz if (ret)
225042536b9fSPhilippe Retornaz return ret;
225142536b9fSPhilippe Retornaz
22527f24e0eeSFabio Estevam sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
22531ec1e82fSSascha Hauer if (!sdma)
22541ec1e82fSSascha Hauer return -ENOMEM;
22551ec1e82fSSascha Hauer
22562ccaef05SRichard Zhao spin_lock_init(&sdma->channel_0_lock);
225773eab978SSascha Hauer
22581ec1e82fSSascha Hauer sdma->dev = &pdev->dev;
225932996419SFabio Estevam sdma->drvdata = of_device_get_match_data(sdma->dev);
22601ec1e82fSSascha Hauer
22611ec1e82fSSascha Hauer irq = platform_get_irq(pdev, 0);
22627f24e0eeSFabio Estevam if (irq < 0)
226363c72e02SFabio Estevam return irq;
22641ec1e82fSSascha Hauer
22654b23603aSTudor Ambarus sdma->regs = devm_platform_ioremap_resource(pdev, 0);
22667f24e0eeSFabio Estevam if (IS_ERR(sdma->regs))
22677f24e0eeSFabio Estevam return PTR_ERR(sdma->regs);
22681ec1e82fSSascha Hauer
22697560e3f3SSascha Hauer sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
22707f24e0eeSFabio Estevam if (IS_ERR(sdma->clk_ipg))
22717f24e0eeSFabio Estevam return PTR_ERR(sdma->clk_ipg);
22721ec1e82fSSascha Hauer
22737560e3f3SSascha Hauer sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
22747f24e0eeSFabio Estevam if (IS_ERR(sdma->clk_ahb))
22757f24e0eeSFabio Estevam return PTR_ERR(sdma->clk_ahb);
22767560e3f3SSascha Hauer
2277fb9caf37SArvind Yadav ret = clk_prepare(sdma->clk_ipg);
2278fb9caf37SArvind Yadav if (ret)
2279fb9caf37SArvind Yadav return ret;
2280fb9caf37SArvind Yadav
2281fb9caf37SArvind Yadav ret = clk_prepare(sdma->clk_ahb);
2282fb9caf37SArvind Yadav if (ret)
2283fb9caf37SArvind Yadav goto err_clk;
22847560e3f3SSascha Hauer
22850951a90eSFabio Estevam ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0,
22860951a90eSFabio Estevam dev_name(&pdev->dev), sdma);
22871ec1e82fSSascha Hauer if (ret)
2288fb9caf37SArvind Yadav goto err_irq;
22891ec1e82fSSascha Hauer
22905bb9dbb5SVinod Koul sdma->irq = irq;
22915bb9dbb5SVinod Koul
22925b28aa31SSascha Hauer sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
2293fb9caf37SArvind Yadav if (!sdma->script_addrs) {
2294fb9caf37SArvind Yadav ret = -ENOMEM;
2295fb9caf37SArvind Yadav goto err_irq;
2296fb9caf37SArvind Yadav }
22971ec1e82fSSascha Hauer
229836e2f21aSSascha Hauer /* initially no scripts available */
229936e2f21aSSascha Hauer saddr_arr = (s32 *)sdma->script_addrs;
2300be4cf718SSascha Hauer for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
230136e2f21aSSascha Hauer saddr_arr[i] = -EINVAL;
230236e2f21aSSascha Hauer
23037214a8b1SSascha Hauer dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
23047214a8b1SSascha Hauer dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
23050f06c027SRobin Gong dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
23068d1b7bd5SKai-Heng Feng dma_cap_set(DMA_PRIVATE, sdma->dma_device.cap_mask);
23077214a8b1SSascha Hauer
23081ec1e82fSSascha Hauer INIT_LIST_HEAD(&sdma->dma_device.channels);
23091ec1e82fSSascha Hauer /* Initialize channel parameters */
23101ec1e82fSSascha Hauer for (i = 0; i < MAX_DMA_CHANNELS; i++) {
23111ec1e82fSSascha Hauer struct sdma_channel *sdmac = &sdma->channel[i];
23121ec1e82fSSascha Hauer
23131ec1e82fSSascha Hauer sdmac->sdma = sdma;
23141ec1e82fSSascha Hauer
23151ec1e82fSSascha Hauer sdmac->channel = i;
231657b772b8SRobin Gong sdmac->vc.desc_free = sdma_desc_free;
23174e2b10beSRobin Gong INIT_LIST_HEAD(&sdmac->terminated);
2318b8603d2aSLucas Stach INIT_WORK(&sdmac->terminate_worker,
2319b8603d2aSLucas Stach sdma_channel_terminate_work);
232023889c63SSascha Hauer /*
232123889c63SSascha Hauer * Add the channel to the DMAC list. Do not add channel 0 though
232223889c63SSascha Hauer * because we need it internally in the SDMA driver. This also means
232323889c63SSascha Hauer * that channel 0 in dmaengine counting matches sdma channel 1.
232423889c63SSascha Hauer */
232523889c63SSascha Hauer if (i)
232657b772b8SRobin Gong vchan_init(&sdmac->vc, &sdma->dma_device);
23271ec1e82fSSascha Hauer }
23281ec1e82fSSascha Hauer
2329802ef223SNicolin Chen if (np) {
2330802ef223SNicolin Chen sdma->iram_pool = of_gen_pool_get(np, "iram", 0);
2331802ef223SNicolin Chen if (sdma->iram_pool)
2332802ef223SNicolin Chen dev_info(&pdev->dev, "alloc bd from iram.\n");
2333802ef223SNicolin Chen }
2334802ef223SNicolin Chen
23355b28aa31SSascha Hauer ret = sdma_init(sdma);
23361ec1e82fSSascha Hauer if (ret)
23371ec1e82fSSascha Hauer goto err_init;
23381ec1e82fSSascha Hauer
2339d078cd1bSZidan Wang ret = sdma_event_remap(sdma);
2340d078cd1bSZidan Wang if (ret)
2341d078cd1bSZidan Wang goto err_init;
2342d078cd1bSZidan Wang
2343dcfec3c0SSascha Hauer if (sdma->drvdata->script_addrs)
2344dcfec3c0SSascha Hauer sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
23455b28aa31SSascha Hauer
23461ec1e82fSSascha Hauer sdma->dma_device.dev = &pdev->dev;
23471ec1e82fSSascha Hauer
23481ec1e82fSSascha Hauer sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
23491ec1e82fSSascha Hauer sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
23501ec1e82fSSascha Hauer sdma->dma_device.device_tx_status = sdma_tx_status;
23511ec1e82fSSascha Hauer sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
23521ec1e82fSSascha Hauer sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
23537b350ab0SMaxime Ripard sdma->dma_device.device_config = sdma_config;
2354a80f2787SSascha Hauer sdma->dma_device.device_terminate_all = sdma_terminate_all;
2355b8603d2aSLucas Stach sdma->dma_device.device_synchronize = sdma_channel_synchronize;
2356f9d4a398SNicolin Chen sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
2357f9d4a398SNicolin Chen sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
2358f9d4a398SNicolin Chen sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
23596f3125ceSLucas Stach sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
23600f06c027SRobin Gong sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
23611ec1e82fSSascha Hauer sdma->dma_device.device_issue_pending = sdma_issue_pending;
2362a3711d49SAngus Ainslie (Purism) sdma->dma_device.copy_align = 2;
23634a6b2e8aSRobin Gong dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
23641ec1e82fSSascha Hauer
236523e11811SVignesh Raman platform_set_drvdata(pdev, sdma);
236623e11811SVignesh Raman
23671ec1e82fSSascha Hauer ret = dma_async_device_register(&sdma->dma_device);
23681ec1e82fSSascha Hauer if (ret) {
23691ec1e82fSSascha Hauer dev_err(&pdev->dev, "unable to register\n");
23701ec1e82fSSascha Hauer goto err_init;
23711ec1e82fSSascha Hauer }
23721ec1e82fSSascha Hauer
23739479e17cSShawn Guo if (np) {
23749479e17cSShawn Guo ret = of_dma_controller_register(np, sdma_xlate, sdma);
23759479e17cSShawn Guo if (ret) {
23769479e17cSShawn Guo dev_err(&pdev->dev, "failed to register controller\n");
23779479e17cSShawn Guo goto err_register;
23789479e17cSShawn Guo }
23798391ecf4SShengjiu Wang
23808391ecf4SShengjiu Wang spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
23818391ecf4SShengjiu Wang ret = of_address_to_resource(spba_bus, 0, &spba_res);
23828391ecf4SShengjiu Wang if (!ret) {
23838391ecf4SShengjiu Wang sdma->spba_start_addr = spba_res.start;
23848391ecf4SShengjiu Wang sdma->spba_end_addr = spba_res.end;
23858391ecf4SShengjiu Wang }
23868391ecf4SShengjiu Wang of_node_put(spba_bus);
23879479e17cSShawn Guo }
23889479e17cSShawn Guo
23892b8066c3SSven Van Asbroeck /*
23902b8066c3SSven Van Asbroeck * Because that device tree does not encode ROM script address,
23912b8066c3SSven Van Asbroeck * the RAM script in firmware is mandatory for device tree
23922b8066c3SSven Van Asbroeck * probe, otherwise it fails.
23932b8066c3SSven Van Asbroeck */
23942b8066c3SSven Van Asbroeck ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
23952b8066c3SSven Van Asbroeck &fw_name);
23962b8066c3SSven Van Asbroeck if (ret) {
23972b8066c3SSven Van Asbroeck dev_warn(&pdev->dev, "failed to get firmware name\n");
23982b8066c3SSven Van Asbroeck } else {
23992b8066c3SSven Van Asbroeck ret = sdma_get_firmware(sdma, fw_name);
24002b8066c3SSven Van Asbroeck if (ret)
24012b8066c3SSven Van Asbroeck dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
24022b8066c3SSven Van Asbroeck }
24032b8066c3SSven Van Asbroeck
24041ec1e82fSSascha Hauer return 0;
24051ec1e82fSSascha Hauer
24069479e17cSShawn Guo err_register:
24079479e17cSShawn Guo dma_async_device_unregister(&sdma->dma_device);
24081ec1e82fSSascha Hauer err_init:
24091ec1e82fSSascha Hauer kfree(sdma->script_addrs);
2410fb9caf37SArvind Yadav err_irq:
2411fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ahb);
2412fb9caf37SArvind Yadav err_clk:
2413fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ipg);
2414939fd4f0SShawn Guo return ret;
24151ec1e82fSSascha Hauer }
24161ec1e82fSSascha Hauer
sdma_remove(struct platform_device * pdev)241706e4f653SUwe Kleine-König static void sdma_remove(struct platform_device *pdev)
24181ec1e82fSSascha Hauer {
241923e11811SVignesh Raman struct sdma_engine *sdma = platform_get_drvdata(pdev);
2420c12fe497SVignesh Raman int i;
242123e11811SVignesh Raman
24225bb9dbb5SVinod Koul devm_free_irq(&pdev->dev, sdma->irq, sdma);
242323e11811SVignesh Raman dma_async_device_unregister(&sdma->dma_device);
242423e11811SVignesh Raman kfree(sdma->script_addrs);
2425fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ahb);
2426fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ipg);
2427c12fe497SVignesh Raman /* Kill the tasklet */
2428c12fe497SVignesh Raman for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2429c12fe497SVignesh Raman struct sdma_channel *sdmac = &sdma->channel[i];
2430c12fe497SVignesh Raman
243157b772b8SRobin Gong tasklet_kill(&sdmac->vc.task);
243257b772b8SRobin Gong sdma_free_chan_resources(&sdmac->vc.chan);
2433c12fe497SVignesh Raman }
243423e11811SVignesh Raman
243523e11811SVignesh Raman platform_set_drvdata(pdev, NULL);
24361ec1e82fSSascha Hauer }
24371ec1e82fSSascha Hauer
24381ec1e82fSSascha Hauer static struct platform_driver sdma_driver = {
24391ec1e82fSSascha Hauer .driver = {
24401ec1e82fSSascha Hauer .name = "imx-sdma",
2441580975d7SShawn Guo .of_match_table = sdma_dt_ids,
24421ec1e82fSSascha Hauer },
244306e4f653SUwe Kleine-König .remove_new = sdma_remove,
244423e11811SVignesh Raman .probe = sdma_probe,
24451ec1e82fSSascha Hauer };
24461ec1e82fSSascha Hauer
244723e11811SVignesh Raman module_platform_driver(sdma_driver);
24481ec1e82fSSascha Hauer
24491ec1e82fSSascha Hauer MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
24501ec1e82fSSascha Hauer MODULE_DESCRIPTION("i.MX SDMA driver");
2451c0879342SNicolas Chauvet #if IS_ENABLED(CONFIG_SOC_IMX6Q)
2452c0879342SNicolas Chauvet MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
2453c0879342SNicolas Chauvet #endif
2454a7cd3cf0SPeter Robinson #if IS_ENABLED(CONFIG_SOC_IMX7D) || IS_ENABLED(CONFIG_SOC_IMX8M)
2455c0879342SNicolas Chauvet MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
2456c0879342SNicolas Chauvet #endif
24571ec1e82fSSascha Hauer MODULE_LICENSE("GPL");
2458