xref: /linux/drivers/dma/imx-sdma.c (revision 63c72e028a1dd18515c356834d3f9cfce1cd3f51)
11ec1e82fSSascha Hauer /*
21ec1e82fSSascha Hauer  * drivers/dma/imx-sdma.c
31ec1e82fSSascha Hauer  *
41ec1e82fSSascha Hauer  * This file contains a driver for the Freescale Smart DMA engine
51ec1e82fSSascha Hauer  *
61ec1e82fSSascha Hauer  * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
71ec1e82fSSascha Hauer  *
81ec1e82fSSascha Hauer  * Based on code from Freescale:
91ec1e82fSSascha Hauer  *
101ec1e82fSSascha Hauer  * Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
111ec1e82fSSascha Hauer  *
121ec1e82fSSascha Hauer  * The code contained herein is licensed under the GNU General Public
131ec1e82fSSascha Hauer  * License. You may obtain a copy of the GNU General Public License
141ec1e82fSSascha Hauer  * Version 2 or later at the following locations:
151ec1e82fSSascha Hauer  *
161ec1e82fSSascha Hauer  * http://www.opensource.org/licenses/gpl-license.html
171ec1e82fSSascha Hauer  * http://www.gnu.org/copyleft/gpl.html
181ec1e82fSSascha Hauer  */
191ec1e82fSSascha Hauer 
201ec1e82fSSascha Hauer #include <linux/init.h>
21f8de8f4cSAxel Lin #include <linux/module.h>
221ec1e82fSSascha Hauer #include <linux/types.h>
230bbc1413SRichard Zhao #include <linux/bitops.h>
241ec1e82fSSascha Hauer #include <linux/mm.h>
251ec1e82fSSascha Hauer #include <linux/interrupt.h>
261ec1e82fSSascha Hauer #include <linux/clk.h>
272ccaef05SRichard Zhao #include <linux/delay.h>
281ec1e82fSSascha Hauer #include <linux/sched.h>
291ec1e82fSSascha Hauer #include <linux/semaphore.h>
301ec1e82fSSascha Hauer #include <linux/spinlock.h>
311ec1e82fSSascha Hauer #include <linux/device.h>
321ec1e82fSSascha Hauer #include <linux/dma-mapping.h>
331ec1e82fSSascha Hauer #include <linux/firmware.h>
341ec1e82fSSascha Hauer #include <linux/slab.h>
351ec1e82fSSascha Hauer #include <linux/platform_device.h>
361ec1e82fSSascha Hauer #include <linux/dmaengine.h>
37580975d7SShawn Guo #include <linux/of.h>
38580975d7SShawn Guo #include <linux/of_device.h>
399479e17cSShawn Guo #include <linux/of_dma.h>
401ec1e82fSSascha Hauer 
411ec1e82fSSascha Hauer #include <asm/irq.h>
4282906b13SArnd Bergmann #include <linux/platform_data/dma-imx-sdma.h>
4382906b13SArnd Bergmann #include <linux/platform_data/dma-imx.h>
441ec1e82fSSascha Hauer 
45d2ebfb33SRussell King - ARM Linux #include "dmaengine.h"
46d2ebfb33SRussell King - ARM Linux 
471ec1e82fSSascha Hauer /* SDMA registers */
481ec1e82fSSascha Hauer #define SDMA_H_C0PTR		0x000
491ec1e82fSSascha Hauer #define SDMA_H_INTR		0x004
501ec1e82fSSascha Hauer #define SDMA_H_STATSTOP		0x008
511ec1e82fSSascha Hauer #define SDMA_H_START		0x00c
521ec1e82fSSascha Hauer #define SDMA_H_EVTOVR		0x010
531ec1e82fSSascha Hauer #define SDMA_H_DSPOVR		0x014
541ec1e82fSSascha Hauer #define SDMA_H_HOSTOVR		0x018
551ec1e82fSSascha Hauer #define SDMA_H_EVTPEND		0x01c
561ec1e82fSSascha Hauer #define SDMA_H_DSPENBL		0x020
571ec1e82fSSascha Hauer #define SDMA_H_RESET		0x024
581ec1e82fSSascha Hauer #define SDMA_H_EVTERR		0x028
591ec1e82fSSascha Hauer #define SDMA_H_INTRMSK		0x02c
601ec1e82fSSascha Hauer #define SDMA_H_PSW		0x030
611ec1e82fSSascha Hauer #define SDMA_H_EVTERRDBG	0x034
621ec1e82fSSascha Hauer #define SDMA_H_CONFIG		0x038
631ec1e82fSSascha Hauer #define SDMA_ONCE_ENB		0x040
641ec1e82fSSascha Hauer #define SDMA_ONCE_DATA		0x044
651ec1e82fSSascha Hauer #define SDMA_ONCE_INSTR		0x048
661ec1e82fSSascha Hauer #define SDMA_ONCE_STAT		0x04c
671ec1e82fSSascha Hauer #define SDMA_ONCE_CMD		0x050
681ec1e82fSSascha Hauer #define SDMA_EVT_MIRROR		0x054
691ec1e82fSSascha Hauer #define SDMA_ILLINSTADDR	0x058
701ec1e82fSSascha Hauer #define SDMA_CHN0ADDR		0x05c
711ec1e82fSSascha Hauer #define SDMA_ONCE_RTB		0x060
721ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF1	0x070
731ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF2	0x074
7462550cd7SShawn Guo #define SDMA_CHNENBL0_IMX35	0x200
7562550cd7SShawn Guo #define SDMA_CHNENBL0_IMX31	0x080
761ec1e82fSSascha Hauer #define SDMA_CHNPRI_0		0x100
771ec1e82fSSascha Hauer 
781ec1e82fSSascha Hauer /*
791ec1e82fSSascha Hauer  * Buffer descriptor status values.
801ec1e82fSSascha Hauer  */
811ec1e82fSSascha Hauer #define BD_DONE  0x01
821ec1e82fSSascha Hauer #define BD_WRAP  0x02
831ec1e82fSSascha Hauer #define BD_CONT  0x04
841ec1e82fSSascha Hauer #define BD_INTR  0x08
851ec1e82fSSascha Hauer #define BD_RROR  0x10
861ec1e82fSSascha Hauer #define BD_LAST  0x20
871ec1e82fSSascha Hauer #define BD_EXTD  0x80
881ec1e82fSSascha Hauer 
891ec1e82fSSascha Hauer /*
901ec1e82fSSascha Hauer  * Data Node descriptor status values.
911ec1e82fSSascha Hauer  */
921ec1e82fSSascha Hauer #define DND_END_OF_FRAME  0x80
931ec1e82fSSascha Hauer #define DND_END_OF_XFER   0x40
941ec1e82fSSascha Hauer #define DND_DONE          0x20
951ec1e82fSSascha Hauer #define DND_UNUSED        0x01
961ec1e82fSSascha Hauer 
971ec1e82fSSascha Hauer /*
981ec1e82fSSascha Hauer  * IPCV2 descriptor status values.
991ec1e82fSSascha Hauer  */
1001ec1e82fSSascha Hauer #define BD_IPCV2_END_OF_FRAME  0x40
1011ec1e82fSSascha Hauer 
1021ec1e82fSSascha Hauer #define IPCV2_MAX_NODES        50
1031ec1e82fSSascha Hauer /*
1041ec1e82fSSascha Hauer  * Error bit set in the CCB status field by the SDMA,
1051ec1e82fSSascha Hauer  * in setbd routine, in case of a transfer error
1061ec1e82fSSascha Hauer  */
1071ec1e82fSSascha Hauer #define DATA_ERROR  0x10000000
1081ec1e82fSSascha Hauer 
1091ec1e82fSSascha Hauer /*
1101ec1e82fSSascha Hauer  * Buffer descriptor commands.
1111ec1e82fSSascha Hauer  */
1121ec1e82fSSascha Hauer #define C0_ADDR             0x01
1131ec1e82fSSascha Hauer #define C0_LOAD             0x02
1141ec1e82fSSascha Hauer #define C0_DUMP             0x03
1151ec1e82fSSascha Hauer #define C0_SETCTX           0x07
1161ec1e82fSSascha Hauer #define C0_GETCTX           0x03
1171ec1e82fSSascha Hauer #define C0_SETDM            0x01
1181ec1e82fSSascha Hauer #define C0_SETPM            0x04
1191ec1e82fSSascha Hauer #define C0_GETDM            0x02
1201ec1e82fSSascha Hauer #define C0_GETPM            0x08
1211ec1e82fSSascha Hauer /*
1221ec1e82fSSascha Hauer  * Change endianness indicator in the BD command field
1231ec1e82fSSascha Hauer  */
1241ec1e82fSSascha Hauer #define CHANGE_ENDIANNESS   0x80
1251ec1e82fSSascha Hauer 
1261ec1e82fSSascha Hauer /*
1271ec1e82fSSascha Hauer  * Mode/Count of data node descriptors - IPCv2
1281ec1e82fSSascha Hauer  */
1291ec1e82fSSascha Hauer struct sdma_mode_count {
1301ec1e82fSSascha Hauer 	u32 count   : 16; /* size of the buffer pointed by this BD */
1311ec1e82fSSascha Hauer 	u32 status  :  8; /* E,R,I,C,W,D status bits stored here */
1321ec1e82fSSascha Hauer 	u32 command :  8; /* command mostlky used for channel 0 */
1331ec1e82fSSascha Hauer };
1341ec1e82fSSascha Hauer 
1351ec1e82fSSascha Hauer /*
1361ec1e82fSSascha Hauer  * Buffer descriptor
1371ec1e82fSSascha Hauer  */
1381ec1e82fSSascha Hauer struct sdma_buffer_descriptor {
1391ec1e82fSSascha Hauer 	struct sdma_mode_count  mode;
1401ec1e82fSSascha Hauer 	u32 buffer_addr;	/* address of the buffer described */
1411ec1e82fSSascha Hauer 	u32 ext_buffer_addr;	/* extended buffer address */
1421ec1e82fSSascha Hauer } __attribute__ ((packed));
1431ec1e82fSSascha Hauer 
1441ec1e82fSSascha Hauer /**
1451ec1e82fSSascha Hauer  * struct sdma_channel_control - Channel control Block
1461ec1e82fSSascha Hauer  *
1471ec1e82fSSascha Hauer  * @current_bd_ptr	current buffer descriptor processed
1481ec1e82fSSascha Hauer  * @base_bd_ptr		first element of buffer descriptor array
1491ec1e82fSSascha Hauer  * @unused		padding. The SDMA engine expects an array of 128 byte
1501ec1e82fSSascha Hauer  *			control blocks
1511ec1e82fSSascha Hauer  */
1521ec1e82fSSascha Hauer struct sdma_channel_control {
1531ec1e82fSSascha Hauer 	u32 current_bd_ptr;
1541ec1e82fSSascha Hauer 	u32 base_bd_ptr;
1551ec1e82fSSascha Hauer 	u32 unused[2];
1561ec1e82fSSascha Hauer } __attribute__ ((packed));
1571ec1e82fSSascha Hauer 
1581ec1e82fSSascha Hauer /**
1591ec1e82fSSascha Hauer  * struct sdma_state_registers - SDMA context for a channel
1601ec1e82fSSascha Hauer  *
1611ec1e82fSSascha Hauer  * @pc:		program counter
1621ec1e82fSSascha Hauer  * @t:		test bit: status of arithmetic & test instruction
1631ec1e82fSSascha Hauer  * @rpc:	return program counter
1641ec1e82fSSascha Hauer  * @sf:		source fault while loading data
1651ec1e82fSSascha Hauer  * @spc:	loop start program counter
1661ec1e82fSSascha Hauer  * @df:		destination fault while storing data
1671ec1e82fSSascha Hauer  * @epc:	loop end program counter
1681ec1e82fSSascha Hauer  * @lm:		loop mode
1691ec1e82fSSascha Hauer  */
1701ec1e82fSSascha Hauer struct sdma_state_registers {
1711ec1e82fSSascha Hauer 	u32 pc     :14;
1721ec1e82fSSascha Hauer 	u32 unused1: 1;
1731ec1e82fSSascha Hauer 	u32 t      : 1;
1741ec1e82fSSascha Hauer 	u32 rpc    :14;
1751ec1e82fSSascha Hauer 	u32 unused0: 1;
1761ec1e82fSSascha Hauer 	u32 sf     : 1;
1771ec1e82fSSascha Hauer 	u32 spc    :14;
1781ec1e82fSSascha Hauer 	u32 unused2: 1;
1791ec1e82fSSascha Hauer 	u32 df     : 1;
1801ec1e82fSSascha Hauer 	u32 epc    :14;
1811ec1e82fSSascha Hauer 	u32 lm     : 2;
1821ec1e82fSSascha Hauer } __attribute__ ((packed));
1831ec1e82fSSascha Hauer 
1841ec1e82fSSascha Hauer /**
1851ec1e82fSSascha Hauer  * struct sdma_context_data - sdma context specific to a channel
1861ec1e82fSSascha Hauer  *
1871ec1e82fSSascha Hauer  * @channel_state:	channel state bits
1881ec1e82fSSascha Hauer  * @gReg:		general registers
1891ec1e82fSSascha Hauer  * @mda:		burst dma destination address register
1901ec1e82fSSascha Hauer  * @msa:		burst dma source address register
1911ec1e82fSSascha Hauer  * @ms:			burst dma status register
1921ec1e82fSSascha Hauer  * @md:			burst dma data register
1931ec1e82fSSascha Hauer  * @pda:		peripheral dma destination address register
1941ec1e82fSSascha Hauer  * @psa:		peripheral dma source address register
1951ec1e82fSSascha Hauer  * @ps:			peripheral dma status register
1961ec1e82fSSascha Hauer  * @pd:			peripheral dma data register
1971ec1e82fSSascha Hauer  * @ca:			CRC polynomial register
1981ec1e82fSSascha Hauer  * @cs:			CRC accumulator register
1991ec1e82fSSascha Hauer  * @dda:		dedicated core destination address register
2001ec1e82fSSascha Hauer  * @dsa:		dedicated core source address register
2011ec1e82fSSascha Hauer  * @ds:			dedicated core status register
2021ec1e82fSSascha Hauer  * @dd:			dedicated core data register
2031ec1e82fSSascha Hauer  */
2041ec1e82fSSascha Hauer struct sdma_context_data {
2051ec1e82fSSascha Hauer 	struct sdma_state_registers  channel_state;
2061ec1e82fSSascha Hauer 	u32  gReg[8];
2071ec1e82fSSascha Hauer 	u32  mda;
2081ec1e82fSSascha Hauer 	u32  msa;
2091ec1e82fSSascha Hauer 	u32  ms;
2101ec1e82fSSascha Hauer 	u32  md;
2111ec1e82fSSascha Hauer 	u32  pda;
2121ec1e82fSSascha Hauer 	u32  psa;
2131ec1e82fSSascha Hauer 	u32  ps;
2141ec1e82fSSascha Hauer 	u32  pd;
2151ec1e82fSSascha Hauer 	u32  ca;
2161ec1e82fSSascha Hauer 	u32  cs;
2171ec1e82fSSascha Hauer 	u32  dda;
2181ec1e82fSSascha Hauer 	u32  dsa;
2191ec1e82fSSascha Hauer 	u32  ds;
2201ec1e82fSSascha Hauer 	u32  dd;
2211ec1e82fSSascha Hauer 	u32  scratch0;
2221ec1e82fSSascha Hauer 	u32  scratch1;
2231ec1e82fSSascha Hauer 	u32  scratch2;
2241ec1e82fSSascha Hauer 	u32  scratch3;
2251ec1e82fSSascha Hauer 	u32  scratch4;
2261ec1e82fSSascha Hauer 	u32  scratch5;
2271ec1e82fSSascha Hauer 	u32  scratch6;
2281ec1e82fSSascha Hauer 	u32  scratch7;
2291ec1e82fSSascha Hauer } __attribute__ ((packed));
2301ec1e82fSSascha Hauer 
2311ec1e82fSSascha Hauer #define NUM_BD (int)(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor))
2321ec1e82fSSascha Hauer 
2331ec1e82fSSascha Hauer struct sdma_engine;
2341ec1e82fSSascha Hauer 
2351ec1e82fSSascha Hauer /**
2361ec1e82fSSascha Hauer  * struct sdma_channel - housekeeping for a SDMA channel
2371ec1e82fSSascha Hauer  *
2381ec1e82fSSascha Hauer  * @sdma		pointer to the SDMA engine for this channel
23923889c63SSascha Hauer  * @channel		the channel number, matches dmaengine chan_id + 1
2401ec1e82fSSascha Hauer  * @direction		transfer type. Needed for setting SDMA script
2411ec1e82fSSascha Hauer  * @peripheral_type	Peripheral type. Needed for setting SDMA script
2421ec1e82fSSascha Hauer  * @event_id0		aka dma request line
2431ec1e82fSSascha Hauer  * @event_id1		for channels that use 2 events
2441ec1e82fSSascha Hauer  * @word_size		peripheral access size
2451ec1e82fSSascha Hauer  * @buf_tail		ID of the buffer that was processed
2461ec1e82fSSascha Hauer  * @num_bd		max NUM_BD. number of descriptors currently handling
2471ec1e82fSSascha Hauer  */
2481ec1e82fSSascha Hauer struct sdma_channel {
2491ec1e82fSSascha Hauer 	struct sdma_engine		*sdma;
2501ec1e82fSSascha Hauer 	unsigned int			channel;
251db8196dfSVinod Koul 	enum dma_transfer_direction		direction;
2521ec1e82fSSascha Hauer 	enum sdma_peripheral_type	peripheral_type;
2531ec1e82fSSascha Hauer 	unsigned int			event_id0;
2541ec1e82fSSascha Hauer 	unsigned int			event_id1;
2551ec1e82fSSascha Hauer 	enum dma_slave_buswidth		word_size;
2561ec1e82fSSascha Hauer 	unsigned int			buf_tail;
2571ec1e82fSSascha Hauer 	unsigned int			num_bd;
258d1a792f3SRussell King - ARM Linux 	unsigned int			period_len;
2591ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor	*bd;
2601ec1e82fSSascha Hauer 	dma_addr_t			bd_phys;
2611ec1e82fSSascha Hauer 	unsigned int			pc_from_device, pc_to_device;
2621ec1e82fSSascha Hauer 	unsigned long			flags;
2631ec1e82fSSascha Hauer 	dma_addr_t			per_address;
2640bbc1413SRichard Zhao 	unsigned long			event_mask[2];
2650bbc1413SRichard Zhao 	unsigned long			watermark_level;
2661ec1e82fSSascha Hauer 	u32				shp_addr, per_addr;
2671ec1e82fSSascha Hauer 	struct dma_chan			chan;
2681ec1e82fSSascha Hauer 	spinlock_t			lock;
2691ec1e82fSSascha Hauer 	struct dma_async_tx_descriptor	desc;
2701ec1e82fSSascha Hauer 	enum dma_status			status;
271ab59a510SHuang Shijie 	unsigned int			chn_count;
272ab59a510SHuang Shijie 	unsigned int			chn_real_count;
273abd9ccc8SHuang Shijie 	struct tasklet_struct		tasklet;
2740b351865SNicolin Chen 	struct imx_dma_data		data;
2751ec1e82fSSascha Hauer };
2761ec1e82fSSascha Hauer 
2770bbc1413SRichard Zhao #define IMX_DMA_SG_LOOP		BIT(0)
2781ec1e82fSSascha Hauer 
2791ec1e82fSSascha Hauer #define MAX_DMA_CHANNELS 32
2801ec1e82fSSascha Hauer #define MXC_SDMA_DEFAULT_PRIORITY 1
2811ec1e82fSSascha Hauer #define MXC_SDMA_MIN_PRIORITY 1
2821ec1e82fSSascha Hauer #define MXC_SDMA_MAX_PRIORITY 7
2831ec1e82fSSascha Hauer 
2841ec1e82fSSascha Hauer #define SDMA_FIRMWARE_MAGIC 0x414d4453
2851ec1e82fSSascha Hauer 
2861ec1e82fSSascha Hauer /**
2871ec1e82fSSascha Hauer  * struct sdma_firmware_header - Layout of the firmware image
2881ec1e82fSSascha Hauer  *
2891ec1e82fSSascha Hauer  * @magic		"SDMA"
2901ec1e82fSSascha Hauer  * @version_major	increased whenever layout of struct sdma_script_start_addrs
2911ec1e82fSSascha Hauer  *			changes.
2921ec1e82fSSascha Hauer  * @version_minor	firmware minor version (for binary compatible changes)
2931ec1e82fSSascha Hauer  * @script_addrs_start	offset of struct sdma_script_start_addrs in this image
2941ec1e82fSSascha Hauer  * @num_script_addrs	Number of script addresses in this image
2951ec1e82fSSascha Hauer  * @ram_code_start	offset of SDMA ram image in this firmware image
2961ec1e82fSSascha Hauer  * @ram_code_size	size of SDMA ram image
2971ec1e82fSSascha Hauer  * @script_addrs	Stores the start address of the SDMA scripts
2981ec1e82fSSascha Hauer  *			(in SDMA memory space)
2991ec1e82fSSascha Hauer  */
3001ec1e82fSSascha Hauer struct sdma_firmware_header {
3011ec1e82fSSascha Hauer 	u32	magic;
3021ec1e82fSSascha Hauer 	u32	version_major;
3031ec1e82fSSascha Hauer 	u32	version_minor;
3041ec1e82fSSascha Hauer 	u32	script_addrs_start;
3051ec1e82fSSascha Hauer 	u32	num_script_addrs;
3061ec1e82fSSascha Hauer 	u32	ram_code_start;
3071ec1e82fSSascha Hauer 	u32	ram_code_size;
3081ec1e82fSSascha Hauer };
3091ec1e82fSSascha Hauer 
31017bba72fSSascha Hauer struct sdma_driver_data {
31117bba72fSSascha Hauer 	int chnenbl0;
31217bba72fSSascha Hauer 	int num_events;
313dcfec3c0SSascha Hauer 	struct sdma_script_start_addrs	*script_addrs;
31462550cd7SShawn Guo };
31562550cd7SShawn Guo 
3161ec1e82fSSascha Hauer struct sdma_engine {
3171ec1e82fSSascha Hauer 	struct device			*dev;
318b9b3f82fSSascha Hauer 	struct device_dma_parameters	dma_parms;
3191ec1e82fSSascha Hauer 	struct sdma_channel		channel[MAX_DMA_CHANNELS];
3201ec1e82fSSascha Hauer 	struct sdma_channel_control	*channel_control;
3211ec1e82fSSascha Hauer 	void __iomem			*regs;
3221ec1e82fSSascha Hauer 	struct sdma_context_data	*context;
3231ec1e82fSSascha Hauer 	dma_addr_t			context_phys;
3241ec1e82fSSascha Hauer 	struct dma_device		dma_device;
3257560e3f3SSascha Hauer 	struct clk			*clk_ipg;
3267560e3f3SSascha Hauer 	struct clk			*clk_ahb;
3272ccaef05SRichard Zhao 	spinlock_t			channel_0_lock;
328cd72b846SNicolin Chen 	u32				script_number;
3291ec1e82fSSascha Hauer 	struct sdma_script_start_addrs	*script_addrs;
33017bba72fSSascha Hauer 	const struct sdma_driver_data	*drvdata;
33117bba72fSSascha Hauer };
33217bba72fSSascha Hauer 
333e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx31 = {
33417bba72fSSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX31,
33517bba72fSSascha Hauer 	.num_events = 32,
33617bba72fSSascha Hauer };
33717bba72fSSascha Hauer 
338dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx25 = {
339dcfec3c0SSascha Hauer 	.ap_2_ap_addr = 729,
340dcfec3c0SSascha Hauer 	.uart_2_mcu_addr = 904,
341dcfec3c0SSascha Hauer 	.per_2_app_addr = 1255,
342dcfec3c0SSascha Hauer 	.mcu_2_app_addr = 834,
343dcfec3c0SSascha Hauer 	.uartsh_2_mcu_addr = 1120,
344dcfec3c0SSascha Hauer 	.per_2_shp_addr = 1329,
345dcfec3c0SSascha Hauer 	.mcu_2_shp_addr = 1048,
346dcfec3c0SSascha Hauer 	.ata_2_mcu_addr = 1560,
347dcfec3c0SSascha Hauer 	.mcu_2_ata_addr = 1479,
348dcfec3c0SSascha Hauer 	.app_2_per_addr = 1189,
349dcfec3c0SSascha Hauer 	.app_2_mcu_addr = 770,
350dcfec3c0SSascha Hauer 	.shp_2_per_addr = 1407,
351dcfec3c0SSascha Hauer 	.shp_2_mcu_addr = 979,
352dcfec3c0SSascha Hauer };
353dcfec3c0SSascha Hauer 
354e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx25 = {
355dcfec3c0SSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX35,
356dcfec3c0SSascha Hauer 	.num_events = 48,
357dcfec3c0SSascha Hauer 	.script_addrs = &sdma_script_imx25,
358dcfec3c0SSascha Hauer };
359dcfec3c0SSascha Hauer 
360e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx35 = {
36117bba72fSSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX35,
36217bba72fSSascha Hauer 	.num_events = 48,
3631ec1e82fSSascha Hauer };
3641ec1e82fSSascha Hauer 
365dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx51 = {
366dcfec3c0SSascha Hauer 	.ap_2_ap_addr = 642,
367dcfec3c0SSascha Hauer 	.uart_2_mcu_addr = 817,
368dcfec3c0SSascha Hauer 	.mcu_2_app_addr = 747,
369dcfec3c0SSascha Hauer 	.mcu_2_shp_addr = 961,
370dcfec3c0SSascha Hauer 	.ata_2_mcu_addr = 1473,
371dcfec3c0SSascha Hauer 	.mcu_2_ata_addr = 1392,
372dcfec3c0SSascha Hauer 	.app_2_per_addr = 1033,
373dcfec3c0SSascha Hauer 	.app_2_mcu_addr = 683,
374dcfec3c0SSascha Hauer 	.shp_2_per_addr = 1251,
375dcfec3c0SSascha Hauer 	.shp_2_mcu_addr = 892,
376dcfec3c0SSascha Hauer };
377dcfec3c0SSascha Hauer 
378e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx51 = {
379dcfec3c0SSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX35,
380dcfec3c0SSascha Hauer 	.num_events = 48,
381dcfec3c0SSascha Hauer 	.script_addrs = &sdma_script_imx51,
382dcfec3c0SSascha Hauer };
383dcfec3c0SSascha Hauer 
384dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx53 = {
385dcfec3c0SSascha Hauer 	.ap_2_ap_addr = 642,
386dcfec3c0SSascha Hauer 	.app_2_mcu_addr = 683,
387dcfec3c0SSascha Hauer 	.mcu_2_app_addr = 747,
388dcfec3c0SSascha Hauer 	.uart_2_mcu_addr = 817,
389dcfec3c0SSascha Hauer 	.shp_2_mcu_addr = 891,
390dcfec3c0SSascha Hauer 	.mcu_2_shp_addr = 960,
391dcfec3c0SSascha Hauer 	.uartsh_2_mcu_addr = 1032,
392dcfec3c0SSascha Hauer 	.spdif_2_mcu_addr = 1100,
393dcfec3c0SSascha Hauer 	.mcu_2_spdif_addr = 1134,
394dcfec3c0SSascha Hauer 	.firi_2_mcu_addr = 1193,
395dcfec3c0SSascha Hauer 	.mcu_2_firi_addr = 1290,
396dcfec3c0SSascha Hauer };
397dcfec3c0SSascha Hauer 
398e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx53 = {
399dcfec3c0SSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX35,
400dcfec3c0SSascha Hauer 	.num_events = 48,
401dcfec3c0SSascha Hauer 	.script_addrs = &sdma_script_imx53,
402dcfec3c0SSascha Hauer };
403dcfec3c0SSascha Hauer 
404dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx6q = {
405dcfec3c0SSascha Hauer 	.ap_2_ap_addr = 642,
406dcfec3c0SSascha Hauer 	.uart_2_mcu_addr = 817,
407dcfec3c0SSascha Hauer 	.mcu_2_app_addr = 747,
408dcfec3c0SSascha Hauer 	.per_2_per_addr = 6331,
409dcfec3c0SSascha Hauer 	.uartsh_2_mcu_addr = 1032,
410dcfec3c0SSascha Hauer 	.mcu_2_shp_addr = 960,
411dcfec3c0SSascha Hauer 	.app_2_mcu_addr = 683,
412dcfec3c0SSascha Hauer 	.shp_2_mcu_addr = 891,
413dcfec3c0SSascha Hauer 	.spdif_2_mcu_addr = 1100,
414dcfec3c0SSascha Hauer 	.mcu_2_spdif_addr = 1134,
415dcfec3c0SSascha Hauer };
416dcfec3c0SSascha Hauer 
417e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx6q = {
418dcfec3c0SSascha Hauer 	.chnenbl0 = SDMA_CHNENBL0_IMX35,
419dcfec3c0SSascha Hauer 	.num_events = 48,
420dcfec3c0SSascha Hauer 	.script_addrs = &sdma_script_imx6q,
421dcfec3c0SSascha Hauer };
422dcfec3c0SSascha Hauer 
42362550cd7SShawn Guo static struct platform_device_id sdma_devtypes[] = {
42462550cd7SShawn Guo 	{
425dcfec3c0SSascha Hauer 		.name = "imx25-sdma",
426dcfec3c0SSascha Hauer 		.driver_data = (unsigned long)&sdma_imx25,
427dcfec3c0SSascha Hauer 	}, {
42862550cd7SShawn Guo 		.name = "imx31-sdma",
42917bba72fSSascha Hauer 		.driver_data = (unsigned long)&sdma_imx31,
43062550cd7SShawn Guo 	}, {
43162550cd7SShawn Guo 		.name = "imx35-sdma",
43217bba72fSSascha Hauer 		.driver_data = (unsigned long)&sdma_imx35,
43362550cd7SShawn Guo 	}, {
434dcfec3c0SSascha Hauer 		.name = "imx51-sdma",
435dcfec3c0SSascha Hauer 		.driver_data = (unsigned long)&sdma_imx51,
436dcfec3c0SSascha Hauer 	}, {
437dcfec3c0SSascha Hauer 		.name = "imx53-sdma",
438dcfec3c0SSascha Hauer 		.driver_data = (unsigned long)&sdma_imx53,
439dcfec3c0SSascha Hauer 	}, {
440dcfec3c0SSascha Hauer 		.name = "imx6q-sdma",
441dcfec3c0SSascha Hauer 		.driver_data = (unsigned long)&sdma_imx6q,
442dcfec3c0SSascha Hauer 	}, {
44362550cd7SShawn Guo 		/* sentinel */
44462550cd7SShawn Guo 	}
44562550cd7SShawn Guo };
44662550cd7SShawn Guo MODULE_DEVICE_TABLE(platform, sdma_devtypes);
44762550cd7SShawn Guo 
448580975d7SShawn Guo static const struct of_device_id sdma_dt_ids[] = {
449dcfec3c0SSascha Hauer 	{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
450dcfec3c0SSascha Hauer 	{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
451dcfec3c0SSascha Hauer 	{ .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
45217bba72fSSascha Hauer 	{ .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
453dcfec3c0SSascha Hauer 	{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
45463edea16SMarkus Pargmann 	{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
455580975d7SShawn Guo 	{ /* sentinel */ }
456580975d7SShawn Guo };
457580975d7SShawn Guo MODULE_DEVICE_TABLE(of, sdma_dt_ids);
458580975d7SShawn Guo 
4590bbc1413SRichard Zhao #define SDMA_H_CONFIG_DSPDMA	BIT(12) /* indicates if the DSPDMA is used */
4600bbc1413SRichard Zhao #define SDMA_H_CONFIG_RTD_PINS	BIT(11) /* indicates if Real-Time Debug pins are enabled */
4610bbc1413SRichard Zhao #define SDMA_H_CONFIG_ACR	BIT(4)  /* indicates if AHB freq /core freq = 2 or 1 */
4621ec1e82fSSascha Hauer #define SDMA_H_CONFIG_CSM	(3)       /* indicates which context switch mode is selected*/
4631ec1e82fSSascha Hauer 
4641ec1e82fSSascha Hauer static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
4651ec1e82fSSascha Hauer {
46617bba72fSSascha Hauer 	u32 chnenbl0 = sdma->drvdata->chnenbl0;
4671ec1e82fSSascha Hauer 	return chnenbl0 + event * 4;
4681ec1e82fSSascha Hauer }
4691ec1e82fSSascha Hauer 
4701ec1e82fSSascha Hauer static int sdma_config_ownership(struct sdma_channel *sdmac,
4711ec1e82fSSascha Hauer 		bool event_override, bool mcu_override, bool dsp_override)
4721ec1e82fSSascha Hauer {
4731ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
4741ec1e82fSSascha Hauer 	int channel = sdmac->channel;
4750bbc1413SRichard Zhao 	unsigned long evt, mcu, dsp;
4761ec1e82fSSascha Hauer 
4771ec1e82fSSascha Hauer 	if (event_override && mcu_override && dsp_override)
4781ec1e82fSSascha Hauer 		return -EINVAL;
4791ec1e82fSSascha Hauer 
480c4b56857SRichard Zhao 	evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
481c4b56857SRichard Zhao 	mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
482c4b56857SRichard Zhao 	dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
4831ec1e82fSSascha Hauer 
4841ec1e82fSSascha Hauer 	if (dsp_override)
4850bbc1413SRichard Zhao 		__clear_bit(channel, &dsp);
4861ec1e82fSSascha Hauer 	else
4870bbc1413SRichard Zhao 		__set_bit(channel, &dsp);
4881ec1e82fSSascha Hauer 
4891ec1e82fSSascha Hauer 	if (event_override)
4900bbc1413SRichard Zhao 		__clear_bit(channel, &evt);
4911ec1e82fSSascha Hauer 	else
4920bbc1413SRichard Zhao 		__set_bit(channel, &evt);
4931ec1e82fSSascha Hauer 
4941ec1e82fSSascha Hauer 	if (mcu_override)
4950bbc1413SRichard Zhao 		__clear_bit(channel, &mcu);
4961ec1e82fSSascha Hauer 	else
4970bbc1413SRichard Zhao 		__set_bit(channel, &mcu);
4981ec1e82fSSascha Hauer 
499c4b56857SRichard Zhao 	writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
500c4b56857SRichard Zhao 	writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
501c4b56857SRichard Zhao 	writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
5021ec1e82fSSascha Hauer 
5031ec1e82fSSascha Hauer 	return 0;
5041ec1e82fSSascha Hauer }
5051ec1e82fSSascha Hauer 
506b9a59166SRichard Zhao static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
507b9a59166SRichard Zhao {
5080bbc1413SRichard Zhao 	writel(BIT(channel), sdma->regs + SDMA_H_START);
509b9a59166SRichard Zhao }
510b9a59166SRichard Zhao 
5111ec1e82fSSascha Hauer /*
5122ccaef05SRichard Zhao  * sdma_run_channel0 - run a channel and wait till it's done
5131ec1e82fSSascha Hauer  */
5142ccaef05SRichard Zhao static int sdma_run_channel0(struct sdma_engine *sdma)
5151ec1e82fSSascha Hauer {
5161ec1e82fSSascha Hauer 	int ret;
5172ccaef05SRichard Zhao 	unsigned long timeout = 500;
5181ec1e82fSSascha Hauer 
5192ccaef05SRichard Zhao 	sdma_enable_channel(sdma, 0);
5201ec1e82fSSascha Hauer 
5212ccaef05SRichard Zhao 	while (!(ret = readl_relaxed(sdma->regs + SDMA_H_INTR) & 1)) {
5222ccaef05SRichard Zhao 		if (timeout-- <= 0)
5232ccaef05SRichard Zhao 			break;
5242ccaef05SRichard Zhao 		udelay(1);
5252ccaef05SRichard Zhao 	}
5261ec1e82fSSascha Hauer 
5272ccaef05SRichard Zhao 	if (ret) {
5282ccaef05SRichard Zhao 		/* Clear the interrupt status */
5292ccaef05SRichard Zhao 		writel_relaxed(ret, sdma->regs + SDMA_H_INTR);
5302ccaef05SRichard Zhao 	} else {
5312ccaef05SRichard Zhao 		dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
5322ccaef05SRichard Zhao 	}
5331ec1e82fSSascha Hauer 
5341ec1e82fSSascha Hauer 	return ret ? 0 : -ETIMEDOUT;
5351ec1e82fSSascha Hauer }
5361ec1e82fSSascha Hauer 
5371ec1e82fSSascha Hauer static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
5381ec1e82fSSascha Hauer 		u32 address)
5391ec1e82fSSascha Hauer {
5401ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
5411ec1e82fSSascha Hauer 	void *buf_virt;
5421ec1e82fSSascha Hauer 	dma_addr_t buf_phys;
5431ec1e82fSSascha Hauer 	int ret;
5442ccaef05SRichard Zhao 	unsigned long flags;
54573eab978SSascha Hauer 
5461ec1e82fSSascha Hauer 	buf_virt = dma_alloc_coherent(NULL,
5471ec1e82fSSascha Hauer 			size,
5481ec1e82fSSascha Hauer 			&buf_phys, GFP_KERNEL);
54973eab978SSascha Hauer 	if (!buf_virt) {
5502ccaef05SRichard Zhao 		return -ENOMEM;
55173eab978SSascha Hauer 	}
5521ec1e82fSSascha Hauer 
5532ccaef05SRichard Zhao 	spin_lock_irqsave(&sdma->channel_0_lock, flags);
5542ccaef05SRichard Zhao 
5551ec1e82fSSascha Hauer 	bd0->mode.command = C0_SETPM;
5561ec1e82fSSascha Hauer 	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
5571ec1e82fSSascha Hauer 	bd0->mode.count = size / 2;
5581ec1e82fSSascha Hauer 	bd0->buffer_addr = buf_phys;
5591ec1e82fSSascha Hauer 	bd0->ext_buffer_addr = address;
5601ec1e82fSSascha Hauer 
5611ec1e82fSSascha Hauer 	memcpy(buf_virt, buf, size);
5621ec1e82fSSascha Hauer 
5632ccaef05SRichard Zhao 	ret = sdma_run_channel0(sdma);
5642ccaef05SRichard Zhao 
5652ccaef05SRichard Zhao 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
5661ec1e82fSSascha Hauer 
5671ec1e82fSSascha Hauer 	dma_free_coherent(NULL, size, buf_virt, buf_phys);
5681ec1e82fSSascha Hauer 
5691ec1e82fSSascha Hauer 	return ret;
5701ec1e82fSSascha Hauer }
5711ec1e82fSSascha Hauer 
5721ec1e82fSSascha Hauer static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
5731ec1e82fSSascha Hauer {
5741ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
5751ec1e82fSSascha Hauer 	int channel = sdmac->channel;
5760bbc1413SRichard Zhao 	unsigned long val;
5771ec1e82fSSascha Hauer 	u32 chnenbl = chnenbl_ofs(sdma, event);
5781ec1e82fSSascha Hauer 
579c4b56857SRichard Zhao 	val = readl_relaxed(sdma->regs + chnenbl);
5800bbc1413SRichard Zhao 	__set_bit(channel, &val);
581c4b56857SRichard Zhao 	writel_relaxed(val, sdma->regs + chnenbl);
5821ec1e82fSSascha Hauer }
5831ec1e82fSSascha Hauer 
5841ec1e82fSSascha Hauer static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
5851ec1e82fSSascha Hauer {
5861ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
5871ec1e82fSSascha Hauer 	int channel = sdmac->channel;
5881ec1e82fSSascha Hauer 	u32 chnenbl = chnenbl_ofs(sdma, event);
5890bbc1413SRichard Zhao 	unsigned long val;
5901ec1e82fSSascha Hauer 
591c4b56857SRichard Zhao 	val = readl_relaxed(sdma->regs + chnenbl);
5920bbc1413SRichard Zhao 	__clear_bit(channel, &val);
593c4b56857SRichard Zhao 	writel_relaxed(val, sdma->regs + chnenbl);
5941ec1e82fSSascha Hauer }
5951ec1e82fSSascha Hauer 
5961ec1e82fSSascha Hauer static void sdma_handle_channel_loop(struct sdma_channel *sdmac)
5971ec1e82fSSascha Hauer {
598d1a792f3SRussell King - ARM Linux 	if (sdmac->desc.callback)
599d1a792f3SRussell King - ARM Linux 		sdmac->desc.callback(sdmac->desc.callback_param);
600d1a792f3SRussell King - ARM Linux }
601d1a792f3SRussell King - ARM Linux 
602d1a792f3SRussell King - ARM Linux static void sdma_update_channel_loop(struct sdma_channel *sdmac)
603d1a792f3SRussell King - ARM Linux {
6041ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd;
6051ec1e82fSSascha Hauer 
6061ec1e82fSSascha Hauer 	/*
6071ec1e82fSSascha Hauer 	 * loop mode. Iterate over descriptors, re-setup them and
6081ec1e82fSSascha Hauer 	 * call callback function.
6091ec1e82fSSascha Hauer 	 */
6101ec1e82fSSascha Hauer 	while (1) {
6111ec1e82fSSascha Hauer 		bd = &sdmac->bd[sdmac->buf_tail];
6121ec1e82fSSascha Hauer 
6131ec1e82fSSascha Hauer 		if (bd->mode.status & BD_DONE)
6141ec1e82fSSascha Hauer 			break;
6151ec1e82fSSascha Hauer 
6161ec1e82fSSascha Hauer 		if (bd->mode.status & BD_RROR)
6171ec1e82fSSascha Hauer 			sdmac->status = DMA_ERROR;
6181ec1e82fSSascha Hauer 
6191ec1e82fSSascha Hauer 		bd->mode.status |= BD_DONE;
6201ec1e82fSSascha Hauer 		sdmac->buf_tail++;
6211ec1e82fSSascha Hauer 		sdmac->buf_tail %= sdmac->num_bd;
6221ec1e82fSSascha Hauer 	}
6231ec1e82fSSascha Hauer }
6241ec1e82fSSascha Hauer 
6251ec1e82fSSascha Hauer static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)
6261ec1e82fSSascha Hauer {
6271ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd;
6281ec1e82fSSascha Hauer 	int i, error = 0;
6291ec1e82fSSascha Hauer 
630ab59a510SHuang Shijie 	sdmac->chn_real_count = 0;
6311ec1e82fSSascha Hauer 	/*
6321ec1e82fSSascha Hauer 	 * non loop mode. Iterate over all descriptors, collect
6331ec1e82fSSascha Hauer 	 * errors and call callback function
6341ec1e82fSSascha Hauer 	 */
6351ec1e82fSSascha Hauer 	for (i = 0; i < sdmac->num_bd; i++) {
6361ec1e82fSSascha Hauer 		bd = &sdmac->bd[i];
6371ec1e82fSSascha Hauer 
6381ec1e82fSSascha Hauer 		 if (bd->mode.status & (BD_DONE | BD_RROR))
6391ec1e82fSSascha Hauer 			error = -EIO;
640ab59a510SHuang Shijie 		 sdmac->chn_real_count += bd->mode.count;
6411ec1e82fSSascha Hauer 	}
6421ec1e82fSSascha Hauer 
6431ec1e82fSSascha Hauer 	if (error)
6441ec1e82fSSascha Hauer 		sdmac->status = DMA_ERROR;
6451ec1e82fSSascha Hauer 	else
646409bff6aSVinod Koul 		sdmac->status = DMA_COMPLETE;
6471ec1e82fSSascha Hauer 
648f7fbce07SRussell King - ARM Linux 	dma_cookie_complete(&sdmac->desc);
6491ec1e82fSSascha Hauer 	if (sdmac->desc.callback)
6501ec1e82fSSascha Hauer 		sdmac->desc.callback(sdmac->desc.callback_param);
6511ec1e82fSSascha Hauer }
6521ec1e82fSSascha Hauer 
653abd9ccc8SHuang Shijie static void sdma_tasklet(unsigned long data)
6541ec1e82fSSascha Hauer {
655abd9ccc8SHuang Shijie 	struct sdma_channel *sdmac = (struct sdma_channel *) data;
656abd9ccc8SHuang Shijie 
6571ec1e82fSSascha Hauer 	if (sdmac->flags & IMX_DMA_SG_LOOP)
6581ec1e82fSSascha Hauer 		sdma_handle_channel_loop(sdmac);
6591ec1e82fSSascha Hauer 	else
6601ec1e82fSSascha Hauer 		mxc_sdma_handle_channel_normal(sdmac);
6611ec1e82fSSascha Hauer }
6621ec1e82fSSascha Hauer 
6631ec1e82fSSascha Hauer static irqreturn_t sdma_int_handler(int irq, void *dev_id)
6641ec1e82fSSascha Hauer {
6651ec1e82fSSascha Hauer 	struct sdma_engine *sdma = dev_id;
6660bbc1413SRichard Zhao 	unsigned long stat;
6671ec1e82fSSascha Hauer 
668c4b56857SRichard Zhao 	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
6692ccaef05SRichard Zhao 	/* not interested in channel 0 interrupts */
6702ccaef05SRichard Zhao 	stat &= ~1;
671c4b56857SRichard Zhao 	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
6721ec1e82fSSascha Hauer 
6731ec1e82fSSascha Hauer 	while (stat) {
6741ec1e82fSSascha Hauer 		int channel = fls(stat) - 1;
6751ec1e82fSSascha Hauer 		struct sdma_channel *sdmac = &sdma->channel[channel];
6761ec1e82fSSascha Hauer 
677d1a792f3SRussell King - ARM Linux 		if (sdmac->flags & IMX_DMA_SG_LOOP)
678d1a792f3SRussell King - ARM Linux 			sdma_update_channel_loop(sdmac);
679d1a792f3SRussell King - ARM Linux 
680abd9ccc8SHuang Shijie 		tasklet_schedule(&sdmac->tasklet);
6811ec1e82fSSascha Hauer 
6820bbc1413SRichard Zhao 		__clear_bit(channel, &stat);
6831ec1e82fSSascha Hauer 	}
6841ec1e82fSSascha Hauer 
6851ec1e82fSSascha Hauer 	return IRQ_HANDLED;
6861ec1e82fSSascha Hauer }
6871ec1e82fSSascha Hauer 
6881ec1e82fSSascha Hauer /*
6891ec1e82fSSascha Hauer  * sets the pc of SDMA script according to the peripheral type
6901ec1e82fSSascha Hauer  */
6911ec1e82fSSascha Hauer static void sdma_get_pc(struct sdma_channel *sdmac,
6921ec1e82fSSascha Hauer 		enum sdma_peripheral_type peripheral_type)
6931ec1e82fSSascha Hauer {
6941ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
6951ec1e82fSSascha Hauer 	int per_2_emi = 0, emi_2_per = 0;
6961ec1e82fSSascha Hauer 	/*
6971ec1e82fSSascha Hauer 	 * These are needed once we start to support transfers between
6981ec1e82fSSascha Hauer 	 * two peripherals or memory-to-memory transfers
6991ec1e82fSSascha Hauer 	 */
7001ec1e82fSSascha Hauer 	int per_2_per = 0, emi_2_emi = 0;
7011ec1e82fSSascha Hauer 
7021ec1e82fSSascha Hauer 	sdmac->pc_from_device = 0;
7031ec1e82fSSascha Hauer 	sdmac->pc_to_device = 0;
7041ec1e82fSSascha Hauer 
7051ec1e82fSSascha Hauer 	switch (peripheral_type) {
7061ec1e82fSSascha Hauer 	case IMX_DMATYPE_MEMORY:
7071ec1e82fSSascha Hauer 		emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
7081ec1e82fSSascha Hauer 		break;
7091ec1e82fSSascha Hauer 	case IMX_DMATYPE_DSP:
7101ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->bp_2_ap_addr;
7111ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->ap_2_bp_addr;
7121ec1e82fSSascha Hauer 		break;
7131ec1e82fSSascha Hauer 	case IMX_DMATYPE_FIRI:
7141ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
7151ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
7161ec1e82fSSascha Hauer 		break;
7171ec1e82fSSascha Hauer 	case IMX_DMATYPE_UART:
7181ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
7191ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
7201ec1e82fSSascha Hauer 		break;
7211ec1e82fSSascha Hauer 	case IMX_DMATYPE_UART_SP:
7221ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
7231ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
7241ec1e82fSSascha Hauer 		break;
7251ec1e82fSSascha Hauer 	case IMX_DMATYPE_ATA:
7261ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
7271ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
7281ec1e82fSSascha Hauer 		break;
7291ec1e82fSSascha Hauer 	case IMX_DMATYPE_CSPI:
7301ec1e82fSSascha Hauer 	case IMX_DMATYPE_EXT:
7311ec1e82fSSascha Hauer 	case IMX_DMATYPE_SSI:
73229aebfdeSNicolin Chen 	case IMX_DMATYPE_SAI:
7331ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->app_2_mcu_addr;
7341ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
7351ec1e82fSSascha Hauer 		break;
7361a895578SNicolin Chen 	case IMX_DMATYPE_SSI_DUAL:
7371a895578SNicolin Chen 		per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
7381a895578SNicolin Chen 		emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
7391a895578SNicolin Chen 		break;
7401ec1e82fSSascha Hauer 	case IMX_DMATYPE_SSI_SP:
7411ec1e82fSSascha Hauer 	case IMX_DMATYPE_MMC:
7421ec1e82fSSascha Hauer 	case IMX_DMATYPE_SDHC:
7431ec1e82fSSascha Hauer 	case IMX_DMATYPE_CSPI_SP:
7441ec1e82fSSascha Hauer 	case IMX_DMATYPE_ESAI:
7451ec1e82fSSascha Hauer 	case IMX_DMATYPE_MSHC_SP:
7461ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
7471ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
7481ec1e82fSSascha Hauer 		break;
7491ec1e82fSSascha Hauer 	case IMX_DMATYPE_ASRC:
7501ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
7511ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
7521ec1e82fSSascha Hauer 		per_2_per = sdma->script_addrs->per_2_per_addr;
7531ec1e82fSSascha Hauer 		break;
754f892afb0SNicolin Chen 	case IMX_DMATYPE_ASRC_SP:
755f892afb0SNicolin Chen 		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
756f892afb0SNicolin Chen 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
757f892afb0SNicolin Chen 		per_2_per = sdma->script_addrs->per_2_per_addr;
758f892afb0SNicolin Chen 		break;
7591ec1e82fSSascha Hauer 	case IMX_DMATYPE_MSHC:
7601ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
7611ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
7621ec1e82fSSascha Hauer 		break;
7631ec1e82fSSascha Hauer 	case IMX_DMATYPE_CCM:
7641ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
7651ec1e82fSSascha Hauer 		break;
7661ec1e82fSSascha Hauer 	case IMX_DMATYPE_SPDIF:
7671ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
7681ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
7691ec1e82fSSascha Hauer 		break;
7701ec1e82fSSascha Hauer 	case IMX_DMATYPE_IPU_MEMORY:
7711ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
7721ec1e82fSSascha Hauer 		break;
7731ec1e82fSSascha Hauer 	default:
7741ec1e82fSSascha Hauer 		break;
7751ec1e82fSSascha Hauer 	}
7761ec1e82fSSascha Hauer 
7771ec1e82fSSascha Hauer 	sdmac->pc_from_device = per_2_emi;
7781ec1e82fSSascha Hauer 	sdmac->pc_to_device = emi_2_per;
7791ec1e82fSSascha Hauer }
7801ec1e82fSSascha Hauer 
7811ec1e82fSSascha Hauer static int sdma_load_context(struct sdma_channel *sdmac)
7821ec1e82fSSascha Hauer {
7831ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
7841ec1e82fSSascha Hauer 	int channel = sdmac->channel;
7851ec1e82fSSascha Hauer 	int load_address;
7861ec1e82fSSascha Hauer 	struct sdma_context_data *context = sdma->context;
7871ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
7881ec1e82fSSascha Hauer 	int ret;
7892ccaef05SRichard Zhao 	unsigned long flags;
7901ec1e82fSSascha Hauer 
791db8196dfSVinod Koul 	if (sdmac->direction == DMA_DEV_TO_MEM) {
7921ec1e82fSSascha Hauer 		load_address = sdmac->pc_from_device;
7931ec1e82fSSascha Hauer 	} else {
7941ec1e82fSSascha Hauer 		load_address = sdmac->pc_to_device;
7951ec1e82fSSascha Hauer 	}
7961ec1e82fSSascha Hauer 
7971ec1e82fSSascha Hauer 	if (load_address < 0)
7981ec1e82fSSascha Hauer 		return load_address;
7991ec1e82fSSascha Hauer 
8001ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "load_address = %d\n", load_address);
8010bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
8021ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
8031ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
8040bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
8050bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
8061ec1e82fSSascha Hauer 
8072ccaef05SRichard Zhao 	spin_lock_irqsave(&sdma->channel_0_lock, flags);
80873eab978SSascha Hauer 
8091ec1e82fSSascha Hauer 	memset(context, 0, sizeof(*context));
8101ec1e82fSSascha Hauer 	context->channel_state.pc = load_address;
8111ec1e82fSSascha Hauer 
8121ec1e82fSSascha Hauer 	/* Send by context the event mask,base address for peripheral
8131ec1e82fSSascha Hauer 	 * and watermark level
8141ec1e82fSSascha Hauer 	 */
8150bbc1413SRichard Zhao 	context->gReg[0] = sdmac->event_mask[1];
8160bbc1413SRichard Zhao 	context->gReg[1] = sdmac->event_mask[0];
8171ec1e82fSSascha Hauer 	context->gReg[2] = sdmac->per_addr;
8181ec1e82fSSascha Hauer 	context->gReg[6] = sdmac->shp_addr;
8191ec1e82fSSascha Hauer 	context->gReg[7] = sdmac->watermark_level;
8201ec1e82fSSascha Hauer 
8211ec1e82fSSascha Hauer 	bd0->mode.command = C0_SETDM;
8221ec1e82fSSascha Hauer 	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
8231ec1e82fSSascha Hauer 	bd0->mode.count = sizeof(*context) / 4;
8241ec1e82fSSascha Hauer 	bd0->buffer_addr = sdma->context_phys;
8251ec1e82fSSascha Hauer 	bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
8262ccaef05SRichard Zhao 	ret = sdma_run_channel0(sdma);
8271ec1e82fSSascha Hauer 
8282ccaef05SRichard Zhao 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
82973eab978SSascha Hauer 
8301ec1e82fSSascha Hauer 	return ret;
8311ec1e82fSSascha Hauer }
8321ec1e82fSSascha Hauer 
8337b350ab0SMaxime Ripard static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
8341ec1e82fSSascha Hauer {
8357b350ab0SMaxime Ripard 	return container_of(chan, struct sdma_channel, chan);
8367b350ab0SMaxime Ripard }
8377b350ab0SMaxime Ripard 
8387b350ab0SMaxime Ripard static int sdma_disable_channel(struct dma_chan *chan)
8397b350ab0SMaxime Ripard {
8407b350ab0SMaxime Ripard 	struct sdma_channel *sdmac = to_sdma_chan(chan);
8411ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
8421ec1e82fSSascha Hauer 	int channel = sdmac->channel;
8431ec1e82fSSascha Hauer 
8440bbc1413SRichard Zhao 	writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
8451ec1e82fSSascha Hauer 	sdmac->status = DMA_ERROR;
8467b350ab0SMaxime Ripard 
8477b350ab0SMaxime Ripard 	return 0;
8481ec1e82fSSascha Hauer }
8491ec1e82fSSascha Hauer 
8507b350ab0SMaxime Ripard static int sdma_config_channel(struct dma_chan *chan)
8511ec1e82fSSascha Hauer {
8527b350ab0SMaxime Ripard 	struct sdma_channel *sdmac = to_sdma_chan(chan);
8531ec1e82fSSascha Hauer 	int ret;
8541ec1e82fSSascha Hauer 
8557b350ab0SMaxime Ripard 	sdma_disable_channel(chan);
8561ec1e82fSSascha Hauer 
8570bbc1413SRichard Zhao 	sdmac->event_mask[0] = 0;
8580bbc1413SRichard Zhao 	sdmac->event_mask[1] = 0;
8591ec1e82fSSascha Hauer 	sdmac->shp_addr = 0;
8601ec1e82fSSascha Hauer 	sdmac->per_addr = 0;
8611ec1e82fSSascha Hauer 
8621ec1e82fSSascha Hauer 	if (sdmac->event_id0) {
86317bba72fSSascha Hauer 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
8641ec1e82fSSascha Hauer 			return -EINVAL;
8651ec1e82fSSascha Hauer 		sdma_event_enable(sdmac, sdmac->event_id0);
8661ec1e82fSSascha Hauer 	}
8671ec1e82fSSascha Hauer 
8681ec1e82fSSascha Hauer 	switch (sdmac->peripheral_type) {
8691ec1e82fSSascha Hauer 	case IMX_DMATYPE_DSP:
8701ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, false, true, true);
8711ec1e82fSSascha Hauer 		break;
8721ec1e82fSSascha Hauer 	case IMX_DMATYPE_MEMORY:
8731ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, false, true, false);
8741ec1e82fSSascha Hauer 		break;
8751ec1e82fSSascha Hauer 	default:
8761ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, true, true, false);
8771ec1e82fSSascha Hauer 		break;
8781ec1e82fSSascha Hauer 	}
8791ec1e82fSSascha Hauer 
8801ec1e82fSSascha Hauer 	sdma_get_pc(sdmac, sdmac->peripheral_type);
8811ec1e82fSSascha Hauer 
8821ec1e82fSSascha Hauer 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
8831ec1e82fSSascha Hauer 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
8841ec1e82fSSascha Hauer 		/* Handle multiple event channels differently */
8851ec1e82fSSascha Hauer 		if (sdmac->event_id1) {
8860bbc1413SRichard Zhao 			sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32);
8871ec1e82fSSascha Hauer 			if (sdmac->event_id1 > 31)
8880bbc1413SRichard Zhao 				__set_bit(31, &sdmac->watermark_level);
8890bbc1413SRichard Zhao 			sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32);
8901ec1e82fSSascha Hauer 			if (sdmac->event_id0 > 31)
8910bbc1413SRichard Zhao 				__set_bit(30, &sdmac->watermark_level);
8921ec1e82fSSascha Hauer 		} else {
8930bbc1413SRichard Zhao 			__set_bit(sdmac->event_id0, sdmac->event_mask);
8941ec1e82fSSascha Hauer 		}
8951ec1e82fSSascha Hauer 		/* Watermark Level */
8961ec1e82fSSascha Hauer 		sdmac->watermark_level |= sdmac->watermark_level;
8971ec1e82fSSascha Hauer 		/* Address */
8981ec1e82fSSascha Hauer 		sdmac->shp_addr = sdmac->per_address;
8991ec1e82fSSascha Hauer 	} else {
9001ec1e82fSSascha Hauer 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
9011ec1e82fSSascha Hauer 	}
9021ec1e82fSSascha Hauer 
9031ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
9041ec1e82fSSascha Hauer 
9051ec1e82fSSascha Hauer 	return ret;
9061ec1e82fSSascha Hauer }
9071ec1e82fSSascha Hauer 
9081ec1e82fSSascha Hauer static int sdma_set_channel_priority(struct sdma_channel *sdmac,
9091ec1e82fSSascha Hauer 		unsigned int priority)
9101ec1e82fSSascha Hauer {
9111ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
9121ec1e82fSSascha Hauer 	int channel = sdmac->channel;
9131ec1e82fSSascha Hauer 
9141ec1e82fSSascha Hauer 	if (priority < MXC_SDMA_MIN_PRIORITY
9151ec1e82fSSascha Hauer 	    || priority > MXC_SDMA_MAX_PRIORITY) {
9161ec1e82fSSascha Hauer 		return -EINVAL;
9171ec1e82fSSascha Hauer 	}
9181ec1e82fSSascha Hauer 
919c4b56857SRichard Zhao 	writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
9201ec1e82fSSascha Hauer 
9211ec1e82fSSascha Hauer 	return 0;
9221ec1e82fSSascha Hauer }
9231ec1e82fSSascha Hauer 
9241ec1e82fSSascha Hauer static int sdma_request_channel(struct sdma_channel *sdmac)
9251ec1e82fSSascha Hauer {
9261ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
9271ec1e82fSSascha Hauer 	int channel = sdmac->channel;
9281ec1e82fSSascha Hauer 	int ret = -EBUSY;
9291ec1e82fSSascha Hauer 
9309f92d223SJoe Perches 	sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys,
9319f92d223SJoe Perches 					GFP_KERNEL);
9321ec1e82fSSascha Hauer 	if (!sdmac->bd) {
9331ec1e82fSSascha Hauer 		ret = -ENOMEM;
9341ec1e82fSSascha Hauer 		goto out;
9351ec1e82fSSascha Hauer 	}
9361ec1e82fSSascha Hauer 
9371ec1e82fSSascha Hauer 	sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;
9381ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
9391ec1e82fSSascha Hauer 
9401ec1e82fSSascha Hauer 	sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY);
9411ec1e82fSSascha Hauer 	return 0;
9421ec1e82fSSascha Hauer out:
9431ec1e82fSSascha Hauer 
9441ec1e82fSSascha Hauer 	return ret;
9451ec1e82fSSascha Hauer }
9461ec1e82fSSascha Hauer 
9471ec1e82fSSascha Hauer static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx)
9481ec1e82fSSascha Hauer {
949f69f2e26SHaitao Zhang 	unsigned long flags;
9501ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(tx->chan);
9511ec1e82fSSascha Hauer 	dma_cookie_t cookie;
9521ec1e82fSSascha Hauer 
953f69f2e26SHaitao Zhang 	spin_lock_irqsave(&sdmac->lock, flags);
9541ec1e82fSSascha Hauer 
955884485e1SRussell King - ARM Linux 	cookie = dma_cookie_assign(tx);
9561ec1e82fSSascha Hauer 
957f69f2e26SHaitao Zhang 	spin_unlock_irqrestore(&sdmac->lock, flags);
9581ec1e82fSSascha Hauer 
9591ec1e82fSSascha Hauer 	return cookie;
9601ec1e82fSSascha Hauer }
9611ec1e82fSSascha Hauer 
9621ec1e82fSSascha Hauer static int sdma_alloc_chan_resources(struct dma_chan *chan)
9631ec1e82fSSascha Hauer {
9641ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
9651ec1e82fSSascha Hauer 	struct imx_dma_data *data = chan->private;
9661ec1e82fSSascha Hauer 	int prio, ret;
9671ec1e82fSSascha Hauer 
9681ec1e82fSSascha Hauer 	if (!data)
9691ec1e82fSSascha Hauer 		return -EINVAL;
9701ec1e82fSSascha Hauer 
9711ec1e82fSSascha Hauer 	switch (data->priority) {
9721ec1e82fSSascha Hauer 	case DMA_PRIO_HIGH:
9731ec1e82fSSascha Hauer 		prio = 3;
9741ec1e82fSSascha Hauer 		break;
9751ec1e82fSSascha Hauer 	case DMA_PRIO_MEDIUM:
9761ec1e82fSSascha Hauer 		prio = 2;
9771ec1e82fSSascha Hauer 		break;
9781ec1e82fSSascha Hauer 	case DMA_PRIO_LOW:
9791ec1e82fSSascha Hauer 	default:
9801ec1e82fSSascha Hauer 		prio = 1;
9811ec1e82fSSascha Hauer 		break;
9821ec1e82fSSascha Hauer 	}
9831ec1e82fSSascha Hauer 
9841ec1e82fSSascha Hauer 	sdmac->peripheral_type = data->peripheral_type;
9851ec1e82fSSascha Hauer 	sdmac->event_id0 = data->dma_request;
986c2c744d3SRichard Zhao 
9877560e3f3SSascha Hauer 	clk_enable(sdmac->sdma->clk_ipg);
9887560e3f3SSascha Hauer 	clk_enable(sdmac->sdma->clk_ahb);
989c2c744d3SRichard Zhao 
9903bb5e7caSRichard Zhao 	ret = sdma_request_channel(sdmac);
9911ec1e82fSSascha Hauer 	if (ret)
9921ec1e82fSSascha Hauer 		return ret;
9931ec1e82fSSascha Hauer 
9943bb5e7caSRichard Zhao 	ret = sdma_set_channel_priority(sdmac, prio);
9951ec1e82fSSascha Hauer 	if (ret)
9961ec1e82fSSascha Hauer 		return ret;
9971ec1e82fSSascha Hauer 
9981ec1e82fSSascha Hauer 	dma_async_tx_descriptor_init(&sdmac->desc, chan);
9991ec1e82fSSascha Hauer 	sdmac->desc.tx_submit = sdma_tx_submit;
10001ec1e82fSSascha Hauer 	/* txd.flags will be overwritten in prep funcs */
10011ec1e82fSSascha Hauer 	sdmac->desc.flags = DMA_CTRL_ACK;
10021ec1e82fSSascha Hauer 
10031ec1e82fSSascha Hauer 	return 0;
10041ec1e82fSSascha Hauer }
10051ec1e82fSSascha Hauer 
10061ec1e82fSSascha Hauer static void sdma_free_chan_resources(struct dma_chan *chan)
10071ec1e82fSSascha Hauer {
10081ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
10091ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
10101ec1e82fSSascha Hauer 
10117b350ab0SMaxime Ripard 	sdma_disable_channel(chan);
10121ec1e82fSSascha Hauer 
10131ec1e82fSSascha Hauer 	if (sdmac->event_id0)
10141ec1e82fSSascha Hauer 		sdma_event_disable(sdmac, sdmac->event_id0);
10151ec1e82fSSascha Hauer 	if (sdmac->event_id1)
10161ec1e82fSSascha Hauer 		sdma_event_disable(sdmac, sdmac->event_id1);
10171ec1e82fSSascha Hauer 
10181ec1e82fSSascha Hauer 	sdmac->event_id0 = 0;
10191ec1e82fSSascha Hauer 	sdmac->event_id1 = 0;
10201ec1e82fSSascha Hauer 
10211ec1e82fSSascha Hauer 	sdma_set_channel_priority(sdmac, 0);
10221ec1e82fSSascha Hauer 
10231ec1e82fSSascha Hauer 	dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys);
10241ec1e82fSSascha Hauer 
10257560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
10267560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
10271ec1e82fSSascha Hauer }
10281ec1e82fSSascha Hauer 
10291ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
10301ec1e82fSSascha Hauer 		struct dma_chan *chan, struct scatterlist *sgl,
1031db8196dfSVinod Koul 		unsigned int sg_len, enum dma_transfer_direction direction,
1032185ecb5fSAlexandre Bounine 		unsigned long flags, void *context)
10331ec1e82fSSascha Hauer {
10341ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
10351ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
10361ec1e82fSSascha Hauer 	int ret, i, count;
103723889c63SSascha Hauer 	int channel = sdmac->channel;
10381ec1e82fSSascha Hauer 	struct scatterlist *sg;
10391ec1e82fSSascha Hauer 
10401ec1e82fSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
10411ec1e82fSSascha Hauer 		return NULL;
10421ec1e82fSSascha Hauer 	sdmac->status = DMA_IN_PROGRESS;
10431ec1e82fSSascha Hauer 
10441ec1e82fSSascha Hauer 	sdmac->flags = 0;
10451ec1e82fSSascha Hauer 
10468e2e27c7SRichard Zhao 	sdmac->buf_tail = 0;
10478e2e27c7SRichard Zhao 
10481ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
10491ec1e82fSSascha Hauer 			sg_len, channel);
10501ec1e82fSSascha Hauer 
10511ec1e82fSSascha Hauer 	sdmac->direction = direction;
10521ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
10531ec1e82fSSascha Hauer 	if (ret)
10541ec1e82fSSascha Hauer 		goto err_out;
10551ec1e82fSSascha Hauer 
10561ec1e82fSSascha Hauer 	if (sg_len > NUM_BD) {
10571ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
10581ec1e82fSSascha Hauer 				channel, sg_len, NUM_BD);
10591ec1e82fSSascha Hauer 		ret = -EINVAL;
10601ec1e82fSSascha Hauer 		goto err_out;
10611ec1e82fSSascha Hauer 	}
10621ec1e82fSSascha Hauer 
1063ab59a510SHuang Shijie 	sdmac->chn_count = 0;
10641ec1e82fSSascha Hauer 	for_each_sg(sgl, sg, sg_len, i) {
10651ec1e82fSSascha Hauer 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
10661ec1e82fSSascha Hauer 		int param;
10671ec1e82fSSascha Hauer 
1068d2f5c276SAnatolij Gustschin 		bd->buffer_addr = sg->dma_address;
10691ec1e82fSSascha Hauer 
1070fdaf9c4bSLars-Peter Clausen 		count = sg_dma_len(sg);
10711ec1e82fSSascha Hauer 
10721ec1e82fSSascha Hauer 		if (count > 0xffff) {
10731ec1e82fSSascha Hauer 			dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
10741ec1e82fSSascha Hauer 					channel, count, 0xffff);
10751ec1e82fSSascha Hauer 			ret = -EINVAL;
10761ec1e82fSSascha Hauer 			goto err_out;
10771ec1e82fSSascha Hauer 		}
10781ec1e82fSSascha Hauer 
10791ec1e82fSSascha Hauer 		bd->mode.count = count;
1080ab59a510SHuang Shijie 		sdmac->chn_count += count;
10811ec1e82fSSascha Hauer 
10821ec1e82fSSascha Hauer 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) {
10831ec1e82fSSascha Hauer 			ret =  -EINVAL;
10841ec1e82fSSascha Hauer 			goto err_out;
10851ec1e82fSSascha Hauer 		}
10861fa81c27SSascha Hauer 
10871fa81c27SSascha Hauer 		switch (sdmac->word_size) {
10881fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_4_BYTES:
10891ec1e82fSSascha Hauer 			bd->mode.command = 0;
10901fa81c27SSascha Hauer 			if (count & 3 || sg->dma_address & 3)
10911fa81c27SSascha Hauer 				return NULL;
10921fa81c27SSascha Hauer 			break;
10931fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_2_BYTES:
10941fa81c27SSascha Hauer 			bd->mode.command = 2;
10951fa81c27SSascha Hauer 			if (count & 1 || sg->dma_address & 1)
10961fa81c27SSascha Hauer 				return NULL;
10971fa81c27SSascha Hauer 			break;
10981fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_1_BYTE:
10991fa81c27SSascha Hauer 			bd->mode.command = 1;
11001fa81c27SSascha Hauer 			break;
11011fa81c27SSascha Hauer 		default:
11021fa81c27SSascha Hauer 			return NULL;
11031fa81c27SSascha Hauer 		}
11041ec1e82fSSascha Hauer 
11051ec1e82fSSascha Hauer 		param = BD_DONE | BD_EXTD | BD_CONT;
11061ec1e82fSSascha Hauer 
1107341b9419SShawn Guo 		if (i + 1 == sg_len) {
11081ec1e82fSSascha Hauer 			param |= BD_INTR;
1109341b9419SShawn Guo 			param |= BD_LAST;
1110341b9419SShawn Guo 			param &= ~BD_CONT;
11111ec1e82fSSascha Hauer 		}
11121ec1e82fSSascha Hauer 
1113c3cc74b2SOlof Johansson 		dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1114c3cc74b2SOlof Johansson 				i, count, (u64)sg->dma_address,
11151ec1e82fSSascha Hauer 				param & BD_WRAP ? "wrap" : "",
11161ec1e82fSSascha Hauer 				param & BD_INTR ? " intr" : "");
11171ec1e82fSSascha Hauer 
11181ec1e82fSSascha Hauer 		bd->mode.status = param;
11191ec1e82fSSascha Hauer 	}
11201ec1e82fSSascha Hauer 
11211ec1e82fSSascha Hauer 	sdmac->num_bd = sg_len;
11221ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
11231ec1e82fSSascha Hauer 
11241ec1e82fSSascha Hauer 	return &sdmac->desc;
11251ec1e82fSSascha Hauer err_out:
11264b2ce9ddSShawn Guo 	sdmac->status = DMA_ERROR;
11271ec1e82fSSascha Hauer 	return NULL;
11281ec1e82fSSascha Hauer }
11291ec1e82fSSascha Hauer 
11301ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
11311ec1e82fSSascha Hauer 		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1132185ecb5fSAlexandre Bounine 		size_t period_len, enum dma_transfer_direction direction,
113331c1e5a1SLaurent Pinchart 		unsigned long flags)
11341ec1e82fSSascha Hauer {
11351ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
11361ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
11371ec1e82fSSascha Hauer 	int num_periods = buf_len / period_len;
113823889c63SSascha Hauer 	int channel = sdmac->channel;
11391ec1e82fSSascha Hauer 	int ret, i = 0, buf = 0;
11401ec1e82fSSascha Hauer 
11411ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
11421ec1e82fSSascha Hauer 
11431ec1e82fSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
11441ec1e82fSSascha Hauer 		return NULL;
11451ec1e82fSSascha Hauer 
11461ec1e82fSSascha Hauer 	sdmac->status = DMA_IN_PROGRESS;
11471ec1e82fSSascha Hauer 
11488e2e27c7SRichard Zhao 	sdmac->buf_tail = 0;
1149d1a792f3SRussell King - ARM Linux 	sdmac->period_len = period_len;
11508e2e27c7SRichard Zhao 
11511ec1e82fSSascha Hauer 	sdmac->flags |= IMX_DMA_SG_LOOP;
11521ec1e82fSSascha Hauer 	sdmac->direction = direction;
11531ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
11541ec1e82fSSascha Hauer 	if (ret)
11551ec1e82fSSascha Hauer 		goto err_out;
11561ec1e82fSSascha Hauer 
11571ec1e82fSSascha Hauer 	if (num_periods > NUM_BD) {
11581ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
11591ec1e82fSSascha Hauer 				channel, num_periods, NUM_BD);
11601ec1e82fSSascha Hauer 		goto err_out;
11611ec1e82fSSascha Hauer 	}
11621ec1e82fSSascha Hauer 
11631ec1e82fSSascha Hauer 	if (period_len > 0xffff) {
11641ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %d > %d\n",
11651ec1e82fSSascha Hauer 				channel, period_len, 0xffff);
11661ec1e82fSSascha Hauer 		goto err_out;
11671ec1e82fSSascha Hauer 	}
11681ec1e82fSSascha Hauer 
11691ec1e82fSSascha Hauer 	while (buf < buf_len) {
11701ec1e82fSSascha Hauer 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
11711ec1e82fSSascha Hauer 		int param;
11721ec1e82fSSascha Hauer 
11731ec1e82fSSascha Hauer 		bd->buffer_addr = dma_addr;
11741ec1e82fSSascha Hauer 
11751ec1e82fSSascha Hauer 		bd->mode.count = period_len;
11761ec1e82fSSascha Hauer 
11771ec1e82fSSascha Hauer 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
11781ec1e82fSSascha Hauer 			goto err_out;
11791ec1e82fSSascha Hauer 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
11801ec1e82fSSascha Hauer 			bd->mode.command = 0;
11811ec1e82fSSascha Hauer 		else
11821ec1e82fSSascha Hauer 			bd->mode.command = sdmac->word_size;
11831ec1e82fSSascha Hauer 
11841ec1e82fSSascha Hauer 		param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
11851ec1e82fSSascha Hauer 		if (i + 1 == num_periods)
11861ec1e82fSSascha Hauer 			param |= BD_WRAP;
11871ec1e82fSSascha Hauer 
1188c3cc74b2SOlof Johansson 		dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1189c3cc74b2SOlof Johansson 				i, period_len, (u64)dma_addr,
11901ec1e82fSSascha Hauer 				param & BD_WRAP ? "wrap" : "",
11911ec1e82fSSascha Hauer 				param & BD_INTR ? " intr" : "");
11921ec1e82fSSascha Hauer 
11931ec1e82fSSascha Hauer 		bd->mode.status = param;
11941ec1e82fSSascha Hauer 
11951ec1e82fSSascha Hauer 		dma_addr += period_len;
11961ec1e82fSSascha Hauer 		buf += period_len;
11971ec1e82fSSascha Hauer 
11981ec1e82fSSascha Hauer 		i++;
11991ec1e82fSSascha Hauer 	}
12001ec1e82fSSascha Hauer 
12011ec1e82fSSascha Hauer 	sdmac->num_bd = num_periods;
12021ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
12031ec1e82fSSascha Hauer 
12041ec1e82fSSascha Hauer 	return &sdmac->desc;
12051ec1e82fSSascha Hauer err_out:
12061ec1e82fSSascha Hauer 	sdmac->status = DMA_ERROR;
12071ec1e82fSSascha Hauer 	return NULL;
12081ec1e82fSSascha Hauer }
12091ec1e82fSSascha Hauer 
12107b350ab0SMaxime Ripard static int sdma_config(struct dma_chan *chan,
12117b350ab0SMaxime Ripard 		       struct dma_slave_config *dmaengine_cfg)
12121ec1e82fSSascha Hauer {
12131ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
12141ec1e82fSSascha Hauer 
1215db8196dfSVinod Koul 	if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) {
12161ec1e82fSSascha Hauer 		sdmac->per_address = dmaengine_cfg->src_addr;
121794ac27a5SPhilippe Rétornaz 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *
121894ac27a5SPhilippe Rétornaz 			dmaengine_cfg->src_addr_width;
12191ec1e82fSSascha Hauer 		sdmac->word_size = dmaengine_cfg->src_addr_width;
12201ec1e82fSSascha Hauer 	} else {
12211ec1e82fSSascha Hauer 		sdmac->per_address = dmaengine_cfg->dst_addr;
122294ac27a5SPhilippe Rétornaz 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
122394ac27a5SPhilippe Rétornaz 			dmaengine_cfg->dst_addr_width;
12241ec1e82fSSascha Hauer 		sdmac->word_size = dmaengine_cfg->dst_addr_width;
12251ec1e82fSSascha Hauer 	}
1226e6966433SHuang Shijie 	sdmac->direction = dmaengine_cfg->direction;
12277b350ab0SMaxime Ripard 	return sdma_config_channel(chan);
12281ec1e82fSSascha Hauer }
12291ec1e82fSSascha Hauer 
12301ec1e82fSSascha Hauer static enum dma_status sdma_tx_status(struct dma_chan *chan,
12311ec1e82fSSascha Hauer 				      dma_cookie_t cookie,
12321ec1e82fSSascha Hauer 				      struct dma_tx_state *txstate)
12331ec1e82fSSascha Hauer {
12341ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1235d1a792f3SRussell King - ARM Linux 	u32 residue;
1236d1a792f3SRussell King - ARM Linux 
1237d1a792f3SRussell King - ARM Linux 	if (sdmac->flags & IMX_DMA_SG_LOOP)
1238d1a792f3SRussell King - ARM Linux 		residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len;
1239d1a792f3SRussell King - ARM Linux 	else
1240d1a792f3SRussell King - ARM Linux 		residue = sdmac->chn_count - sdmac->chn_real_count;
12411ec1e82fSSascha Hauer 
1242e8e3a790SAndy Shevchenko 	dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
1243d1a792f3SRussell King - ARM Linux 			 residue);
12441ec1e82fSSascha Hauer 
12458a965911SShawn Guo 	return sdmac->status;
12461ec1e82fSSascha Hauer }
12471ec1e82fSSascha Hauer 
12481ec1e82fSSascha Hauer static void sdma_issue_pending(struct dma_chan *chan)
12491ec1e82fSSascha Hauer {
12502b4f130eSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
12512b4f130eSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
12522b4f130eSSascha Hauer 
12532b4f130eSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
12542b4f130eSSascha Hauer 		sdma_enable_channel(sdma, sdmac->channel);
12551ec1e82fSSascha Hauer }
12561ec1e82fSSascha Hauer 
12575b28aa31SSascha Hauer #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1	34
1258cd72b846SNicolin Chen #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2	38
12595b28aa31SSascha Hauer 
12605b28aa31SSascha Hauer static void sdma_add_scripts(struct sdma_engine *sdma,
12615b28aa31SSascha Hauer 		const struct sdma_script_start_addrs *addr)
12625b28aa31SSascha Hauer {
12635b28aa31SSascha Hauer 	s32 *addr_arr = (u32 *)addr;
12645b28aa31SSascha Hauer 	s32 *saddr_arr = (u32 *)sdma->script_addrs;
12655b28aa31SSascha Hauer 	int i;
12665b28aa31SSascha Hauer 
126770dabaedSNicolin Chen 	/* use the default firmware in ROM if missing external firmware */
126870dabaedSNicolin Chen 	if (!sdma->script_number)
126970dabaedSNicolin Chen 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
127070dabaedSNicolin Chen 
1271cd72b846SNicolin Chen 	for (i = 0; i < sdma->script_number; i++)
12725b28aa31SSascha Hauer 		if (addr_arr[i] > 0)
12735b28aa31SSascha Hauer 			saddr_arr[i] = addr_arr[i];
12745b28aa31SSascha Hauer }
12755b28aa31SSascha Hauer 
12767b4b88e0SSascha Hauer static void sdma_load_firmware(const struct firmware *fw, void *context)
12775b28aa31SSascha Hauer {
12787b4b88e0SSascha Hauer 	struct sdma_engine *sdma = context;
12795b28aa31SSascha Hauer 	const struct sdma_firmware_header *header;
12805b28aa31SSascha Hauer 	const struct sdma_script_start_addrs *addr;
12815b28aa31SSascha Hauer 	unsigned short *ram_code;
12825b28aa31SSascha Hauer 
12837b4b88e0SSascha Hauer 	if (!fw) {
12840f927a11SSascha Hauer 		dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
12850f927a11SSascha Hauer 		/* In this case we just use the ROM firmware. */
12867b4b88e0SSascha Hauer 		return;
12877b4b88e0SSascha Hauer 	}
12885b28aa31SSascha Hauer 
12895b28aa31SSascha Hauer 	if (fw->size < sizeof(*header))
12905b28aa31SSascha Hauer 		goto err_firmware;
12915b28aa31SSascha Hauer 
12925b28aa31SSascha Hauer 	header = (struct sdma_firmware_header *)fw->data;
12935b28aa31SSascha Hauer 
12945b28aa31SSascha Hauer 	if (header->magic != SDMA_FIRMWARE_MAGIC)
12955b28aa31SSascha Hauer 		goto err_firmware;
12965b28aa31SSascha Hauer 	if (header->ram_code_start + header->ram_code_size > fw->size)
12975b28aa31SSascha Hauer 		goto err_firmware;
1298cd72b846SNicolin Chen 	switch (header->version_major) {
1299cd72b846SNicolin Chen 		case 1:
1300cd72b846SNicolin Chen 			sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1301cd72b846SNicolin Chen 			break;
1302cd72b846SNicolin Chen 		case 2:
1303cd72b846SNicolin Chen 			sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
1304cd72b846SNicolin Chen 			break;
1305cd72b846SNicolin Chen 		default:
1306cd72b846SNicolin Chen 			dev_err(sdma->dev, "unknown firmware version\n");
1307cd72b846SNicolin Chen 			goto err_firmware;
1308cd72b846SNicolin Chen 	}
13095b28aa31SSascha Hauer 
13105b28aa31SSascha Hauer 	addr = (void *)header + header->script_addrs_start;
13115b28aa31SSascha Hauer 	ram_code = (void *)header + header->ram_code_start;
13125b28aa31SSascha Hauer 
13137560e3f3SSascha Hauer 	clk_enable(sdma->clk_ipg);
13147560e3f3SSascha Hauer 	clk_enable(sdma->clk_ahb);
13155b28aa31SSascha Hauer 	/* download the RAM image for SDMA */
13165b28aa31SSascha Hauer 	sdma_load_script(sdma, ram_code,
13175b28aa31SSascha Hauer 			header->ram_code_size,
13186866fd3bSSascha Hauer 			addr->ram_code_start_addr);
13197560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
13207560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
13215b28aa31SSascha Hauer 
13225b28aa31SSascha Hauer 	sdma_add_scripts(sdma, addr);
13235b28aa31SSascha Hauer 
13245b28aa31SSascha Hauer 	dev_info(sdma->dev, "loaded firmware %d.%d\n",
13255b28aa31SSascha Hauer 			header->version_major,
13265b28aa31SSascha Hauer 			header->version_minor);
13275b28aa31SSascha Hauer 
13285b28aa31SSascha Hauer err_firmware:
13295b28aa31SSascha Hauer 	release_firmware(fw);
13307b4b88e0SSascha Hauer }
13317b4b88e0SSascha Hauer 
1332fe6cf289SArnd Bergmann static int sdma_get_firmware(struct sdma_engine *sdma,
13337b4b88e0SSascha Hauer 		const char *fw_name)
13347b4b88e0SSascha Hauer {
13357b4b88e0SSascha Hauer 	int ret;
13367b4b88e0SSascha Hauer 
13377b4b88e0SSascha Hauer 	ret = request_firmware_nowait(THIS_MODULE,
13387b4b88e0SSascha Hauer 			FW_ACTION_HOTPLUG, fw_name, sdma->dev,
13397b4b88e0SSascha Hauer 			GFP_KERNEL, sdma, sdma_load_firmware);
13405b28aa31SSascha Hauer 
13415b28aa31SSascha Hauer 	return ret;
13425b28aa31SSascha Hauer }
13435b28aa31SSascha Hauer 
134419bfc772SJingoo Han static int sdma_init(struct sdma_engine *sdma)
13451ec1e82fSSascha Hauer {
13461ec1e82fSSascha Hauer 	int i, ret;
13471ec1e82fSSascha Hauer 	dma_addr_t ccb_phys;
13481ec1e82fSSascha Hauer 
13497560e3f3SSascha Hauer 	clk_enable(sdma->clk_ipg);
13507560e3f3SSascha Hauer 	clk_enable(sdma->clk_ahb);
13511ec1e82fSSascha Hauer 
13521ec1e82fSSascha Hauer 	/* Be sure SDMA has not started yet */
1353c4b56857SRichard Zhao 	writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
13541ec1e82fSSascha Hauer 
13551ec1e82fSSascha Hauer 	sdma->channel_control = dma_alloc_coherent(NULL,
13561ec1e82fSSascha Hauer 			MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
13571ec1e82fSSascha Hauer 			sizeof(struct sdma_context_data),
13581ec1e82fSSascha Hauer 			&ccb_phys, GFP_KERNEL);
13591ec1e82fSSascha Hauer 
13601ec1e82fSSascha Hauer 	if (!sdma->channel_control) {
13611ec1e82fSSascha Hauer 		ret = -ENOMEM;
13621ec1e82fSSascha Hauer 		goto err_dma_alloc;
13631ec1e82fSSascha Hauer 	}
13641ec1e82fSSascha Hauer 
13651ec1e82fSSascha Hauer 	sdma->context = (void *)sdma->channel_control +
13661ec1e82fSSascha Hauer 		MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
13671ec1e82fSSascha Hauer 	sdma->context_phys = ccb_phys +
13681ec1e82fSSascha Hauer 		MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
13691ec1e82fSSascha Hauer 
13701ec1e82fSSascha Hauer 	/* Zero-out the CCB structures array just allocated */
13711ec1e82fSSascha Hauer 	memset(sdma->channel_control, 0,
13721ec1e82fSSascha Hauer 			MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control));
13731ec1e82fSSascha Hauer 
13741ec1e82fSSascha Hauer 	/* disable all channels */
137517bba72fSSascha Hauer 	for (i = 0; i < sdma->drvdata->num_events; i++)
1376c4b56857SRichard Zhao 		writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
13771ec1e82fSSascha Hauer 
13781ec1e82fSSascha Hauer 	/* All channels have priority 0 */
13791ec1e82fSSascha Hauer 	for (i = 0; i < MAX_DMA_CHANNELS; i++)
1380c4b56857SRichard Zhao 		writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
13811ec1e82fSSascha Hauer 
13821ec1e82fSSascha Hauer 	ret = sdma_request_channel(&sdma->channel[0]);
13831ec1e82fSSascha Hauer 	if (ret)
13841ec1e82fSSascha Hauer 		goto err_dma_alloc;
13851ec1e82fSSascha Hauer 
13861ec1e82fSSascha Hauer 	sdma_config_ownership(&sdma->channel[0], false, true, false);
13871ec1e82fSSascha Hauer 
13881ec1e82fSSascha Hauer 	/* Set Command Channel (Channel Zero) */
1389c4b56857SRichard Zhao 	writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
13901ec1e82fSSascha Hauer 
13911ec1e82fSSascha Hauer 	/* Set bits of CONFIG register but with static context switching */
13921ec1e82fSSascha Hauer 	/* FIXME: Check whether to set ACR bit depending on clock ratios */
1393c4b56857SRichard Zhao 	writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
13941ec1e82fSSascha Hauer 
1395c4b56857SRichard Zhao 	writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
13961ec1e82fSSascha Hauer 
13971ec1e82fSSascha Hauer 	/* Set bits of CONFIG register with given context switching mode */
1398c4b56857SRichard Zhao 	writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG);
13991ec1e82fSSascha Hauer 
14001ec1e82fSSascha Hauer 	/* Initializes channel's priorities */
14011ec1e82fSSascha Hauer 	sdma_set_channel_priority(&sdma->channel[0], 7);
14021ec1e82fSSascha Hauer 
14037560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
14047560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
14051ec1e82fSSascha Hauer 
14061ec1e82fSSascha Hauer 	return 0;
14071ec1e82fSSascha Hauer 
14081ec1e82fSSascha Hauer err_dma_alloc:
14097560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
14107560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
14111ec1e82fSSascha Hauer 	dev_err(sdma->dev, "initialisation failed with %d\n", ret);
14121ec1e82fSSascha Hauer 	return ret;
14131ec1e82fSSascha Hauer }
14141ec1e82fSSascha Hauer 
14159479e17cSShawn Guo static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
14169479e17cSShawn Guo {
14170b351865SNicolin Chen 	struct sdma_channel *sdmac = to_sdma_chan(chan);
14189479e17cSShawn Guo 	struct imx_dma_data *data = fn_param;
14199479e17cSShawn Guo 
14209479e17cSShawn Guo 	if (!imx_dma_is_general_purpose(chan))
14219479e17cSShawn Guo 		return false;
14229479e17cSShawn Guo 
14230b351865SNicolin Chen 	sdmac->data = *data;
14240b351865SNicolin Chen 	chan->private = &sdmac->data;
14259479e17cSShawn Guo 
14269479e17cSShawn Guo 	return true;
14279479e17cSShawn Guo }
14289479e17cSShawn Guo 
14299479e17cSShawn Guo static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
14309479e17cSShawn Guo 				   struct of_dma *ofdma)
14319479e17cSShawn Guo {
14329479e17cSShawn Guo 	struct sdma_engine *sdma = ofdma->of_dma_data;
14339479e17cSShawn Guo 	dma_cap_mask_t mask = sdma->dma_device.cap_mask;
14349479e17cSShawn Guo 	struct imx_dma_data data;
14359479e17cSShawn Guo 
14369479e17cSShawn Guo 	if (dma_spec->args_count != 3)
14379479e17cSShawn Guo 		return NULL;
14389479e17cSShawn Guo 
14399479e17cSShawn Guo 	data.dma_request = dma_spec->args[0];
14409479e17cSShawn Guo 	data.peripheral_type = dma_spec->args[1];
14419479e17cSShawn Guo 	data.priority = dma_spec->args[2];
14429479e17cSShawn Guo 
14439479e17cSShawn Guo 	return dma_request_channel(mask, sdma_filter_fn, &data);
14449479e17cSShawn Guo }
14459479e17cSShawn Guo 
1446e34b731fSMark Brown static int sdma_probe(struct platform_device *pdev)
14471ec1e82fSSascha Hauer {
1448580975d7SShawn Guo 	const struct of_device_id *of_id =
1449580975d7SShawn Guo 			of_match_device(sdma_dt_ids, &pdev->dev);
1450580975d7SShawn Guo 	struct device_node *np = pdev->dev.of_node;
1451580975d7SShawn Guo 	const char *fw_name;
14521ec1e82fSSascha Hauer 	int ret;
14531ec1e82fSSascha Hauer 	int irq;
14541ec1e82fSSascha Hauer 	struct resource *iores;
1455d4adcc01SJingoo Han 	struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev);
14561ec1e82fSSascha Hauer 	int i;
14571ec1e82fSSascha Hauer 	struct sdma_engine *sdma;
145836e2f21aSSascha Hauer 	s32 *saddr_arr;
145917bba72fSSascha Hauer 	const struct sdma_driver_data *drvdata = NULL;
146017bba72fSSascha Hauer 
146117bba72fSSascha Hauer 	if (of_id)
146217bba72fSSascha Hauer 		drvdata = of_id->data;
146317bba72fSSascha Hauer 	else if (pdev->id_entry)
146417bba72fSSascha Hauer 		drvdata = (void *)pdev->id_entry->driver_data;
146517bba72fSSascha Hauer 
146617bba72fSSascha Hauer 	if (!drvdata) {
146717bba72fSSascha Hauer 		dev_err(&pdev->dev, "unable to find driver data\n");
146817bba72fSSascha Hauer 		return -EINVAL;
146917bba72fSSascha Hauer 	}
14701ec1e82fSSascha Hauer 
147142536b9fSPhilippe Retornaz 	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
147242536b9fSPhilippe Retornaz 	if (ret)
147342536b9fSPhilippe Retornaz 		return ret;
147442536b9fSPhilippe Retornaz 
14757f24e0eeSFabio Estevam 	sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
14761ec1e82fSSascha Hauer 	if (!sdma)
14771ec1e82fSSascha Hauer 		return -ENOMEM;
14781ec1e82fSSascha Hauer 
14792ccaef05SRichard Zhao 	spin_lock_init(&sdma->channel_0_lock);
148073eab978SSascha Hauer 
14811ec1e82fSSascha Hauer 	sdma->dev = &pdev->dev;
148217bba72fSSascha Hauer 	sdma->drvdata = drvdata;
14831ec1e82fSSascha Hauer 
14841ec1e82fSSascha Hauer 	irq = platform_get_irq(pdev, 0);
14857f24e0eeSFabio Estevam 	if (irq < 0)
1486*63c72e02SFabio Estevam 		return irq;
14871ec1e82fSSascha Hauer 
14887f24e0eeSFabio Estevam 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
14897f24e0eeSFabio Estevam 	sdma->regs = devm_ioremap_resource(&pdev->dev, iores);
14907f24e0eeSFabio Estevam 	if (IS_ERR(sdma->regs))
14917f24e0eeSFabio Estevam 		return PTR_ERR(sdma->regs);
14921ec1e82fSSascha Hauer 
14937560e3f3SSascha Hauer 	sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
14947f24e0eeSFabio Estevam 	if (IS_ERR(sdma->clk_ipg))
14957f24e0eeSFabio Estevam 		return PTR_ERR(sdma->clk_ipg);
14961ec1e82fSSascha Hauer 
14977560e3f3SSascha Hauer 	sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
14987f24e0eeSFabio Estevam 	if (IS_ERR(sdma->clk_ahb))
14997f24e0eeSFabio Estevam 		return PTR_ERR(sdma->clk_ahb);
15007560e3f3SSascha Hauer 
15017560e3f3SSascha Hauer 	clk_prepare(sdma->clk_ipg);
15027560e3f3SSascha Hauer 	clk_prepare(sdma->clk_ahb);
15037560e3f3SSascha Hauer 
15047f24e0eeSFabio Estevam 	ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
15057f24e0eeSFabio Estevam 			       sdma);
15061ec1e82fSSascha Hauer 	if (ret)
15077f24e0eeSFabio Estevam 		return ret;
15081ec1e82fSSascha Hauer 
15095b28aa31SSascha Hauer 	sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
15107f24e0eeSFabio Estevam 	if (!sdma->script_addrs)
15117f24e0eeSFabio Estevam 		return -ENOMEM;
15121ec1e82fSSascha Hauer 
151336e2f21aSSascha Hauer 	/* initially no scripts available */
151436e2f21aSSascha Hauer 	saddr_arr = (s32 *)sdma->script_addrs;
151536e2f21aSSascha Hauer 	for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++)
151636e2f21aSSascha Hauer 		saddr_arr[i] = -EINVAL;
151736e2f21aSSascha Hauer 
15187214a8b1SSascha Hauer 	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
15197214a8b1SSascha Hauer 	dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
15207214a8b1SSascha Hauer 
15211ec1e82fSSascha Hauer 	INIT_LIST_HEAD(&sdma->dma_device.channels);
15221ec1e82fSSascha Hauer 	/* Initialize channel parameters */
15231ec1e82fSSascha Hauer 	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
15241ec1e82fSSascha Hauer 		struct sdma_channel *sdmac = &sdma->channel[i];
15251ec1e82fSSascha Hauer 
15261ec1e82fSSascha Hauer 		sdmac->sdma = sdma;
15271ec1e82fSSascha Hauer 		spin_lock_init(&sdmac->lock);
15281ec1e82fSSascha Hauer 
15291ec1e82fSSascha Hauer 		sdmac->chan.device = &sdma->dma_device;
15308ac69546SRussell King - ARM Linux 		dma_cookie_init(&sdmac->chan);
15311ec1e82fSSascha Hauer 		sdmac->channel = i;
15321ec1e82fSSascha Hauer 
1533abd9ccc8SHuang Shijie 		tasklet_init(&sdmac->tasklet, sdma_tasklet,
1534abd9ccc8SHuang Shijie 			     (unsigned long) sdmac);
153523889c63SSascha Hauer 		/*
153623889c63SSascha Hauer 		 * Add the channel to the DMAC list. Do not add channel 0 though
153723889c63SSascha Hauer 		 * because we need it internally in the SDMA driver. This also means
153823889c63SSascha Hauer 		 * that channel 0 in dmaengine counting matches sdma channel 1.
153923889c63SSascha Hauer 		 */
154023889c63SSascha Hauer 		if (i)
154123889c63SSascha Hauer 			list_add_tail(&sdmac->chan.device_node,
154223889c63SSascha Hauer 					&sdma->dma_device.channels);
15431ec1e82fSSascha Hauer 	}
15441ec1e82fSSascha Hauer 
15455b28aa31SSascha Hauer 	ret = sdma_init(sdma);
15461ec1e82fSSascha Hauer 	if (ret)
15471ec1e82fSSascha Hauer 		goto err_init;
15481ec1e82fSSascha Hauer 
1549dcfec3c0SSascha Hauer 	if (sdma->drvdata->script_addrs)
1550dcfec3c0SSascha Hauer 		sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
1551580975d7SShawn Guo 	if (pdata && pdata->script_addrs)
15525b28aa31SSascha Hauer 		sdma_add_scripts(sdma, pdata->script_addrs);
15535b28aa31SSascha Hauer 
1554580975d7SShawn Guo 	if (pdata) {
15556d0d7e2dSFabio Estevam 		ret = sdma_get_firmware(sdma, pdata->fw_name);
15566d0d7e2dSFabio Estevam 		if (ret)
1557ad1122e5SFabio Estevam 			dev_warn(&pdev->dev, "failed to get firmware from platform data\n");
1558580975d7SShawn Guo 	} else {
1559580975d7SShawn Guo 		/*
1560580975d7SShawn Guo 		 * Because that device tree does not encode ROM script address,
1561580975d7SShawn Guo 		 * the RAM script in firmware is mandatory for device tree
1562580975d7SShawn Guo 		 * probe, otherwise it fails.
1563580975d7SShawn Guo 		 */
1564580975d7SShawn Guo 		ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
1565580975d7SShawn Guo 					      &fw_name);
15666602b0ddSFabio Estevam 		if (ret)
1567ad1122e5SFabio Estevam 			dev_warn(&pdev->dev, "failed to get firmware name\n");
15686602b0ddSFabio Estevam 		else {
1569580975d7SShawn Guo 			ret = sdma_get_firmware(sdma, fw_name);
15706602b0ddSFabio Estevam 			if (ret)
1571ad1122e5SFabio Estevam 				dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
1572580975d7SShawn Guo 		}
1573580975d7SShawn Guo 	}
15745b28aa31SSascha Hauer 
15751ec1e82fSSascha Hauer 	sdma->dma_device.dev = &pdev->dev;
15761ec1e82fSSascha Hauer 
15771ec1e82fSSascha Hauer 	sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
15781ec1e82fSSascha Hauer 	sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
15791ec1e82fSSascha Hauer 	sdma->dma_device.device_tx_status = sdma_tx_status;
15801ec1e82fSSascha Hauer 	sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
15811ec1e82fSSascha Hauer 	sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
15827b350ab0SMaxime Ripard 	sdma->dma_device.device_config = sdma_config;
15837b350ab0SMaxime Ripard 	sdma->dma_device.device_terminate_all = sdma_disable_channel;
15841e4a4f50SFabio Estevam 	sdma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
15851e4a4f50SFabio Estevam 	sdma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
15861e4a4f50SFabio Estevam 	sdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
15871e4a4f50SFabio Estevam 	sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
15881ec1e82fSSascha Hauer 	sdma->dma_device.device_issue_pending = sdma_issue_pending;
1589b9b3f82fSSascha Hauer 	sdma->dma_device.dev->dma_parms = &sdma->dma_parms;
1590b9b3f82fSSascha Hauer 	dma_set_max_seg_size(sdma->dma_device.dev, 65535);
15911ec1e82fSSascha Hauer 
159223e11811SVignesh Raman 	platform_set_drvdata(pdev, sdma);
159323e11811SVignesh Raman 
15941ec1e82fSSascha Hauer 	ret = dma_async_device_register(&sdma->dma_device);
15951ec1e82fSSascha Hauer 	if (ret) {
15961ec1e82fSSascha Hauer 		dev_err(&pdev->dev, "unable to register\n");
15971ec1e82fSSascha Hauer 		goto err_init;
15981ec1e82fSSascha Hauer 	}
15991ec1e82fSSascha Hauer 
16009479e17cSShawn Guo 	if (np) {
16019479e17cSShawn Guo 		ret = of_dma_controller_register(np, sdma_xlate, sdma);
16029479e17cSShawn Guo 		if (ret) {
16039479e17cSShawn Guo 			dev_err(&pdev->dev, "failed to register controller\n");
16049479e17cSShawn Guo 			goto err_register;
16059479e17cSShawn Guo 		}
16069479e17cSShawn Guo 	}
16079479e17cSShawn Guo 
16085b28aa31SSascha Hauer 	dev_info(sdma->dev, "initialized\n");
16091ec1e82fSSascha Hauer 
16101ec1e82fSSascha Hauer 	return 0;
16111ec1e82fSSascha Hauer 
16129479e17cSShawn Guo err_register:
16139479e17cSShawn Guo 	dma_async_device_unregister(&sdma->dma_device);
16141ec1e82fSSascha Hauer err_init:
16151ec1e82fSSascha Hauer 	kfree(sdma->script_addrs);
1616939fd4f0SShawn Guo 	return ret;
16171ec1e82fSSascha Hauer }
16181ec1e82fSSascha Hauer 
16191d1bbd30SMaxin B. John static int sdma_remove(struct platform_device *pdev)
16201ec1e82fSSascha Hauer {
162123e11811SVignesh Raman 	struct sdma_engine *sdma = platform_get_drvdata(pdev);
1622c12fe497SVignesh Raman 	int i;
162323e11811SVignesh Raman 
162423e11811SVignesh Raman 	dma_async_device_unregister(&sdma->dma_device);
162523e11811SVignesh Raman 	kfree(sdma->script_addrs);
1626c12fe497SVignesh Raman 	/* Kill the tasklet */
1627c12fe497SVignesh Raman 	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
1628c12fe497SVignesh Raman 		struct sdma_channel *sdmac = &sdma->channel[i];
1629c12fe497SVignesh Raman 
1630c12fe497SVignesh Raman 		tasklet_kill(&sdmac->tasklet);
1631c12fe497SVignesh Raman 	}
163223e11811SVignesh Raman 
163323e11811SVignesh Raman 	platform_set_drvdata(pdev, NULL);
163423e11811SVignesh Raman 	dev_info(&pdev->dev, "Removed...\n");
163523e11811SVignesh Raman 	return 0;
16361ec1e82fSSascha Hauer }
16371ec1e82fSSascha Hauer 
16381ec1e82fSSascha Hauer static struct platform_driver sdma_driver = {
16391ec1e82fSSascha Hauer 	.driver		= {
16401ec1e82fSSascha Hauer 		.name	= "imx-sdma",
1641580975d7SShawn Guo 		.of_match_table = sdma_dt_ids,
16421ec1e82fSSascha Hauer 	},
164362550cd7SShawn Guo 	.id_table	= sdma_devtypes,
16441d1bbd30SMaxin B. John 	.remove		= sdma_remove,
164523e11811SVignesh Raman 	.probe		= sdma_probe,
16461ec1e82fSSascha Hauer };
16471ec1e82fSSascha Hauer 
164823e11811SVignesh Raman module_platform_driver(sdma_driver);
16491ec1e82fSSascha Hauer 
16501ec1e82fSSascha Hauer MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
16511ec1e82fSSascha Hauer MODULE_DESCRIPTION("i.MX SDMA driver");
16521ec1e82fSSascha Hauer MODULE_LICENSE("GPL");
1653