xref: /linux/drivers/dma/imx-sdma.c (revision ec8b5e48c03790a68cb875fe5064007a9cbdfdd0)
11ec1e82fSSascha Hauer /*
21ec1e82fSSascha Hauer  * drivers/dma/imx-sdma.c
31ec1e82fSSascha Hauer  *
41ec1e82fSSascha Hauer  * This file contains a driver for the Freescale Smart DMA engine
51ec1e82fSSascha Hauer  *
61ec1e82fSSascha Hauer  * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
71ec1e82fSSascha Hauer  *
81ec1e82fSSascha Hauer  * Based on code from Freescale:
91ec1e82fSSascha Hauer  *
101ec1e82fSSascha Hauer  * Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
111ec1e82fSSascha Hauer  *
121ec1e82fSSascha Hauer  * The code contained herein is licensed under the GNU General Public
131ec1e82fSSascha Hauer  * License. You may obtain a copy of the GNU General Public License
141ec1e82fSSascha Hauer  * Version 2 or later at the following locations:
151ec1e82fSSascha Hauer  *
161ec1e82fSSascha Hauer  * http://www.opensource.org/licenses/gpl-license.html
171ec1e82fSSascha Hauer  * http://www.gnu.org/copyleft/gpl.html
181ec1e82fSSascha Hauer  */
191ec1e82fSSascha Hauer 
201ec1e82fSSascha Hauer #include <linux/init.h>
21f8de8f4cSAxel Lin #include <linux/module.h>
221ec1e82fSSascha Hauer #include <linux/types.h>
230bbc1413SRichard Zhao #include <linux/bitops.h>
241ec1e82fSSascha Hauer #include <linux/mm.h>
251ec1e82fSSascha Hauer #include <linux/interrupt.h>
261ec1e82fSSascha Hauer #include <linux/clk.h>
272ccaef05SRichard Zhao #include <linux/delay.h>
281ec1e82fSSascha Hauer #include <linux/sched.h>
291ec1e82fSSascha Hauer #include <linux/semaphore.h>
301ec1e82fSSascha Hauer #include <linux/spinlock.h>
311ec1e82fSSascha Hauer #include <linux/device.h>
321ec1e82fSSascha Hauer #include <linux/dma-mapping.h>
331ec1e82fSSascha Hauer #include <linux/firmware.h>
341ec1e82fSSascha Hauer #include <linux/slab.h>
351ec1e82fSSascha Hauer #include <linux/platform_device.h>
361ec1e82fSSascha Hauer #include <linux/dmaengine.h>
37580975d7SShawn Guo #include <linux/of.h>
38580975d7SShawn Guo #include <linux/of_device.h>
391ec1e82fSSascha Hauer 
401ec1e82fSSascha Hauer #include <asm/irq.h>
411ec1e82fSSascha Hauer #include <mach/sdma.h>
421ec1e82fSSascha Hauer #include <mach/dma.h>
431ec1e82fSSascha Hauer #include <mach/hardware.h>
441ec1e82fSSascha Hauer 
45d2ebfb33SRussell King - ARM Linux #include "dmaengine.h"
46d2ebfb33SRussell King - ARM Linux 
471ec1e82fSSascha Hauer /* SDMA registers */
481ec1e82fSSascha Hauer #define SDMA_H_C0PTR		0x000
491ec1e82fSSascha Hauer #define SDMA_H_INTR		0x004
501ec1e82fSSascha Hauer #define SDMA_H_STATSTOP		0x008
511ec1e82fSSascha Hauer #define SDMA_H_START		0x00c
521ec1e82fSSascha Hauer #define SDMA_H_EVTOVR		0x010
531ec1e82fSSascha Hauer #define SDMA_H_DSPOVR		0x014
541ec1e82fSSascha Hauer #define SDMA_H_HOSTOVR		0x018
551ec1e82fSSascha Hauer #define SDMA_H_EVTPEND		0x01c
561ec1e82fSSascha Hauer #define SDMA_H_DSPENBL		0x020
571ec1e82fSSascha Hauer #define SDMA_H_RESET		0x024
581ec1e82fSSascha Hauer #define SDMA_H_EVTERR		0x028
591ec1e82fSSascha Hauer #define SDMA_H_INTRMSK		0x02c
601ec1e82fSSascha Hauer #define SDMA_H_PSW		0x030
611ec1e82fSSascha Hauer #define SDMA_H_EVTERRDBG	0x034
621ec1e82fSSascha Hauer #define SDMA_H_CONFIG		0x038
631ec1e82fSSascha Hauer #define SDMA_ONCE_ENB		0x040
641ec1e82fSSascha Hauer #define SDMA_ONCE_DATA		0x044
651ec1e82fSSascha Hauer #define SDMA_ONCE_INSTR		0x048
661ec1e82fSSascha Hauer #define SDMA_ONCE_STAT		0x04c
671ec1e82fSSascha Hauer #define SDMA_ONCE_CMD		0x050
681ec1e82fSSascha Hauer #define SDMA_EVT_MIRROR		0x054
691ec1e82fSSascha Hauer #define SDMA_ILLINSTADDR	0x058
701ec1e82fSSascha Hauer #define SDMA_CHN0ADDR		0x05c
711ec1e82fSSascha Hauer #define SDMA_ONCE_RTB		0x060
721ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF1	0x070
731ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF2	0x074
7462550cd7SShawn Guo #define SDMA_CHNENBL0_IMX35	0x200
7562550cd7SShawn Guo #define SDMA_CHNENBL0_IMX31	0x080
761ec1e82fSSascha Hauer #define SDMA_CHNPRI_0		0x100
771ec1e82fSSascha Hauer 
781ec1e82fSSascha Hauer /*
791ec1e82fSSascha Hauer  * Buffer descriptor status values.
801ec1e82fSSascha Hauer  */
811ec1e82fSSascha Hauer #define BD_DONE  0x01
821ec1e82fSSascha Hauer #define BD_WRAP  0x02
831ec1e82fSSascha Hauer #define BD_CONT  0x04
841ec1e82fSSascha Hauer #define BD_INTR  0x08
851ec1e82fSSascha Hauer #define BD_RROR  0x10
861ec1e82fSSascha Hauer #define BD_LAST  0x20
871ec1e82fSSascha Hauer #define BD_EXTD  0x80
881ec1e82fSSascha Hauer 
891ec1e82fSSascha Hauer /*
901ec1e82fSSascha Hauer  * Data Node descriptor status values.
911ec1e82fSSascha Hauer  */
921ec1e82fSSascha Hauer #define DND_END_OF_FRAME  0x80
931ec1e82fSSascha Hauer #define DND_END_OF_XFER   0x40
941ec1e82fSSascha Hauer #define DND_DONE          0x20
951ec1e82fSSascha Hauer #define DND_UNUSED        0x01
961ec1e82fSSascha Hauer 
971ec1e82fSSascha Hauer /*
981ec1e82fSSascha Hauer  * IPCV2 descriptor status values.
991ec1e82fSSascha Hauer  */
1001ec1e82fSSascha Hauer #define BD_IPCV2_END_OF_FRAME  0x40
1011ec1e82fSSascha Hauer 
1021ec1e82fSSascha Hauer #define IPCV2_MAX_NODES        50
1031ec1e82fSSascha Hauer /*
1041ec1e82fSSascha Hauer  * Error bit set in the CCB status field by the SDMA,
1051ec1e82fSSascha Hauer  * in setbd routine, in case of a transfer error
1061ec1e82fSSascha Hauer  */
1071ec1e82fSSascha Hauer #define DATA_ERROR  0x10000000
1081ec1e82fSSascha Hauer 
1091ec1e82fSSascha Hauer /*
1101ec1e82fSSascha Hauer  * Buffer descriptor commands.
1111ec1e82fSSascha Hauer  */
1121ec1e82fSSascha Hauer #define C0_ADDR             0x01
1131ec1e82fSSascha Hauer #define C0_LOAD             0x02
1141ec1e82fSSascha Hauer #define C0_DUMP             0x03
1151ec1e82fSSascha Hauer #define C0_SETCTX           0x07
1161ec1e82fSSascha Hauer #define C0_GETCTX           0x03
1171ec1e82fSSascha Hauer #define C0_SETDM            0x01
1181ec1e82fSSascha Hauer #define C0_SETPM            0x04
1191ec1e82fSSascha Hauer #define C0_GETDM            0x02
1201ec1e82fSSascha Hauer #define C0_GETPM            0x08
1211ec1e82fSSascha Hauer /*
1221ec1e82fSSascha Hauer  * Change endianness indicator in the BD command field
1231ec1e82fSSascha Hauer  */
1241ec1e82fSSascha Hauer #define CHANGE_ENDIANNESS   0x80
1251ec1e82fSSascha Hauer 
1261ec1e82fSSascha Hauer /*
1271ec1e82fSSascha Hauer  * Mode/Count of data node descriptors - IPCv2
1281ec1e82fSSascha Hauer  */
1291ec1e82fSSascha Hauer struct sdma_mode_count {
1301ec1e82fSSascha Hauer 	u32 count   : 16; /* size of the buffer pointed by this BD */
1311ec1e82fSSascha Hauer 	u32 status  :  8; /* E,R,I,C,W,D status bits stored here */
1321ec1e82fSSascha Hauer 	u32 command :  8; /* command mostlky used for channel 0 */
1331ec1e82fSSascha Hauer };
1341ec1e82fSSascha Hauer 
1351ec1e82fSSascha Hauer /*
1361ec1e82fSSascha Hauer  * Buffer descriptor
1371ec1e82fSSascha Hauer  */
1381ec1e82fSSascha Hauer struct sdma_buffer_descriptor {
1391ec1e82fSSascha Hauer 	struct sdma_mode_count  mode;
1401ec1e82fSSascha Hauer 	u32 buffer_addr;	/* address of the buffer described */
1411ec1e82fSSascha Hauer 	u32 ext_buffer_addr;	/* extended buffer address */
1421ec1e82fSSascha Hauer } __attribute__ ((packed));
1431ec1e82fSSascha Hauer 
1441ec1e82fSSascha Hauer /**
1451ec1e82fSSascha Hauer  * struct sdma_channel_control - Channel control Block
1461ec1e82fSSascha Hauer  *
1471ec1e82fSSascha Hauer  * @current_bd_ptr	current buffer descriptor processed
1481ec1e82fSSascha Hauer  * @base_bd_ptr		first element of buffer descriptor array
1491ec1e82fSSascha Hauer  * @unused		padding. The SDMA engine expects an array of 128 byte
1501ec1e82fSSascha Hauer  *			control blocks
1511ec1e82fSSascha Hauer  */
1521ec1e82fSSascha Hauer struct sdma_channel_control {
1531ec1e82fSSascha Hauer 	u32 current_bd_ptr;
1541ec1e82fSSascha Hauer 	u32 base_bd_ptr;
1551ec1e82fSSascha Hauer 	u32 unused[2];
1561ec1e82fSSascha Hauer } __attribute__ ((packed));
1571ec1e82fSSascha Hauer 
1581ec1e82fSSascha Hauer /**
1591ec1e82fSSascha Hauer  * struct sdma_state_registers - SDMA context for a channel
1601ec1e82fSSascha Hauer  *
1611ec1e82fSSascha Hauer  * @pc:		program counter
1621ec1e82fSSascha Hauer  * @t:		test bit: status of arithmetic & test instruction
1631ec1e82fSSascha Hauer  * @rpc:	return program counter
1641ec1e82fSSascha Hauer  * @sf:		source fault while loading data
1651ec1e82fSSascha Hauer  * @spc:	loop start program counter
1661ec1e82fSSascha Hauer  * @df:		destination fault while storing data
1671ec1e82fSSascha Hauer  * @epc:	loop end program counter
1681ec1e82fSSascha Hauer  * @lm:		loop mode
1691ec1e82fSSascha Hauer  */
1701ec1e82fSSascha Hauer struct sdma_state_registers {
1711ec1e82fSSascha Hauer 	u32 pc     :14;
1721ec1e82fSSascha Hauer 	u32 unused1: 1;
1731ec1e82fSSascha Hauer 	u32 t      : 1;
1741ec1e82fSSascha Hauer 	u32 rpc    :14;
1751ec1e82fSSascha Hauer 	u32 unused0: 1;
1761ec1e82fSSascha Hauer 	u32 sf     : 1;
1771ec1e82fSSascha Hauer 	u32 spc    :14;
1781ec1e82fSSascha Hauer 	u32 unused2: 1;
1791ec1e82fSSascha Hauer 	u32 df     : 1;
1801ec1e82fSSascha Hauer 	u32 epc    :14;
1811ec1e82fSSascha Hauer 	u32 lm     : 2;
1821ec1e82fSSascha Hauer } __attribute__ ((packed));
1831ec1e82fSSascha Hauer 
1841ec1e82fSSascha Hauer /**
1851ec1e82fSSascha Hauer  * struct sdma_context_data - sdma context specific to a channel
1861ec1e82fSSascha Hauer  *
1871ec1e82fSSascha Hauer  * @channel_state:	channel state bits
1881ec1e82fSSascha Hauer  * @gReg:		general registers
1891ec1e82fSSascha Hauer  * @mda:		burst dma destination address register
1901ec1e82fSSascha Hauer  * @msa:		burst dma source address register
1911ec1e82fSSascha Hauer  * @ms:			burst dma status register
1921ec1e82fSSascha Hauer  * @md:			burst dma data register
1931ec1e82fSSascha Hauer  * @pda:		peripheral dma destination address register
1941ec1e82fSSascha Hauer  * @psa:		peripheral dma source address register
1951ec1e82fSSascha Hauer  * @ps:			peripheral dma status register
1961ec1e82fSSascha Hauer  * @pd:			peripheral dma data register
1971ec1e82fSSascha Hauer  * @ca:			CRC polynomial register
1981ec1e82fSSascha Hauer  * @cs:			CRC accumulator register
1991ec1e82fSSascha Hauer  * @dda:		dedicated core destination address register
2001ec1e82fSSascha Hauer  * @dsa:		dedicated core source address register
2011ec1e82fSSascha Hauer  * @ds:			dedicated core status register
2021ec1e82fSSascha Hauer  * @dd:			dedicated core data register
2031ec1e82fSSascha Hauer  */
2041ec1e82fSSascha Hauer struct sdma_context_data {
2051ec1e82fSSascha Hauer 	struct sdma_state_registers  channel_state;
2061ec1e82fSSascha Hauer 	u32  gReg[8];
2071ec1e82fSSascha Hauer 	u32  mda;
2081ec1e82fSSascha Hauer 	u32  msa;
2091ec1e82fSSascha Hauer 	u32  ms;
2101ec1e82fSSascha Hauer 	u32  md;
2111ec1e82fSSascha Hauer 	u32  pda;
2121ec1e82fSSascha Hauer 	u32  psa;
2131ec1e82fSSascha Hauer 	u32  ps;
2141ec1e82fSSascha Hauer 	u32  pd;
2151ec1e82fSSascha Hauer 	u32  ca;
2161ec1e82fSSascha Hauer 	u32  cs;
2171ec1e82fSSascha Hauer 	u32  dda;
2181ec1e82fSSascha Hauer 	u32  dsa;
2191ec1e82fSSascha Hauer 	u32  ds;
2201ec1e82fSSascha Hauer 	u32  dd;
2211ec1e82fSSascha Hauer 	u32  scratch0;
2221ec1e82fSSascha Hauer 	u32  scratch1;
2231ec1e82fSSascha Hauer 	u32  scratch2;
2241ec1e82fSSascha Hauer 	u32  scratch3;
2251ec1e82fSSascha Hauer 	u32  scratch4;
2261ec1e82fSSascha Hauer 	u32  scratch5;
2271ec1e82fSSascha Hauer 	u32  scratch6;
2281ec1e82fSSascha Hauer 	u32  scratch7;
2291ec1e82fSSascha Hauer } __attribute__ ((packed));
2301ec1e82fSSascha Hauer 
2311ec1e82fSSascha Hauer #define NUM_BD (int)(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor))
2321ec1e82fSSascha Hauer 
2331ec1e82fSSascha Hauer struct sdma_engine;
2341ec1e82fSSascha Hauer 
2351ec1e82fSSascha Hauer /**
2361ec1e82fSSascha Hauer  * struct sdma_channel - housekeeping for a SDMA channel
2371ec1e82fSSascha Hauer  *
2381ec1e82fSSascha Hauer  * @sdma		pointer to the SDMA engine for this channel
23923889c63SSascha Hauer  * @channel		the channel number, matches dmaengine chan_id + 1
2401ec1e82fSSascha Hauer  * @direction		transfer type. Needed for setting SDMA script
2411ec1e82fSSascha Hauer  * @peripheral_type	Peripheral type. Needed for setting SDMA script
2421ec1e82fSSascha Hauer  * @event_id0		aka dma request line
2431ec1e82fSSascha Hauer  * @event_id1		for channels that use 2 events
2441ec1e82fSSascha Hauer  * @word_size		peripheral access size
2451ec1e82fSSascha Hauer  * @buf_tail		ID of the buffer that was processed
2461ec1e82fSSascha Hauer  * @done		channel completion
2471ec1e82fSSascha Hauer  * @num_bd		max NUM_BD. number of descriptors currently handling
2481ec1e82fSSascha Hauer  */
2491ec1e82fSSascha Hauer struct sdma_channel {
2501ec1e82fSSascha Hauer 	struct sdma_engine		*sdma;
2511ec1e82fSSascha Hauer 	unsigned int			channel;
252db8196dfSVinod Koul 	enum dma_transfer_direction		direction;
2531ec1e82fSSascha Hauer 	enum sdma_peripheral_type	peripheral_type;
2541ec1e82fSSascha Hauer 	unsigned int			event_id0;
2551ec1e82fSSascha Hauer 	unsigned int			event_id1;
2561ec1e82fSSascha Hauer 	enum dma_slave_buswidth		word_size;
2571ec1e82fSSascha Hauer 	unsigned int			buf_tail;
2581ec1e82fSSascha Hauer 	struct completion		done;
2591ec1e82fSSascha Hauer 	unsigned int			num_bd;
2601ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor	*bd;
2611ec1e82fSSascha Hauer 	dma_addr_t			bd_phys;
2621ec1e82fSSascha Hauer 	unsigned int			pc_from_device, pc_to_device;
2631ec1e82fSSascha Hauer 	unsigned long			flags;
2641ec1e82fSSascha Hauer 	dma_addr_t			per_address;
2650bbc1413SRichard Zhao 	unsigned long			event_mask[2];
2660bbc1413SRichard Zhao 	unsigned long			watermark_level;
2671ec1e82fSSascha Hauer 	u32				shp_addr, per_addr;
2681ec1e82fSSascha Hauer 	struct dma_chan			chan;
2691ec1e82fSSascha Hauer 	spinlock_t			lock;
2701ec1e82fSSascha Hauer 	struct dma_async_tx_descriptor	desc;
2711ec1e82fSSascha Hauer 	enum dma_status			status;
272ab59a510SHuang Shijie 	unsigned int			chn_count;
273ab59a510SHuang Shijie 	unsigned int			chn_real_count;
274abd9ccc8SHuang Shijie 	struct tasklet_struct		tasklet;
2751ec1e82fSSascha Hauer };
2761ec1e82fSSascha Hauer 
2770bbc1413SRichard Zhao #define IMX_DMA_SG_LOOP		BIT(0)
2781ec1e82fSSascha Hauer 
2791ec1e82fSSascha Hauer #define MAX_DMA_CHANNELS 32
2801ec1e82fSSascha Hauer #define MXC_SDMA_DEFAULT_PRIORITY 1
2811ec1e82fSSascha Hauer #define MXC_SDMA_MIN_PRIORITY 1
2821ec1e82fSSascha Hauer #define MXC_SDMA_MAX_PRIORITY 7
2831ec1e82fSSascha Hauer 
2841ec1e82fSSascha Hauer #define SDMA_FIRMWARE_MAGIC 0x414d4453
2851ec1e82fSSascha Hauer 
2861ec1e82fSSascha Hauer /**
2871ec1e82fSSascha Hauer  * struct sdma_firmware_header - Layout of the firmware image
2881ec1e82fSSascha Hauer  *
2891ec1e82fSSascha Hauer  * @magic		"SDMA"
2901ec1e82fSSascha Hauer  * @version_major	increased whenever layout of struct sdma_script_start_addrs
2911ec1e82fSSascha Hauer  *			changes.
2921ec1e82fSSascha Hauer  * @version_minor	firmware minor version (for binary compatible changes)
2931ec1e82fSSascha Hauer  * @script_addrs_start	offset of struct sdma_script_start_addrs in this image
2941ec1e82fSSascha Hauer  * @num_script_addrs	Number of script addresses in this image
2951ec1e82fSSascha Hauer  * @ram_code_start	offset of SDMA ram image in this firmware image
2961ec1e82fSSascha Hauer  * @ram_code_size	size of SDMA ram image
2971ec1e82fSSascha Hauer  * @script_addrs	Stores the start address of the SDMA scripts
2981ec1e82fSSascha Hauer  *			(in SDMA memory space)
2991ec1e82fSSascha Hauer  */
3001ec1e82fSSascha Hauer struct sdma_firmware_header {
3011ec1e82fSSascha Hauer 	u32	magic;
3021ec1e82fSSascha Hauer 	u32	version_major;
3031ec1e82fSSascha Hauer 	u32	version_minor;
3041ec1e82fSSascha Hauer 	u32	script_addrs_start;
3051ec1e82fSSascha Hauer 	u32	num_script_addrs;
3061ec1e82fSSascha Hauer 	u32	ram_code_start;
3071ec1e82fSSascha Hauer 	u32	ram_code_size;
3081ec1e82fSSascha Hauer };
3091ec1e82fSSascha Hauer 
31062550cd7SShawn Guo enum sdma_devtype {
31162550cd7SShawn Guo 	IMX31_SDMA,	/* runs on i.mx31 */
31262550cd7SShawn Guo 	IMX35_SDMA,	/* runs on i.mx35 and later */
31362550cd7SShawn Guo };
31462550cd7SShawn Guo 
3151ec1e82fSSascha Hauer struct sdma_engine {
3161ec1e82fSSascha Hauer 	struct device			*dev;
317b9b3f82fSSascha Hauer 	struct device_dma_parameters	dma_parms;
3181ec1e82fSSascha Hauer 	struct sdma_channel		channel[MAX_DMA_CHANNELS];
3191ec1e82fSSascha Hauer 	struct sdma_channel_control	*channel_control;
3201ec1e82fSSascha Hauer 	void __iomem			*regs;
32162550cd7SShawn Guo 	enum sdma_devtype		devtype;
3221ec1e82fSSascha Hauer 	unsigned int			num_events;
3231ec1e82fSSascha Hauer 	struct sdma_context_data	*context;
3241ec1e82fSSascha Hauer 	dma_addr_t			context_phys;
3251ec1e82fSSascha Hauer 	struct dma_device		dma_device;
3267560e3f3SSascha Hauer 	struct clk			*clk_ipg;
3277560e3f3SSascha Hauer 	struct clk			*clk_ahb;
3282ccaef05SRichard Zhao 	spinlock_t			channel_0_lock;
3291ec1e82fSSascha Hauer 	struct sdma_script_start_addrs	*script_addrs;
3301ec1e82fSSascha Hauer };
3311ec1e82fSSascha Hauer 
33262550cd7SShawn Guo static struct platform_device_id sdma_devtypes[] = {
33362550cd7SShawn Guo 	{
33462550cd7SShawn Guo 		.name = "imx31-sdma",
33562550cd7SShawn Guo 		.driver_data = IMX31_SDMA,
33662550cd7SShawn Guo 	}, {
33762550cd7SShawn Guo 		.name = "imx35-sdma",
33862550cd7SShawn Guo 		.driver_data = IMX35_SDMA,
33962550cd7SShawn Guo 	}, {
34062550cd7SShawn Guo 		/* sentinel */
34162550cd7SShawn Guo 	}
34262550cd7SShawn Guo };
34362550cd7SShawn Guo MODULE_DEVICE_TABLE(platform, sdma_devtypes);
34462550cd7SShawn Guo 
345580975d7SShawn Guo static const struct of_device_id sdma_dt_ids[] = {
346580975d7SShawn Guo 	{ .compatible = "fsl,imx31-sdma", .data = &sdma_devtypes[IMX31_SDMA], },
347580975d7SShawn Guo 	{ .compatible = "fsl,imx35-sdma", .data = &sdma_devtypes[IMX35_SDMA], },
348580975d7SShawn Guo 	{ /* sentinel */ }
349580975d7SShawn Guo };
350580975d7SShawn Guo MODULE_DEVICE_TABLE(of, sdma_dt_ids);
351580975d7SShawn Guo 
3520bbc1413SRichard Zhao #define SDMA_H_CONFIG_DSPDMA	BIT(12) /* indicates if the DSPDMA is used */
3530bbc1413SRichard Zhao #define SDMA_H_CONFIG_RTD_PINS	BIT(11) /* indicates if Real-Time Debug pins are enabled */
3540bbc1413SRichard Zhao #define SDMA_H_CONFIG_ACR	BIT(4)  /* indicates if AHB freq /core freq = 2 or 1 */
3551ec1e82fSSascha Hauer #define SDMA_H_CONFIG_CSM	(3)       /* indicates which context switch mode is selected*/
3561ec1e82fSSascha Hauer 
3571ec1e82fSSascha Hauer static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
3581ec1e82fSSascha Hauer {
35962550cd7SShawn Guo 	u32 chnenbl0 = (sdma->devtype == IMX31_SDMA ? SDMA_CHNENBL0_IMX31 :
36062550cd7SShawn Guo 						      SDMA_CHNENBL0_IMX35);
3611ec1e82fSSascha Hauer 	return chnenbl0 + event * 4;
3621ec1e82fSSascha Hauer }
3631ec1e82fSSascha Hauer 
3641ec1e82fSSascha Hauer static int sdma_config_ownership(struct sdma_channel *sdmac,
3651ec1e82fSSascha Hauer 		bool event_override, bool mcu_override, bool dsp_override)
3661ec1e82fSSascha Hauer {
3671ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
3681ec1e82fSSascha Hauer 	int channel = sdmac->channel;
3690bbc1413SRichard Zhao 	unsigned long evt, mcu, dsp;
3701ec1e82fSSascha Hauer 
3711ec1e82fSSascha Hauer 	if (event_override && mcu_override && dsp_override)
3721ec1e82fSSascha Hauer 		return -EINVAL;
3731ec1e82fSSascha Hauer 
374c4b56857SRichard Zhao 	evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
375c4b56857SRichard Zhao 	mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
376c4b56857SRichard Zhao 	dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
3771ec1e82fSSascha Hauer 
3781ec1e82fSSascha Hauer 	if (dsp_override)
3790bbc1413SRichard Zhao 		__clear_bit(channel, &dsp);
3801ec1e82fSSascha Hauer 	else
3810bbc1413SRichard Zhao 		__set_bit(channel, &dsp);
3821ec1e82fSSascha Hauer 
3831ec1e82fSSascha Hauer 	if (event_override)
3840bbc1413SRichard Zhao 		__clear_bit(channel, &evt);
3851ec1e82fSSascha Hauer 	else
3860bbc1413SRichard Zhao 		__set_bit(channel, &evt);
3871ec1e82fSSascha Hauer 
3881ec1e82fSSascha Hauer 	if (mcu_override)
3890bbc1413SRichard Zhao 		__clear_bit(channel, &mcu);
3901ec1e82fSSascha Hauer 	else
3910bbc1413SRichard Zhao 		__set_bit(channel, &mcu);
3921ec1e82fSSascha Hauer 
393c4b56857SRichard Zhao 	writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
394c4b56857SRichard Zhao 	writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
395c4b56857SRichard Zhao 	writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
3961ec1e82fSSascha Hauer 
3971ec1e82fSSascha Hauer 	return 0;
3981ec1e82fSSascha Hauer }
3991ec1e82fSSascha Hauer 
400b9a59166SRichard Zhao static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
401b9a59166SRichard Zhao {
4020bbc1413SRichard Zhao 	writel(BIT(channel), sdma->regs + SDMA_H_START);
403b9a59166SRichard Zhao }
404b9a59166SRichard Zhao 
4051ec1e82fSSascha Hauer /*
4062ccaef05SRichard Zhao  * sdma_run_channel0 - run a channel and wait till it's done
4071ec1e82fSSascha Hauer  */
4082ccaef05SRichard Zhao static int sdma_run_channel0(struct sdma_engine *sdma)
4091ec1e82fSSascha Hauer {
4101ec1e82fSSascha Hauer 	int ret;
4112ccaef05SRichard Zhao 	unsigned long timeout = 500;
4121ec1e82fSSascha Hauer 
4132ccaef05SRichard Zhao 	sdma_enable_channel(sdma, 0);
4141ec1e82fSSascha Hauer 
4152ccaef05SRichard Zhao 	while (!(ret = readl_relaxed(sdma->regs + SDMA_H_INTR) & 1)) {
4162ccaef05SRichard Zhao 		if (timeout-- <= 0)
4172ccaef05SRichard Zhao 			break;
4182ccaef05SRichard Zhao 		udelay(1);
4192ccaef05SRichard Zhao 	}
4201ec1e82fSSascha Hauer 
4212ccaef05SRichard Zhao 	if (ret) {
4222ccaef05SRichard Zhao 		/* Clear the interrupt status */
4232ccaef05SRichard Zhao 		writel_relaxed(ret, sdma->regs + SDMA_H_INTR);
4242ccaef05SRichard Zhao 	} else {
4252ccaef05SRichard Zhao 		dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
4262ccaef05SRichard Zhao 	}
4271ec1e82fSSascha Hauer 
4281ec1e82fSSascha Hauer 	return ret ? 0 : -ETIMEDOUT;
4291ec1e82fSSascha Hauer }
4301ec1e82fSSascha Hauer 
4311ec1e82fSSascha Hauer static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
4321ec1e82fSSascha Hauer 		u32 address)
4331ec1e82fSSascha Hauer {
4341ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
4351ec1e82fSSascha Hauer 	void *buf_virt;
4361ec1e82fSSascha Hauer 	dma_addr_t buf_phys;
4371ec1e82fSSascha Hauer 	int ret;
4382ccaef05SRichard Zhao 	unsigned long flags;
43973eab978SSascha Hauer 
4401ec1e82fSSascha Hauer 	buf_virt = dma_alloc_coherent(NULL,
4411ec1e82fSSascha Hauer 			size,
4421ec1e82fSSascha Hauer 			&buf_phys, GFP_KERNEL);
44373eab978SSascha Hauer 	if (!buf_virt) {
4442ccaef05SRichard Zhao 		return -ENOMEM;
44573eab978SSascha Hauer 	}
4461ec1e82fSSascha Hauer 
4472ccaef05SRichard Zhao 	spin_lock_irqsave(&sdma->channel_0_lock, flags);
4482ccaef05SRichard Zhao 
4491ec1e82fSSascha Hauer 	bd0->mode.command = C0_SETPM;
4501ec1e82fSSascha Hauer 	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
4511ec1e82fSSascha Hauer 	bd0->mode.count = size / 2;
4521ec1e82fSSascha Hauer 	bd0->buffer_addr = buf_phys;
4531ec1e82fSSascha Hauer 	bd0->ext_buffer_addr = address;
4541ec1e82fSSascha Hauer 
4551ec1e82fSSascha Hauer 	memcpy(buf_virt, buf, size);
4561ec1e82fSSascha Hauer 
4572ccaef05SRichard Zhao 	ret = sdma_run_channel0(sdma);
4582ccaef05SRichard Zhao 
4592ccaef05SRichard Zhao 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
4601ec1e82fSSascha Hauer 
4611ec1e82fSSascha Hauer 	dma_free_coherent(NULL, size, buf_virt, buf_phys);
4621ec1e82fSSascha Hauer 
4631ec1e82fSSascha Hauer 	return ret;
4641ec1e82fSSascha Hauer }
4651ec1e82fSSascha Hauer 
4661ec1e82fSSascha Hauer static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
4671ec1e82fSSascha Hauer {
4681ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
4691ec1e82fSSascha Hauer 	int channel = sdmac->channel;
4700bbc1413SRichard Zhao 	unsigned long val;
4711ec1e82fSSascha Hauer 	u32 chnenbl = chnenbl_ofs(sdma, event);
4721ec1e82fSSascha Hauer 
473c4b56857SRichard Zhao 	val = readl_relaxed(sdma->regs + chnenbl);
4740bbc1413SRichard Zhao 	__set_bit(channel, &val);
475c4b56857SRichard Zhao 	writel_relaxed(val, sdma->regs + chnenbl);
4761ec1e82fSSascha Hauer }
4771ec1e82fSSascha Hauer 
4781ec1e82fSSascha Hauer static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
4791ec1e82fSSascha Hauer {
4801ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
4811ec1e82fSSascha Hauer 	int channel = sdmac->channel;
4821ec1e82fSSascha Hauer 	u32 chnenbl = chnenbl_ofs(sdma, event);
4830bbc1413SRichard Zhao 	unsigned long val;
4841ec1e82fSSascha Hauer 
485c4b56857SRichard Zhao 	val = readl_relaxed(sdma->regs + chnenbl);
4860bbc1413SRichard Zhao 	__clear_bit(channel, &val);
487c4b56857SRichard Zhao 	writel_relaxed(val, sdma->regs + chnenbl);
4881ec1e82fSSascha Hauer }
4891ec1e82fSSascha Hauer 
4901ec1e82fSSascha Hauer static void sdma_handle_channel_loop(struct sdma_channel *sdmac)
4911ec1e82fSSascha Hauer {
4921ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd;
4931ec1e82fSSascha Hauer 
4941ec1e82fSSascha Hauer 	/*
4951ec1e82fSSascha Hauer 	 * loop mode. Iterate over descriptors, re-setup them and
4961ec1e82fSSascha Hauer 	 * call callback function.
4971ec1e82fSSascha Hauer 	 */
4981ec1e82fSSascha Hauer 	while (1) {
4991ec1e82fSSascha Hauer 		bd = &sdmac->bd[sdmac->buf_tail];
5001ec1e82fSSascha Hauer 
5011ec1e82fSSascha Hauer 		if (bd->mode.status & BD_DONE)
5021ec1e82fSSascha Hauer 			break;
5031ec1e82fSSascha Hauer 
5041ec1e82fSSascha Hauer 		if (bd->mode.status & BD_RROR)
5051ec1e82fSSascha Hauer 			sdmac->status = DMA_ERROR;
5061ec1e82fSSascha Hauer 		else
5071e9cebb4SShawn Guo 			sdmac->status = DMA_IN_PROGRESS;
5081ec1e82fSSascha Hauer 
5091ec1e82fSSascha Hauer 		bd->mode.status |= BD_DONE;
5101ec1e82fSSascha Hauer 		sdmac->buf_tail++;
5111ec1e82fSSascha Hauer 		sdmac->buf_tail %= sdmac->num_bd;
5121ec1e82fSSascha Hauer 
5131ec1e82fSSascha Hauer 		if (sdmac->desc.callback)
5141ec1e82fSSascha Hauer 			sdmac->desc.callback(sdmac->desc.callback_param);
5151ec1e82fSSascha Hauer 	}
5161ec1e82fSSascha Hauer }
5171ec1e82fSSascha Hauer 
5181ec1e82fSSascha Hauer static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)
5191ec1e82fSSascha Hauer {
5201ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd;
5211ec1e82fSSascha Hauer 	int i, error = 0;
5221ec1e82fSSascha Hauer 
523ab59a510SHuang Shijie 	sdmac->chn_real_count = 0;
5241ec1e82fSSascha Hauer 	/*
5251ec1e82fSSascha Hauer 	 * non loop mode. Iterate over all descriptors, collect
5261ec1e82fSSascha Hauer 	 * errors and call callback function
5271ec1e82fSSascha Hauer 	 */
5281ec1e82fSSascha Hauer 	for (i = 0; i < sdmac->num_bd; i++) {
5291ec1e82fSSascha Hauer 		bd = &sdmac->bd[i];
5301ec1e82fSSascha Hauer 
5311ec1e82fSSascha Hauer 		 if (bd->mode.status & (BD_DONE | BD_RROR))
5321ec1e82fSSascha Hauer 			error = -EIO;
533ab59a510SHuang Shijie 		 sdmac->chn_real_count += bd->mode.count;
5341ec1e82fSSascha Hauer 	}
5351ec1e82fSSascha Hauer 
5361ec1e82fSSascha Hauer 	if (error)
5371ec1e82fSSascha Hauer 		sdmac->status = DMA_ERROR;
5381ec1e82fSSascha Hauer 	else
5391ec1e82fSSascha Hauer 		sdmac->status = DMA_SUCCESS;
5401ec1e82fSSascha Hauer 
541f7fbce07SRussell King - ARM Linux 	dma_cookie_complete(&sdmac->desc);
5421ec1e82fSSascha Hauer 	if (sdmac->desc.callback)
5431ec1e82fSSascha Hauer 		sdmac->desc.callback(sdmac->desc.callback_param);
5441ec1e82fSSascha Hauer }
5451ec1e82fSSascha Hauer 
546abd9ccc8SHuang Shijie static void sdma_tasklet(unsigned long data)
5471ec1e82fSSascha Hauer {
548abd9ccc8SHuang Shijie 	struct sdma_channel *sdmac = (struct sdma_channel *) data;
549abd9ccc8SHuang Shijie 
5501ec1e82fSSascha Hauer 	complete(&sdmac->done);
5511ec1e82fSSascha Hauer 
5521ec1e82fSSascha Hauer 	if (sdmac->flags & IMX_DMA_SG_LOOP)
5531ec1e82fSSascha Hauer 		sdma_handle_channel_loop(sdmac);
5541ec1e82fSSascha Hauer 	else
5551ec1e82fSSascha Hauer 		mxc_sdma_handle_channel_normal(sdmac);
5561ec1e82fSSascha Hauer }
5571ec1e82fSSascha Hauer 
5581ec1e82fSSascha Hauer static irqreturn_t sdma_int_handler(int irq, void *dev_id)
5591ec1e82fSSascha Hauer {
5601ec1e82fSSascha Hauer 	struct sdma_engine *sdma = dev_id;
5610bbc1413SRichard Zhao 	unsigned long stat;
5621ec1e82fSSascha Hauer 
563c4b56857SRichard Zhao 	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
5642ccaef05SRichard Zhao 	/* not interested in channel 0 interrupts */
5652ccaef05SRichard Zhao 	stat &= ~1;
566c4b56857SRichard Zhao 	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
5671ec1e82fSSascha Hauer 
5681ec1e82fSSascha Hauer 	while (stat) {
5691ec1e82fSSascha Hauer 		int channel = fls(stat) - 1;
5701ec1e82fSSascha Hauer 		struct sdma_channel *sdmac = &sdma->channel[channel];
5711ec1e82fSSascha Hauer 
572abd9ccc8SHuang Shijie 		tasklet_schedule(&sdmac->tasklet);
5731ec1e82fSSascha Hauer 
5740bbc1413SRichard Zhao 		__clear_bit(channel, &stat);
5751ec1e82fSSascha Hauer 	}
5761ec1e82fSSascha Hauer 
5771ec1e82fSSascha Hauer 	return IRQ_HANDLED;
5781ec1e82fSSascha Hauer }
5791ec1e82fSSascha Hauer 
5801ec1e82fSSascha Hauer /*
5811ec1e82fSSascha Hauer  * sets the pc of SDMA script according to the peripheral type
5821ec1e82fSSascha Hauer  */
5831ec1e82fSSascha Hauer static void sdma_get_pc(struct sdma_channel *sdmac,
5841ec1e82fSSascha Hauer 		enum sdma_peripheral_type peripheral_type)
5851ec1e82fSSascha Hauer {
5861ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
5871ec1e82fSSascha Hauer 	int per_2_emi = 0, emi_2_per = 0;
5881ec1e82fSSascha Hauer 	/*
5891ec1e82fSSascha Hauer 	 * These are needed once we start to support transfers between
5901ec1e82fSSascha Hauer 	 * two peripherals or memory-to-memory transfers
5911ec1e82fSSascha Hauer 	 */
5921ec1e82fSSascha Hauer 	int per_2_per = 0, emi_2_emi = 0;
5931ec1e82fSSascha Hauer 
5941ec1e82fSSascha Hauer 	sdmac->pc_from_device = 0;
5951ec1e82fSSascha Hauer 	sdmac->pc_to_device = 0;
5961ec1e82fSSascha Hauer 
5971ec1e82fSSascha Hauer 	switch (peripheral_type) {
5981ec1e82fSSascha Hauer 	case IMX_DMATYPE_MEMORY:
5991ec1e82fSSascha Hauer 		emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
6001ec1e82fSSascha Hauer 		break;
6011ec1e82fSSascha Hauer 	case IMX_DMATYPE_DSP:
6021ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->bp_2_ap_addr;
6031ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->ap_2_bp_addr;
6041ec1e82fSSascha Hauer 		break;
6051ec1e82fSSascha Hauer 	case IMX_DMATYPE_FIRI:
6061ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
6071ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
6081ec1e82fSSascha Hauer 		break;
6091ec1e82fSSascha Hauer 	case IMX_DMATYPE_UART:
6101ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
6111ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
6121ec1e82fSSascha Hauer 		break;
6131ec1e82fSSascha Hauer 	case IMX_DMATYPE_UART_SP:
6141ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
6151ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
6161ec1e82fSSascha Hauer 		break;
6171ec1e82fSSascha Hauer 	case IMX_DMATYPE_ATA:
6181ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
6191ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
6201ec1e82fSSascha Hauer 		break;
6211ec1e82fSSascha Hauer 	case IMX_DMATYPE_CSPI:
6221ec1e82fSSascha Hauer 	case IMX_DMATYPE_EXT:
6231ec1e82fSSascha Hauer 	case IMX_DMATYPE_SSI:
6241ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->app_2_mcu_addr;
6251ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
6261ec1e82fSSascha Hauer 		break;
6271ec1e82fSSascha Hauer 	case IMX_DMATYPE_SSI_SP:
6281ec1e82fSSascha Hauer 	case IMX_DMATYPE_MMC:
6291ec1e82fSSascha Hauer 	case IMX_DMATYPE_SDHC:
6301ec1e82fSSascha Hauer 	case IMX_DMATYPE_CSPI_SP:
6311ec1e82fSSascha Hauer 	case IMX_DMATYPE_ESAI:
6321ec1e82fSSascha Hauer 	case IMX_DMATYPE_MSHC_SP:
6331ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
6341ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
6351ec1e82fSSascha Hauer 		break;
6361ec1e82fSSascha Hauer 	case IMX_DMATYPE_ASRC:
6371ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
6381ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
6391ec1e82fSSascha Hauer 		per_2_per = sdma->script_addrs->per_2_per_addr;
6401ec1e82fSSascha Hauer 		break;
6411ec1e82fSSascha Hauer 	case IMX_DMATYPE_MSHC:
6421ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
6431ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
6441ec1e82fSSascha Hauer 		break;
6451ec1e82fSSascha Hauer 	case IMX_DMATYPE_CCM:
6461ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
6471ec1e82fSSascha Hauer 		break;
6481ec1e82fSSascha Hauer 	case IMX_DMATYPE_SPDIF:
6491ec1e82fSSascha Hauer 		per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
6501ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
6511ec1e82fSSascha Hauer 		break;
6521ec1e82fSSascha Hauer 	case IMX_DMATYPE_IPU_MEMORY:
6531ec1e82fSSascha Hauer 		emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
6541ec1e82fSSascha Hauer 		break;
6551ec1e82fSSascha Hauer 	default:
6561ec1e82fSSascha Hauer 		break;
6571ec1e82fSSascha Hauer 	}
6581ec1e82fSSascha Hauer 
6591ec1e82fSSascha Hauer 	sdmac->pc_from_device = per_2_emi;
6601ec1e82fSSascha Hauer 	sdmac->pc_to_device = emi_2_per;
6611ec1e82fSSascha Hauer }
6621ec1e82fSSascha Hauer 
6631ec1e82fSSascha Hauer static int sdma_load_context(struct sdma_channel *sdmac)
6641ec1e82fSSascha Hauer {
6651ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
6661ec1e82fSSascha Hauer 	int channel = sdmac->channel;
6671ec1e82fSSascha Hauer 	int load_address;
6681ec1e82fSSascha Hauer 	struct sdma_context_data *context = sdma->context;
6691ec1e82fSSascha Hauer 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;
6701ec1e82fSSascha Hauer 	int ret;
6712ccaef05SRichard Zhao 	unsigned long flags;
6721ec1e82fSSascha Hauer 
673db8196dfSVinod Koul 	if (sdmac->direction == DMA_DEV_TO_MEM) {
6741ec1e82fSSascha Hauer 		load_address = sdmac->pc_from_device;
6751ec1e82fSSascha Hauer 	} else {
6761ec1e82fSSascha Hauer 		load_address = sdmac->pc_to_device;
6771ec1e82fSSascha Hauer 	}
6781ec1e82fSSascha Hauer 
6791ec1e82fSSascha Hauer 	if (load_address < 0)
6801ec1e82fSSascha Hauer 		return load_address;
6811ec1e82fSSascha Hauer 
6821ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "load_address = %d\n", load_address);
6830bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
6841ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
6851ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
6860bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
6870bbc1413SRichard Zhao 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
6881ec1e82fSSascha Hauer 
6892ccaef05SRichard Zhao 	spin_lock_irqsave(&sdma->channel_0_lock, flags);
69073eab978SSascha Hauer 
6911ec1e82fSSascha Hauer 	memset(context, 0, sizeof(*context));
6921ec1e82fSSascha Hauer 	context->channel_state.pc = load_address;
6931ec1e82fSSascha Hauer 
6941ec1e82fSSascha Hauer 	/* Send by context the event mask,base address for peripheral
6951ec1e82fSSascha Hauer 	 * and watermark level
6961ec1e82fSSascha Hauer 	 */
6970bbc1413SRichard Zhao 	context->gReg[0] = sdmac->event_mask[1];
6980bbc1413SRichard Zhao 	context->gReg[1] = sdmac->event_mask[0];
6991ec1e82fSSascha Hauer 	context->gReg[2] = sdmac->per_addr;
7001ec1e82fSSascha Hauer 	context->gReg[6] = sdmac->shp_addr;
7011ec1e82fSSascha Hauer 	context->gReg[7] = sdmac->watermark_level;
7021ec1e82fSSascha Hauer 
7031ec1e82fSSascha Hauer 	bd0->mode.command = C0_SETDM;
7041ec1e82fSSascha Hauer 	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
7051ec1e82fSSascha Hauer 	bd0->mode.count = sizeof(*context) / 4;
7061ec1e82fSSascha Hauer 	bd0->buffer_addr = sdma->context_phys;
7071ec1e82fSSascha Hauer 	bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
7082ccaef05SRichard Zhao 	ret = sdma_run_channel0(sdma);
7091ec1e82fSSascha Hauer 
7102ccaef05SRichard Zhao 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
71173eab978SSascha Hauer 
7121ec1e82fSSascha Hauer 	return ret;
7131ec1e82fSSascha Hauer }
7141ec1e82fSSascha Hauer 
7151ec1e82fSSascha Hauer static void sdma_disable_channel(struct sdma_channel *sdmac)
7161ec1e82fSSascha Hauer {
7171ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
7181ec1e82fSSascha Hauer 	int channel = sdmac->channel;
7191ec1e82fSSascha Hauer 
7200bbc1413SRichard Zhao 	writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
7211ec1e82fSSascha Hauer 	sdmac->status = DMA_ERROR;
7221ec1e82fSSascha Hauer }
7231ec1e82fSSascha Hauer 
7241ec1e82fSSascha Hauer static int sdma_config_channel(struct sdma_channel *sdmac)
7251ec1e82fSSascha Hauer {
7261ec1e82fSSascha Hauer 	int ret;
7271ec1e82fSSascha Hauer 
7281ec1e82fSSascha Hauer 	sdma_disable_channel(sdmac);
7291ec1e82fSSascha Hauer 
7300bbc1413SRichard Zhao 	sdmac->event_mask[0] = 0;
7310bbc1413SRichard Zhao 	sdmac->event_mask[1] = 0;
7321ec1e82fSSascha Hauer 	sdmac->shp_addr = 0;
7331ec1e82fSSascha Hauer 	sdmac->per_addr = 0;
7341ec1e82fSSascha Hauer 
7351ec1e82fSSascha Hauer 	if (sdmac->event_id0) {
736b78bd91fSRichard Zhao 		if (sdmac->event_id0 >= sdmac->sdma->num_events)
7371ec1e82fSSascha Hauer 			return -EINVAL;
7381ec1e82fSSascha Hauer 		sdma_event_enable(sdmac, sdmac->event_id0);
7391ec1e82fSSascha Hauer 	}
7401ec1e82fSSascha Hauer 
7411ec1e82fSSascha Hauer 	switch (sdmac->peripheral_type) {
7421ec1e82fSSascha Hauer 	case IMX_DMATYPE_DSP:
7431ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, false, true, true);
7441ec1e82fSSascha Hauer 		break;
7451ec1e82fSSascha Hauer 	case IMX_DMATYPE_MEMORY:
7461ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, false, true, false);
7471ec1e82fSSascha Hauer 		break;
7481ec1e82fSSascha Hauer 	default:
7491ec1e82fSSascha Hauer 		sdma_config_ownership(sdmac, true, true, false);
7501ec1e82fSSascha Hauer 		break;
7511ec1e82fSSascha Hauer 	}
7521ec1e82fSSascha Hauer 
7531ec1e82fSSascha Hauer 	sdma_get_pc(sdmac, sdmac->peripheral_type);
7541ec1e82fSSascha Hauer 
7551ec1e82fSSascha Hauer 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
7561ec1e82fSSascha Hauer 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
7571ec1e82fSSascha Hauer 		/* Handle multiple event channels differently */
7581ec1e82fSSascha Hauer 		if (sdmac->event_id1) {
7590bbc1413SRichard Zhao 			sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32);
7601ec1e82fSSascha Hauer 			if (sdmac->event_id1 > 31)
7610bbc1413SRichard Zhao 				__set_bit(31, &sdmac->watermark_level);
7620bbc1413SRichard Zhao 			sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32);
7631ec1e82fSSascha Hauer 			if (sdmac->event_id0 > 31)
7640bbc1413SRichard Zhao 				__set_bit(30, &sdmac->watermark_level);
7651ec1e82fSSascha Hauer 		} else {
7660bbc1413SRichard Zhao 			__set_bit(sdmac->event_id0, sdmac->event_mask);
7671ec1e82fSSascha Hauer 		}
7681ec1e82fSSascha Hauer 		/* Watermark Level */
7691ec1e82fSSascha Hauer 		sdmac->watermark_level |= sdmac->watermark_level;
7701ec1e82fSSascha Hauer 		/* Address */
7711ec1e82fSSascha Hauer 		sdmac->shp_addr = sdmac->per_address;
7721ec1e82fSSascha Hauer 	} else {
7731ec1e82fSSascha Hauer 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
7741ec1e82fSSascha Hauer 	}
7751ec1e82fSSascha Hauer 
7761ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
7771ec1e82fSSascha Hauer 
7781ec1e82fSSascha Hauer 	return ret;
7791ec1e82fSSascha Hauer }
7801ec1e82fSSascha Hauer 
7811ec1e82fSSascha Hauer static int sdma_set_channel_priority(struct sdma_channel *sdmac,
7821ec1e82fSSascha Hauer 		unsigned int priority)
7831ec1e82fSSascha Hauer {
7841ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
7851ec1e82fSSascha Hauer 	int channel = sdmac->channel;
7861ec1e82fSSascha Hauer 
7871ec1e82fSSascha Hauer 	if (priority < MXC_SDMA_MIN_PRIORITY
7881ec1e82fSSascha Hauer 	    || priority > MXC_SDMA_MAX_PRIORITY) {
7891ec1e82fSSascha Hauer 		return -EINVAL;
7901ec1e82fSSascha Hauer 	}
7911ec1e82fSSascha Hauer 
792c4b56857SRichard Zhao 	writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
7931ec1e82fSSascha Hauer 
7941ec1e82fSSascha Hauer 	return 0;
7951ec1e82fSSascha Hauer }
7961ec1e82fSSascha Hauer 
7971ec1e82fSSascha Hauer static int sdma_request_channel(struct sdma_channel *sdmac)
7981ec1e82fSSascha Hauer {
7991ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
8001ec1e82fSSascha Hauer 	int channel = sdmac->channel;
8011ec1e82fSSascha Hauer 	int ret = -EBUSY;
8021ec1e82fSSascha Hauer 
8031ec1e82fSSascha Hauer 	sdmac->bd = dma_alloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, GFP_KERNEL);
8041ec1e82fSSascha Hauer 	if (!sdmac->bd) {
8051ec1e82fSSascha Hauer 		ret = -ENOMEM;
8061ec1e82fSSascha Hauer 		goto out;
8071ec1e82fSSascha Hauer 	}
8081ec1e82fSSascha Hauer 
8091ec1e82fSSascha Hauer 	memset(sdmac->bd, 0, PAGE_SIZE);
8101ec1e82fSSascha Hauer 
8111ec1e82fSSascha Hauer 	sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;
8121ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
8131ec1e82fSSascha Hauer 
8141ec1e82fSSascha Hauer 	sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY);
8151ec1e82fSSascha Hauer 
8161ec1e82fSSascha Hauer 	init_completion(&sdmac->done);
8171ec1e82fSSascha Hauer 
8181ec1e82fSSascha Hauer 	return 0;
8191ec1e82fSSascha Hauer out:
8201ec1e82fSSascha Hauer 
8211ec1e82fSSascha Hauer 	return ret;
8221ec1e82fSSascha Hauer }
8231ec1e82fSSascha Hauer 
8241ec1e82fSSascha Hauer static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
8251ec1e82fSSascha Hauer {
8261ec1e82fSSascha Hauer 	return container_of(chan, struct sdma_channel, chan);
8271ec1e82fSSascha Hauer }
8281ec1e82fSSascha Hauer 
8291ec1e82fSSascha Hauer static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx)
8301ec1e82fSSascha Hauer {
831f69f2e26SHaitao Zhang 	unsigned long flags;
8321ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(tx->chan);
8331ec1e82fSSascha Hauer 	dma_cookie_t cookie;
8341ec1e82fSSascha Hauer 
835f69f2e26SHaitao Zhang 	spin_lock_irqsave(&sdmac->lock, flags);
8361ec1e82fSSascha Hauer 
837884485e1SRussell King - ARM Linux 	cookie = dma_cookie_assign(tx);
8381ec1e82fSSascha Hauer 
839f69f2e26SHaitao Zhang 	spin_unlock_irqrestore(&sdmac->lock, flags);
8401ec1e82fSSascha Hauer 
8411ec1e82fSSascha Hauer 	return cookie;
8421ec1e82fSSascha Hauer }
8431ec1e82fSSascha Hauer 
8441ec1e82fSSascha Hauer static int sdma_alloc_chan_resources(struct dma_chan *chan)
8451ec1e82fSSascha Hauer {
8461ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
8471ec1e82fSSascha Hauer 	struct imx_dma_data *data = chan->private;
8481ec1e82fSSascha Hauer 	int prio, ret;
8491ec1e82fSSascha Hauer 
8501ec1e82fSSascha Hauer 	if (!data)
8511ec1e82fSSascha Hauer 		return -EINVAL;
8521ec1e82fSSascha Hauer 
8531ec1e82fSSascha Hauer 	switch (data->priority) {
8541ec1e82fSSascha Hauer 	case DMA_PRIO_HIGH:
8551ec1e82fSSascha Hauer 		prio = 3;
8561ec1e82fSSascha Hauer 		break;
8571ec1e82fSSascha Hauer 	case DMA_PRIO_MEDIUM:
8581ec1e82fSSascha Hauer 		prio = 2;
8591ec1e82fSSascha Hauer 		break;
8601ec1e82fSSascha Hauer 	case DMA_PRIO_LOW:
8611ec1e82fSSascha Hauer 	default:
8621ec1e82fSSascha Hauer 		prio = 1;
8631ec1e82fSSascha Hauer 		break;
8641ec1e82fSSascha Hauer 	}
8651ec1e82fSSascha Hauer 
8661ec1e82fSSascha Hauer 	sdmac->peripheral_type = data->peripheral_type;
8671ec1e82fSSascha Hauer 	sdmac->event_id0 = data->dma_request;
868c2c744d3SRichard Zhao 
8697560e3f3SSascha Hauer 	clk_enable(sdmac->sdma->clk_ipg);
8707560e3f3SSascha Hauer 	clk_enable(sdmac->sdma->clk_ahb);
871c2c744d3SRichard Zhao 
8723bb5e7caSRichard Zhao 	ret = sdma_request_channel(sdmac);
8731ec1e82fSSascha Hauer 	if (ret)
8741ec1e82fSSascha Hauer 		return ret;
8751ec1e82fSSascha Hauer 
8763bb5e7caSRichard Zhao 	ret = sdma_set_channel_priority(sdmac, prio);
8771ec1e82fSSascha Hauer 	if (ret)
8781ec1e82fSSascha Hauer 		return ret;
8791ec1e82fSSascha Hauer 
8801ec1e82fSSascha Hauer 	dma_async_tx_descriptor_init(&sdmac->desc, chan);
8811ec1e82fSSascha Hauer 	sdmac->desc.tx_submit = sdma_tx_submit;
8821ec1e82fSSascha Hauer 	/* txd.flags will be overwritten in prep funcs */
8831ec1e82fSSascha Hauer 	sdmac->desc.flags = DMA_CTRL_ACK;
8841ec1e82fSSascha Hauer 
8851ec1e82fSSascha Hauer 	return 0;
8861ec1e82fSSascha Hauer }
8871ec1e82fSSascha Hauer 
8881ec1e82fSSascha Hauer static void sdma_free_chan_resources(struct dma_chan *chan)
8891ec1e82fSSascha Hauer {
8901ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
8911ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
8921ec1e82fSSascha Hauer 
8931ec1e82fSSascha Hauer 	sdma_disable_channel(sdmac);
8941ec1e82fSSascha Hauer 
8951ec1e82fSSascha Hauer 	if (sdmac->event_id0)
8961ec1e82fSSascha Hauer 		sdma_event_disable(sdmac, sdmac->event_id0);
8971ec1e82fSSascha Hauer 	if (sdmac->event_id1)
8981ec1e82fSSascha Hauer 		sdma_event_disable(sdmac, sdmac->event_id1);
8991ec1e82fSSascha Hauer 
9001ec1e82fSSascha Hauer 	sdmac->event_id0 = 0;
9011ec1e82fSSascha Hauer 	sdmac->event_id1 = 0;
9021ec1e82fSSascha Hauer 
9031ec1e82fSSascha Hauer 	sdma_set_channel_priority(sdmac, 0);
9041ec1e82fSSascha Hauer 
9051ec1e82fSSascha Hauer 	dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys);
9061ec1e82fSSascha Hauer 
9077560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
9087560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
9091ec1e82fSSascha Hauer }
9101ec1e82fSSascha Hauer 
9111ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
9121ec1e82fSSascha Hauer 		struct dma_chan *chan, struct scatterlist *sgl,
913db8196dfSVinod Koul 		unsigned int sg_len, enum dma_transfer_direction direction,
914185ecb5fSAlexandre Bounine 		unsigned long flags, void *context)
9151ec1e82fSSascha Hauer {
9161ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
9171ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
9181ec1e82fSSascha Hauer 	int ret, i, count;
91923889c63SSascha Hauer 	int channel = sdmac->channel;
9201ec1e82fSSascha Hauer 	struct scatterlist *sg;
9211ec1e82fSSascha Hauer 
9221ec1e82fSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
9231ec1e82fSSascha Hauer 		return NULL;
9241ec1e82fSSascha Hauer 	sdmac->status = DMA_IN_PROGRESS;
9251ec1e82fSSascha Hauer 
9261ec1e82fSSascha Hauer 	sdmac->flags = 0;
9271ec1e82fSSascha Hauer 
9288e2e27c7SRichard Zhao 	sdmac->buf_tail = 0;
9298e2e27c7SRichard Zhao 
9301ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
9311ec1e82fSSascha Hauer 			sg_len, channel);
9321ec1e82fSSascha Hauer 
9331ec1e82fSSascha Hauer 	sdmac->direction = direction;
9341ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
9351ec1e82fSSascha Hauer 	if (ret)
9361ec1e82fSSascha Hauer 		goto err_out;
9371ec1e82fSSascha Hauer 
9381ec1e82fSSascha Hauer 	if (sg_len > NUM_BD) {
9391ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
9401ec1e82fSSascha Hauer 				channel, sg_len, NUM_BD);
9411ec1e82fSSascha Hauer 		ret = -EINVAL;
9421ec1e82fSSascha Hauer 		goto err_out;
9431ec1e82fSSascha Hauer 	}
9441ec1e82fSSascha Hauer 
945ab59a510SHuang Shijie 	sdmac->chn_count = 0;
9461ec1e82fSSascha Hauer 	for_each_sg(sgl, sg, sg_len, i) {
9471ec1e82fSSascha Hauer 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
9481ec1e82fSSascha Hauer 		int param;
9491ec1e82fSSascha Hauer 
950d2f5c276SAnatolij Gustschin 		bd->buffer_addr = sg->dma_address;
9511ec1e82fSSascha Hauer 
952fdaf9c4bSLars-Peter Clausen 		count = sg_dma_len(sg);
9531ec1e82fSSascha Hauer 
9541ec1e82fSSascha Hauer 		if (count > 0xffff) {
9551ec1e82fSSascha Hauer 			dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
9561ec1e82fSSascha Hauer 					channel, count, 0xffff);
9571ec1e82fSSascha Hauer 			ret = -EINVAL;
9581ec1e82fSSascha Hauer 			goto err_out;
9591ec1e82fSSascha Hauer 		}
9601ec1e82fSSascha Hauer 
9611ec1e82fSSascha Hauer 		bd->mode.count = count;
962ab59a510SHuang Shijie 		sdmac->chn_count += count;
9631ec1e82fSSascha Hauer 
9641ec1e82fSSascha Hauer 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) {
9651ec1e82fSSascha Hauer 			ret =  -EINVAL;
9661ec1e82fSSascha Hauer 			goto err_out;
9671ec1e82fSSascha Hauer 		}
9681fa81c27SSascha Hauer 
9691fa81c27SSascha Hauer 		switch (sdmac->word_size) {
9701fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_4_BYTES:
9711ec1e82fSSascha Hauer 			bd->mode.command = 0;
9721fa81c27SSascha Hauer 			if (count & 3 || sg->dma_address & 3)
9731fa81c27SSascha Hauer 				return NULL;
9741fa81c27SSascha Hauer 			break;
9751fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_2_BYTES:
9761fa81c27SSascha Hauer 			bd->mode.command = 2;
9771fa81c27SSascha Hauer 			if (count & 1 || sg->dma_address & 1)
9781fa81c27SSascha Hauer 				return NULL;
9791fa81c27SSascha Hauer 			break;
9801fa81c27SSascha Hauer 		case DMA_SLAVE_BUSWIDTH_1_BYTE:
9811fa81c27SSascha Hauer 			bd->mode.command = 1;
9821fa81c27SSascha Hauer 			break;
9831fa81c27SSascha Hauer 		default:
9841fa81c27SSascha Hauer 			return NULL;
9851fa81c27SSascha Hauer 		}
9861ec1e82fSSascha Hauer 
9871ec1e82fSSascha Hauer 		param = BD_DONE | BD_EXTD | BD_CONT;
9881ec1e82fSSascha Hauer 
989341b9419SShawn Guo 		if (i + 1 == sg_len) {
9901ec1e82fSSascha Hauer 			param |= BD_INTR;
991341b9419SShawn Guo 			param |= BD_LAST;
992341b9419SShawn Guo 			param &= ~BD_CONT;
9931ec1e82fSSascha Hauer 		}
9941ec1e82fSSascha Hauer 
9951ec1e82fSSascha Hauer 		dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n",
9961ec1e82fSSascha Hauer 				i, count, sg->dma_address,
9971ec1e82fSSascha Hauer 				param & BD_WRAP ? "wrap" : "",
9981ec1e82fSSascha Hauer 				param & BD_INTR ? " intr" : "");
9991ec1e82fSSascha Hauer 
10001ec1e82fSSascha Hauer 		bd->mode.status = param;
10011ec1e82fSSascha Hauer 	}
10021ec1e82fSSascha Hauer 
10031ec1e82fSSascha Hauer 	sdmac->num_bd = sg_len;
10041ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
10051ec1e82fSSascha Hauer 
10061ec1e82fSSascha Hauer 	return &sdmac->desc;
10071ec1e82fSSascha Hauer err_out:
10084b2ce9ddSShawn Guo 	sdmac->status = DMA_ERROR;
10091ec1e82fSSascha Hauer 	return NULL;
10101ec1e82fSSascha Hauer }
10111ec1e82fSSascha Hauer 
10121ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
10131ec1e82fSSascha Hauer 		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1014185ecb5fSAlexandre Bounine 		size_t period_len, enum dma_transfer_direction direction,
1015*ec8b5e48SPeter Ujfalusi 		unsigned long flags, void *context)
10161ec1e82fSSascha Hauer {
10171ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
10181ec1e82fSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
10191ec1e82fSSascha Hauer 	int num_periods = buf_len / period_len;
102023889c63SSascha Hauer 	int channel = sdmac->channel;
10211ec1e82fSSascha Hauer 	int ret, i = 0, buf = 0;
10221ec1e82fSSascha Hauer 
10231ec1e82fSSascha Hauer 	dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
10241ec1e82fSSascha Hauer 
10251ec1e82fSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
10261ec1e82fSSascha Hauer 		return NULL;
10271ec1e82fSSascha Hauer 
10281ec1e82fSSascha Hauer 	sdmac->status = DMA_IN_PROGRESS;
10291ec1e82fSSascha Hauer 
10308e2e27c7SRichard Zhao 	sdmac->buf_tail = 0;
10318e2e27c7SRichard Zhao 
10321ec1e82fSSascha Hauer 	sdmac->flags |= IMX_DMA_SG_LOOP;
10331ec1e82fSSascha Hauer 	sdmac->direction = direction;
10341ec1e82fSSascha Hauer 	ret = sdma_load_context(sdmac);
10351ec1e82fSSascha Hauer 	if (ret)
10361ec1e82fSSascha Hauer 		goto err_out;
10371ec1e82fSSascha Hauer 
10381ec1e82fSSascha Hauer 	if (num_periods > NUM_BD) {
10391ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",
10401ec1e82fSSascha Hauer 				channel, num_periods, NUM_BD);
10411ec1e82fSSascha Hauer 		goto err_out;
10421ec1e82fSSascha Hauer 	}
10431ec1e82fSSascha Hauer 
10441ec1e82fSSascha Hauer 	if (period_len > 0xffff) {
10451ec1e82fSSascha Hauer 		dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %d > %d\n",
10461ec1e82fSSascha Hauer 				channel, period_len, 0xffff);
10471ec1e82fSSascha Hauer 		goto err_out;
10481ec1e82fSSascha Hauer 	}
10491ec1e82fSSascha Hauer 
10501ec1e82fSSascha Hauer 	while (buf < buf_len) {
10511ec1e82fSSascha Hauer 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];
10521ec1e82fSSascha Hauer 		int param;
10531ec1e82fSSascha Hauer 
10541ec1e82fSSascha Hauer 		bd->buffer_addr = dma_addr;
10551ec1e82fSSascha Hauer 
10561ec1e82fSSascha Hauer 		bd->mode.count = period_len;
10571ec1e82fSSascha Hauer 
10581ec1e82fSSascha Hauer 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
10591ec1e82fSSascha Hauer 			goto err_out;
10601ec1e82fSSascha Hauer 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
10611ec1e82fSSascha Hauer 			bd->mode.command = 0;
10621ec1e82fSSascha Hauer 		else
10631ec1e82fSSascha Hauer 			bd->mode.command = sdmac->word_size;
10641ec1e82fSSascha Hauer 
10651ec1e82fSSascha Hauer 		param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
10661ec1e82fSSascha Hauer 		if (i + 1 == num_periods)
10671ec1e82fSSascha Hauer 			param |= BD_WRAP;
10681ec1e82fSSascha Hauer 
10691ec1e82fSSascha Hauer 		dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n",
10701ec1e82fSSascha Hauer 				i, period_len, dma_addr,
10711ec1e82fSSascha Hauer 				param & BD_WRAP ? "wrap" : "",
10721ec1e82fSSascha Hauer 				param & BD_INTR ? " intr" : "");
10731ec1e82fSSascha Hauer 
10741ec1e82fSSascha Hauer 		bd->mode.status = param;
10751ec1e82fSSascha Hauer 
10761ec1e82fSSascha Hauer 		dma_addr += period_len;
10771ec1e82fSSascha Hauer 		buf += period_len;
10781ec1e82fSSascha Hauer 
10791ec1e82fSSascha Hauer 		i++;
10801ec1e82fSSascha Hauer 	}
10811ec1e82fSSascha Hauer 
10821ec1e82fSSascha Hauer 	sdmac->num_bd = num_periods;
10831ec1e82fSSascha Hauer 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;
10841ec1e82fSSascha Hauer 
10851ec1e82fSSascha Hauer 	return &sdmac->desc;
10861ec1e82fSSascha Hauer err_out:
10871ec1e82fSSascha Hauer 	sdmac->status = DMA_ERROR;
10881ec1e82fSSascha Hauer 	return NULL;
10891ec1e82fSSascha Hauer }
10901ec1e82fSSascha Hauer 
10911ec1e82fSSascha Hauer static int sdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
10921ec1e82fSSascha Hauer 		unsigned long arg)
10931ec1e82fSSascha Hauer {
10941ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
10951ec1e82fSSascha Hauer 	struct dma_slave_config *dmaengine_cfg = (void *)arg;
10961ec1e82fSSascha Hauer 
10971ec1e82fSSascha Hauer 	switch (cmd) {
10981ec1e82fSSascha Hauer 	case DMA_TERMINATE_ALL:
10991ec1e82fSSascha Hauer 		sdma_disable_channel(sdmac);
11001ec1e82fSSascha Hauer 		return 0;
11011ec1e82fSSascha Hauer 	case DMA_SLAVE_CONFIG:
1102db8196dfSVinod Koul 		if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) {
11031ec1e82fSSascha Hauer 			sdmac->per_address = dmaengine_cfg->src_addr;
110494ac27a5SPhilippe Rétornaz 			sdmac->watermark_level = dmaengine_cfg->src_maxburst *
110594ac27a5SPhilippe Rétornaz 						dmaengine_cfg->src_addr_width;
11061ec1e82fSSascha Hauer 			sdmac->word_size = dmaengine_cfg->src_addr_width;
11071ec1e82fSSascha Hauer 		} else {
11081ec1e82fSSascha Hauer 			sdmac->per_address = dmaengine_cfg->dst_addr;
110994ac27a5SPhilippe Rétornaz 			sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
111094ac27a5SPhilippe Rétornaz 						dmaengine_cfg->dst_addr_width;
11111ec1e82fSSascha Hauer 			sdmac->word_size = dmaengine_cfg->dst_addr_width;
11121ec1e82fSSascha Hauer 		}
1113e6966433SHuang Shijie 		sdmac->direction = dmaengine_cfg->direction;
11141ec1e82fSSascha Hauer 		return sdma_config_channel(sdmac);
11151ec1e82fSSascha Hauer 	default:
11161ec1e82fSSascha Hauer 		return -ENOSYS;
11171ec1e82fSSascha Hauer 	}
11181ec1e82fSSascha Hauer 
11191ec1e82fSSascha Hauer 	return -EINVAL;
11201ec1e82fSSascha Hauer }
11211ec1e82fSSascha Hauer 
11221ec1e82fSSascha Hauer static enum dma_status sdma_tx_status(struct dma_chan *chan,
11231ec1e82fSSascha Hauer 					    dma_cookie_t cookie,
11241ec1e82fSSascha Hauer 					    struct dma_tx_state *txstate)
11251ec1e82fSSascha Hauer {
11261ec1e82fSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
11271ec1e82fSSascha Hauer 	dma_cookie_t last_used;
11281ec1e82fSSascha Hauer 
11291ec1e82fSSascha Hauer 	last_used = chan->cookie;
11301ec1e82fSSascha Hauer 
11314d4e58deSRussell King - ARM Linux 	dma_set_tx_state(txstate, chan->completed_cookie, last_used,
1132ab59a510SHuang Shijie 			sdmac->chn_count - sdmac->chn_real_count);
11331ec1e82fSSascha Hauer 
11348a965911SShawn Guo 	return sdmac->status;
11351ec1e82fSSascha Hauer }
11361ec1e82fSSascha Hauer 
11371ec1e82fSSascha Hauer static void sdma_issue_pending(struct dma_chan *chan)
11381ec1e82fSSascha Hauer {
11392b4f130eSSascha Hauer 	struct sdma_channel *sdmac = to_sdma_chan(chan);
11402b4f130eSSascha Hauer 	struct sdma_engine *sdma = sdmac->sdma;
11412b4f130eSSascha Hauer 
11422b4f130eSSascha Hauer 	if (sdmac->status == DMA_IN_PROGRESS)
11432b4f130eSSascha Hauer 		sdma_enable_channel(sdma, sdmac->channel);
11441ec1e82fSSascha Hauer }
11451ec1e82fSSascha Hauer 
11465b28aa31SSascha Hauer #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1	34
11475b28aa31SSascha Hauer 
11485b28aa31SSascha Hauer static void sdma_add_scripts(struct sdma_engine *sdma,
11495b28aa31SSascha Hauer 		const struct sdma_script_start_addrs *addr)
11505b28aa31SSascha Hauer {
11515b28aa31SSascha Hauer 	s32 *addr_arr = (u32 *)addr;
11525b28aa31SSascha Hauer 	s32 *saddr_arr = (u32 *)sdma->script_addrs;
11535b28aa31SSascha Hauer 	int i;
11545b28aa31SSascha Hauer 
11555b28aa31SSascha Hauer 	for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++)
11565b28aa31SSascha Hauer 		if (addr_arr[i] > 0)
11575b28aa31SSascha Hauer 			saddr_arr[i] = addr_arr[i];
11585b28aa31SSascha Hauer }
11595b28aa31SSascha Hauer 
11607b4b88e0SSascha Hauer static void sdma_load_firmware(const struct firmware *fw, void *context)
11615b28aa31SSascha Hauer {
11627b4b88e0SSascha Hauer 	struct sdma_engine *sdma = context;
11635b28aa31SSascha Hauer 	const struct sdma_firmware_header *header;
11645b28aa31SSascha Hauer 	const struct sdma_script_start_addrs *addr;
11655b28aa31SSascha Hauer 	unsigned short *ram_code;
11665b28aa31SSascha Hauer 
11677b4b88e0SSascha Hauer 	if (!fw) {
11687b4b88e0SSascha Hauer 		dev_err(sdma->dev, "firmware not found\n");
11697b4b88e0SSascha Hauer 		return;
11707b4b88e0SSascha Hauer 	}
11715b28aa31SSascha Hauer 
11725b28aa31SSascha Hauer 	if (fw->size < sizeof(*header))
11735b28aa31SSascha Hauer 		goto err_firmware;
11745b28aa31SSascha Hauer 
11755b28aa31SSascha Hauer 	header = (struct sdma_firmware_header *)fw->data;
11765b28aa31SSascha Hauer 
11775b28aa31SSascha Hauer 	if (header->magic != SDMA_FIRMWARE_MAGIC)
11785b28aa31SSascha Hauer 		goto err_firmware;
11795b28aa31SSascha Hauer 	if (header->ram_code_start + header->ram_code_size > fw->size)
11805b28aa31SSascha Hauer 		goto err_firmware;
11815b28aa31SSascha Hauer 
11825b28aa31SSascha Hauer 	addr = (void *)header + header->script_addrs_start;
11835b28aa31SSascha Hauer 	ram_code = (void *)header + header->ram_code_start;
11845b28aa31SSascha Hauer 
11857560e3f3SSascha Hauer 	clk_enable(sdma->clk_ipg);
11867560e3f3SSascha Hauer 	clk_enable(sdma->clk_ahb);
11875b28aa31SSascha Hauer 	/* download the RAM image for SDMA */
11885b28aa31SSascha Hauer 	sdma_load_script(sdma, ram_code,
11895b28aa31SSascha Hauer 			header->ram_code_size,
11906866fd3bSSascha Hauer 			addr->ram_code_start_addr);
11917560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
11927560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
11935b28aa31SSascha Hauer 
11945b28aa31SSascha Hauer 	sdma_add_scripts(sdma, addr);
11955b28aa31SSascha Hauer 
11965b28aa31SSascha Hauer 	dev_info(sdma->dev, "loaded firmware %d.%d\n",
11975b28aa31SSascha Hauer 			header->version_major,
11985b28aa31SSascha Hauer 			header->version_minor);
11995b28aa31SSascha Hauer 
12005b28aa31SSascha Hauer err_firmware:
12015b28aa31SSascha Hauer 	release_firmware(fw);
12027b4b88e0SSascha Hauer }
12037b4b88e0SSascha Hauer 
12047b4b88e0SSascha Hauer static int __init sdma_get_firmware(struct sdma_engine *sdma,
12057b4b88e0SSascha Hauer 		const char *fw_name)
12067b4b88e0SSascha Hauer {
12077b4b88e0SSascha Hauer 	int ret;
12087b4b88e0SSascha Hauer 
12097b4b88e0SSascha Hauer 	ret = request_firmware_nowait(THIS_MODULE,
12107b4b88e0SSascha Hauer 			FW_ACTION_HOTPLUG, fw_name, sdma->dev,
12117b4b88e0SSascha Hauer 			GFP_KERNEL, sdma, sdma_load_firmware);
12125b28aa31SSascha Hauer 
12135b28aa31SSascha Hauer 	return ret;
12145b28aa31SSascha Hauer }
12155b28aa31SSascha Hauer 
12165b28aa31SSascha Hauer static int __init sdma_init(struct sdma_engine *sdma)
12171ec1e82fSSascha Hauer {
12181ec1e82fSSascha Hauer 	int i, ret;
12191ec1e82fSSascha Hauer 	dma_addr_t ccb_phys;
12201ec1e82fSSascha Hauer 
122162550cd7SShawn Guo 	switch (sdma->devtype) {
122262550cd7SShawn Guo 	case IMX31_SDMA:
12231ec1e82fSSascha Hauer 		sdma->num_events = 32;
12241ec1e82fSSascha Hauer 		break;
122562550cd7SShawn Guo 	case IMX35_SDMA:
12261ec1e82fSSascha Hauer 		sdma->num_events = 48;
12271ec1e82fSSascha Hauer 		break;
12281ec1e82fSSascha Hauer 	default:
122962550cd7SShawn Guo 		dev_err(sdma->dev, "Unknown sdma type %d. aborting\n",
123062550cd7SShawn Guo 			sdma->devtype);
12311ec1e82fSSascha Hauer 		return -ENODEV;
12321ec1e82fSSascha Hauer 	}
12331ec1e82fSSascha Hauer 
12347560e3f3SSascha Hauer 	clk_enable(sdma->clk_ipg);
12357560e3f3SSascha Hauer 	clk_enable(sdma->clk_ahb);
12361ec1e82fSSascha Hauer 
12371ec1e82fSSascha Hauer 	/* Be sure SDMA has not started yet */
1238c4b56857SRichard Zhao 	writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
12391ec1e82fSSascha Hauer 
12401ec1e82fSSascha Hauer 	sdma->channel_control = dma_alloc_coherent(NULL,
12411ec1e82fSSascha Hauer 			MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
12421ec1e82fSSascha Hauer 			sizeof(struct sdma_context_data),
12431ec1e82fSSascha Hauer 			&ccb_phys, GFP_KERNEL);
12441ec1e82fSSascha Hauer 
12451ec1e82fSSascha Hauer 	if (!sdma->channel_control) {
12461ec1e82fSSascha Hauer 		ret = -ENOMEM;
12471ec1e82fSSascha Hauer 		goto err_dma_alloc;
12481ec1e82fSSascha Hauer 	}
12491ec1e82fSSascha Hauer 
12501ec1e82fSSascha Hauer 	sdma->context = (void *)sdma->channel_control +
12511ec1e82fSSascha Hauer 		MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
12521ec1e82fSSascha Hauer 	sdma->context_phys = ccb_phys +
12531ec1e82fSSascha Hauer 		MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
12541ec1e82fSSascha Hauer 
12551ec1e82fSSascha Hauer 	/* Zero-out the CCB structures array just allocated */
12561ec1e82fSSascha Hauer 	memset(sdma->channel_control, 0,
12571ec1e82fSSascha Hauer 			MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control));
12581ec1e82fSSascha Hauer 
12591ec1e82fSSascha Hauer 	/* disable all channels */
12601ec1e82fSSascha Hauer 	for (i = 0; i < sdma->num_events; i++)
1261c4b56857SRichard Zhao 		writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
12621ec1e82fSSascha Hauer 
12631ec1e82fSSascha Hauer 	/* All channels have priority 0 */
12641ec1e82fSSascha Hauer 	for (i = 0; i < MAX_DMA_CHANNELS; i++)
1265c4b56857SRichard Zhao 		writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
12661ec1e82fSSascha Hauer 
12671ec1e82fSSascha Hauer 	ret = sdma_request_channel(&sdma->channel[0]);
12681ec1e82fSSascha Hauer 	if (ret)
12691ec1e82fSSascha Hauer 		goto err_dma_alloc;
12701ec1e82fSSascha Hauer 
12711ec1e82fSSascha Hauer 	sdma_config_ownership(&sdma->channel[0], false, true, false);
12721ec1e82fSSascha Hauer 
12731ec1e82fSSascha Hauer 	/* Set Command Channel (Channel Zero) */
1274c4b56857SRichard Zhao 	writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
12751ec1e82fSSascha Hauer 
12761ec1e82fSSascha Hauer 	/* Set bits of CONFIG register but with static context switching */
12771ec1e82fSSascha Hauer 	/* FIXME: Check whether to set ACR bit depending on clock ratios */
1278c4b56857SRichard Zhao 	writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
12791ec1e82fSSascha Hauer 
1280c4b56857SRichard Zhao 	writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
12811ec1e82fSSascha Hauer 
12821ec1e82fSSascha Hauer 	/* Set bits of CONFIG register with given context switching mode */
1283c4b56857SRichard Zhao 	writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG);
12841ec1e82fSSascha Hauer 
12851ec1e82fSSascha Hauer 	/* Initializes channel's priorities */
12861ec1e82fSSascha Hauer 	sdma_set_channel_priority(&sdma->channel[0], 7);
12871ec1e82fSSascha Hauer 
12887560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
12897560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
12901ec1e82fSSascha Hauer 
12911ec1e82fSSascha Hauer 	return 0;
12921ec1e82fSSascha Hauer 
12931ec1e82fSSascha Hauer err_dma_alloc:
12947560e3f3SSascha Hauer 	clk_disable(sdma->clk_ipg);
12957560e3f3SSascha Hauer 	clk_disable(sdma->clk_ahb);
12961ec1e82fSSascha Hauer 	dev_err(sdma->dev, "initialisation failed with %d\n", ret);
12971ec1e82fSSascha Hauer 	return ret;
12981ec1e82fSSascha Hauer }
12991ec1e82fSSascha Hauer 
13001ec1e82fSSascha Hauer static int __init sdma_probe(struct platform_device *pdev)
13011ec1e82fSSascha Hauer {
1302580975d7SShawn Guo 	const struct of_device_id *of_id =
1303580975d7SShawn Guo 			of_match_device(sdma_dt_ids, &pdev->dev);
1304580975d7SShawn Guo 	struct device_node *np = pdev->dev.of_node;
1305580975d7SShawn Guo 	const char *fw_name;
13061ec1e82fSSascha Hauer 	int ret;
13071ec1e82fSSascha Hauer 	int irq;
13081ec1e82fSSascha Hauer 	struct resource *iores;
13091ec1e82fSSascha Hauer 	struct sdma_platform_data *pdata = pdev->dev.platform_data;
13101ec1e82fSSascha Hauer 	int i;
13111ec1e82fSSascha Hauer 	struct sdma_engine *sdma;
131236e2f21aSSascha Hauer 	s32 *saddr_arr;
13131ec1e82fSSascha Hauer 
13141ec1e82fSSascha Hauer 	sdma = kzalloc(sizeof(*sdma), GFP_KERNEL);
13151ec1e82fSSascha Hauer 	if (!sdma)
13161ec1e82fSSascha Hauer 		return -ENOMEM;
13171ec1e82fSSascha Hauer 
13182ccaef05SRichard Zhao 	spin_lock_init(&sdma->channel_0_lock);
131973eab978SSascha Hauer 
13201ec1e82fSSascha Hauer 	sdma->dev = &pdev->dev;
13211ec1e82fSSascha Hauer 
13221ec1e82fSSascha Hauer 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
13231ec1e82fSSascha Hauer 	irq = platform_get_irq(pdev, 0);
1324580975d7SShawn Guo 	if (!iores || irq < 0) {
13251ec1e82fSSascha Hauer 		ret = -EINVAL;
13261ec1e82fSSascha Hauer 		goto err_irq;
13271ec1e82fSSascha Hauer 	}
13281ec1e82fSSascha Hauer 
13291ec1e82fSSascha Hauer 	if (!request_mem_region(iores->start, resource_size(iores), pdev->name)) {
13301ec1e82fSSascha Hauer 		ret = -EBUSY;
13311ec1e82fSSascha Hauer 		goto err_request_region;
13321ec1e82fSSascha Hauer 	}
13331ec1e82fSSascha Hauer 
13347560e3f3SSascha Hauer 	sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
13357560e3f3SSascha Hauer 	if (IS_ERR(sdma->clk_ipg)) {
13367560e3f3SSascha Hauer 		ret = PTR_ERR(sdma->clk_ipg);
13371ec1e82fSSascha Hauer 		goto err_clk;
13381ec1e82fSSascha Hauer 	}
13391ec1e82fSSascha Hauer 
13407560e3f3SSascha Hauer 	sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
13417560e3f3SSascha Hauer 	if (IS_ERR(sdma->clk_ahb)) {
13427560e3f3SSascha Hauer 		ret = PTR_ERR(sdma->clk_ahb);
13437560e3f3SSascha Hauer 		goto err_clk;
13447560e3f3SSascha Hauer 	}
13457560e3f3SSascha Hauer 
13467560e3f3SSascha Hauer 	clk_prepare(sdma->clk_ipg);
13477560e3f3SSascha Hauer 	clk_prepare(sdma->clk_ahb);
13487560e3f3SSascha Hauer 
13491ec1e82fSSascha Hauer 	sdma->regs = ioremap(iores->start, resource_size(iores));
13501ec1e82fSSascha Hauer 	if (!sdma->regs) {
13511ec1e82fSSascha Hauer 		ret = -ENOMEM;
13521ec1e82fSSascha Hauer 		goto err_ioremap;
13531ec1e82fSSascha Hauer 	}
13541ec1e82fSSascha Hauer 
13551ec1e82fSSascha Hauer 	ret = request_irq(irq, sdma_int_handler, 0, "sdma", sdma);
13561ec1e82fSSascha Hauer 	if (ret)
13571ec1e82fSSascha Hauer 		goto err_request_irq;
13581ec1e82fSSascha Hauer 
13595b28aa31SSascha Hauer 	sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
13601c1d9547SAxel Lin 	if (!sdma->script_addrs) {
13611c1d9547SAxel Lin 		ret = -ENOMEM;
13625b28aa31SSascha Hauer 		goto err_alloc;
13631c1d9547SAxel Lin 	}
13641ec1e82fSSascha Hauer 
136536e2f21aSSascha Hauer 	/* initially no scripts available */
136636e2f21aSSascha Hauer 	saddr_arr = (s32 *)sdma->script_addrs;
136736e2f21aSSascha Hauer 	for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++)
136836e2f21aSSascha Hauer 		saddr_arr[i] = -EINVAL;
136936e2f21aSSascha Hauer 
1370580975d7SShawn Guo 	if (of_id)
1371580975d7SShawn Guo 		pdev->id_entry = of_id->data;
137262550cd7SShawn Guo 	sdma->devtype = pdev->id_entry->driver_data;
13731ec1e82fSSascha Hauer 
13747214a8b1SSascha Hauer 	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
13757214a8b1SSascha Hauer 	dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
13767214a8b1SSascha Hauer 
13771ec1e82fSSascha Hauer 	INIT_LIST_HEAD(&sdma->dma_device.channels);
13781ec1e82fSSascha Hauer 	/* Initialize channel parameters */
13791ec1e82fSSascha Hauer 	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
13801ec1e82fSSascha Hauer 		struct sdma_channel *sdmac = &sdma->channel[i];
13811ec1e82fSSascha Hauer 
13821ec1e82fSSascha Hauer 		sdmac->sdma = sdma;
13831ec1e82fSSascha Hauer 		spin_lock_init(&sdmac->lock);
13841ec1e82fSSascha Hauer 
13851ec1e82fSSascha Hauer 		sdmac->chan.device = &sdma->dma_device;
13868ac69546SRussell King - ARM Linux 		dma_cookie_init(&sdmac->chan);
13871ec1e82fSSascha Hauer 		sdmac->channel = i;
13881ec1e82fSSascha Hauer 
1389abd9ccc8SHuang Shijie 		tasklet_init(&sdmac->tasklet, sdma_tasklet,
1390abd9ccc8SHuang Shijie 			     (unsigned long) sdmac);
139123889c63SSascha Hauer 		/*
139223889c63SSascha Hauer 		 * Add the channel to the DMAC list. Do not add channel 0 though
139323889c63SSascha Hauer 		 * because we need it internally in the SDMA driver. This also means
139423889c63SSascha Hauer 		 * that channel 0 in dmaengine counting matches sdma channel 1.
139523889c63SSascha Hauer 		 */
139623889c63SSascha Hauer 		if (i)
139723889c63SSascha Hauer 			list_add_tail(&sdmac->chan.device_node,
139823889c63SSascha Hauer 					&sdma->dma_device.channels);
13991ec1e82fSSascha Hauer 	}
14001ec1e82fSSascha Hauer 
14015b28aa31SSascha Hauer 	ret = sdma_init(sdma);
14021ec1e82fSSascha Hauer 	if (ret)
14031ec1e82fSSascha Hauer 		goto err_init;
14041ec1e82fSSascha Hauer 
1405580975d7SShawn Guo 	if (pdata && pdata->script_addrs)
14065b28aa31SSascha Hauer 		sdma_add_scripts(sdma, pdata->script_addrs);
14075b28aa31SSascha Hauer 
1408580975d7SShawn Guo 	if (pdata) {
14096d0d7e2dSFabio Estevam 		ret = sdma_get_firmware(sdma, pdata->fw_name);
14106d0d7e2dSFabio Estevam 		if (ret)
1411ad1122e5SFabio Estevam 			dev_warn(&pdev->dev, "failed to get firmware from platform data\n");
1412580975d7SShawn Guo 	} else {
1413580975d7SShawn Guo 		/*
1414580975d7SShawn Guo 		 * Because that device tree does not encode ROM script address,
1415580975d7SShawn Guo 		 * the RAM script in firmware is mandatory for device tree
1416580975d7SShawn Guo 		 * probe, otherwise it fails.
1417580975d7SShawn Guo 		 */
1418580975d7SShawn Guo 		ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
1419580975d7SShawn Guo 					      &fw_name);
14206602b0ddSFabio Estevam 		if (ret)
1421ad1122e5SFabio Estevam 			dev_warn(&pdev->dev, "failed to get firmware name\n");
14226602b0ddSFabio Estevam 		else {
1423580975d7SShawn Guo 			ret = sdma_get_firmware(sdma, fw_name);
14246602b0ddSFabio Estevam 			if (ret)
1425ad1122e5SFabio Estevam 				dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
1426580975d7SShawn Guo 		}
1427580975d7SShawn Guo 	}
14285b28aa31SSascha Hauer 
14291ec1e82fSSascha Hauer 	sdma->dma_device.dev = &pdev->dev;
14301ec1e82fSSascha Hauer 
14311ec1e82fSSascha Hauer 	sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
14321ec1e82fSSascha Hauer 	sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
14331ec1e82fSSascha Hauer 	sdma->dma_device.device_tx_status = sdma_tx_status;
14341ec1e82fSSascha Hauer 	sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
14351ec1e82fSSascha Hauer 	sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
14361ec1e82fSSascha Hauer 	sdma->dma_device.device_control = sdma_control;
14371ec1e82fSSascha Hauer 	sdma->dma_device.device_issue_pending = sdma_issue_pending;
1438b9b3f82fSSascha Hauer 	sdma->dma_device.dev->dma_parms = &sdma->dma_parms;
1439b9b3f82fSSascha Hauer 	dma_set_max_seg_size(sdma->dma_device.dev, 65535);
14401ec1e82fSSascha Hauer 
14411ec1e82fSSascha Hauer 	ret = dma_async_device_register(&sdma->dma_device);
14421ec1e82fSSascha Hauer 	if (ret) {
14431ec1e82fSSascha Hauer 		dev_err(&pdev->dev, "unable to register\n");
14441ec1e82fSSascha Hauer 		goto err_init;
14451ec1e82fSSascha Hauer 	}
14461ec1e82fSSascha Hauer 
14475b28aa31SSascha Hauer 	dev_info(sdma->dev, "initialized\n");
14481ec1e82fSSascha Hauer 
14491ec1e82fSSascha Hauer 	return 0;
14501ec1e82fSSascha Hauer 
14511ec1e82fSSascha Hauer err_init:
14521ec1e82fSSascha Hauer 	kfree(sdma->script_addrs);
14535b28aa31SSascha Hauer err_alloc:
14541ec1e82fSSascha Hauer 	free_irq(irq, sdma);
14551ec1e82fSSascha Hauer err_request_irq:
14561ec1e82fSSascha Hauer 	iounmap(sdma->regs);
14571ec1e82fSSascha Hauer err_ioremap:
14581ec1e82fSSascha Hauer err_clk:
14591ec1e82fSSascha Hauer 	release_mem_region(iores->start, resource_size(iores));
14601ec1e82fSSascha Hauer err_request_region:
14611ec1e82fSSascha Hauer err_irq:
14621ec1e82fSSascha Hauer 	kfree(sdma);
1463939fd4f0SShawn Guo 	return ret;
14641ec1e82fSSascha Hauer }
14651ec1e82fSSascha Hauer 
14661ec1e82fSSascha Hauer static int __exit sdma_remove(struct platform_device *pdev)
14671ec1e82fSSascha Hauer {
14681ec1e82fSSascha Hauer 	return -EBUSY;
14691ec1e82fSSascha Hauer }
14701ec1e82fSSascha Hauer 
14711ec1e82fSSascha Hauer static struct platform_driver sdma_driver = {
14721ec1e82fSSascha Hauer 	.driver		= {
14731ec1e82fSSascha Hauer 		.name	= "imx-sdma",
1474580975d7SShawn Guo 		.of_match_table = sdma_dt_ids,
14751ec1e82fSSascha Hauer 	},
147662550cd7SShawn Guo 	.id_table	= sdma_devtypes,
14771ec1e82fSSascha Hauer 	.remove		= __exit_p(sdma_remove),
14781ec1e82fSSascha Hauer };
14791ec1e82fSSascha Hauer 
14801ec1e82fSSascha Hauer static int __init sdma_module_init(void)
14811ec1e82fSSascha Hauer {
14821ec1e82fSSascha Hauer 	return platform_driver_probe(&sdma_driver, sdma_probe);
14831ec1e82fSSascha Hauer }
1484c989a7fcSSascha Hauer module_init(sdma_module_init);
14851ec1e82fSSascha Hauer 
14861ec1e82fSSascha Hauer MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
14871ec1e82fSSascha Hauer MODULE_DESCRIPTION("i.MX SDMA driver");
14881ec1e82fSSascha Hauer MODULE_LICENSE("GPL");
1489