1c01faacaSFabio Estevam // SPDX-License-Identifier: GPL-2.0+ 2c01faacaSFabio Estevam // 3c01faacaSFabio Estevam // drivers/dma/imx-sdma.c 4c01faacaSFabio Estevam // 5c01faacaSFabio Estevam // This file contains a driver for the Freescale Smart DMA engine 6c01faacaSFabio Estevam // 7c01faacaSFabio Estevam // Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> 8c01faacaSFabio Estevam // 9c01faacaSFabio Estevam // Based on code from Freescale: 10c01faacaSFabio Estevam // 11c01faacaSFabio Estevam // Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved. 121ec1e82fSSascha Hauer 131ec1e82fSSascha Hauer #include <linux/init.h> 141d069bfaSMichael Olbrich #include <linux/iopoll.h> 15f8de8f4cSAxel Lin #include <linux/module.h> 161ec1e82fSSascha Hauer #include <linux/types.h> 170bbc1413SRichard Zhao #include <linux/bitops.h> 181ec1e82fSSascha Hauer #include <linux/mm.h> 191ec1e82fSSascha Hauer #include <linux/interrupt.h> 201ec1e82fSSascha Hauer #include <linux/clk.h> 212ccaef05SRichard Zhao #include <linux/delay.h> 221ec1e82fSSascha Hauer #include <linux/sched.h> 231ec1e82fSSascha Hauer #include <linux/semaphore.h> 241ec1e82fSSascha Hauer #include <linux/spinlock.h> 251ec1e82fSSascha Hauer #include <linux/device.h> 261ec1e82fSSascha Hauer #include <linux/dma-mapping.h> 271ec1e82fSSascha Hauer #include <linux/firmware.h> 281ec1e82fSSascha Hauer #include <linux/slab.h> 291ec1e82fSSascha Hauer #include <linux/platform_device.h> 301ec1e82fSSascha Hauer #include <linux/dmaengine.h> 31580975d7SShawn Guo #include <linux/of.h> 328391ecf4SShengjiu Wang #include <linux/of_address.h> 33580975d7SShawn Guo #include <linux/of_device.h> 349479e17cSShawn Guo #include <linux/of_dma.h> 35b8603d2aSLucas Stach #include <linux/workqueue.h> 361ec1e82fSSascha Hauer 371ec1e82fSSascha Hauer #include <asm/irq.h> 3882906b13SArnd Bergmann #include <linux/platform_data/dma-imx.h> 39d078cd1bSZidan Wang #include <linux/regmap.h> 40d078cd1bSZidan Wang #include <linux/mfd/syscon.h> 41d078cd1bSZidan Wang #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h> 421ec1e82fSSascha Hauer 43d2ebfb33SRussell King - ARM Linux #include "dmaengine.h" 4457b772b8SRobin Gong #include "virt-dma.h" 45d2ebfb33SRussell King - ARM Linux 461ec1e82fSSascha Hauer /* SDMA registers */ 471ec1e82fSSascha Hauer #define SDMA_H_C0PTR 0x000 481ec1e82fSSascha Hauer #define SDMA_H_INTR 0x004 491ec1e82fSSascha Hauer #define SDMA_H_STATSTOP 0x008 501ec1e82fSSascha Hauer #define SDMA_H_START 0x00c 511ec1e82fSSascha Hauer #define SDMA_H_EVTOVR 0x010 521ec1e82fSSascha Hauer #define SDMA_H_DSPOVR 0x014 531ec1e82fSSascha Hauer #define SDMA_H_HOSTOVR 0x018 541ec1e82fSSascha Hauer #define SDMA_H_EVTPEND 0x01c 551ec1e82fSSascha Hauer #define SDMA_H_DSPENBL 0x020 561ec1e82fSSascha Hauer #define SDMA_H_RESET 0x024 571ec1e82fSSascha Hauer #define SDMA_H_EVTERR 0x028 581ec1e82fSSascha Hauer #define SDMA_H_INTRMSK 0x02c 591ec1e82fSSascha Hauer #define SDMA_H_PSW 0x030 601ec1e82fSSascha Hauer #define SDMA_H_EVTERRDBG 0x034 611ec1e82fSSascha Hauer #define SDMA_H_CONFIG 0x038 621ec1e82fSSascha Hauer #define SDMA_ONCE_ENB 0x040 631ec1e82fSSascha Hauer #define SDMA_ONCE_DATA 0x044 641ec1e82fSSascha Hauer #define SDMA_ONCE_INSTR 0x048 651ec1e82fSSascha Hauer #define SDMA_ONCE_STAT 0x04c 661ec1e82fSSascha Hauer #define SDMA_ONCE_CMD 0x050 671ec1e82fSSascha Hauer #define SDMA_EVT_MIRROR 0x054 681ec1e82fSSascha Hauer #define SDMA_ILLINSTADDR 0x058 691ec1e82fSSascha Hauer #define SDMA_CHN0ADDR 0x05c 701ec1e82fSSascha Hauer #define SDMA_ONCE_RTB 0x060 711ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF1 0x070 721ec1e82fSSascha Hauer #define SDMA_XTRIG_CONF2 0x074 7362550cd7SShawn Guo #define SDMA_CHNENBL0_IMX35 0x200 7462550cd7SShawn Guo #define SDMA_CHNENBL0_IMX31 0x080 751ec1e82fSSascha Hauer #define SDMA_CHNPRI_0 0x100 761ec1e82fSSascha Hauer 771ec1e82fSSascha Hauer /* 781ec1e82fSSascha Hauer * Buffer descriptor status values. 791ec1e82fSSascha Hauer */ 801ec1e82fSSascha Hauer #define BD_DONE 0x01 811ec1e82fSSascha Hauer #define BD_WRAP 0x02 821ec1e82fSSascha Hauer #define BD_CONT 0x04 831ec1e82fSSascha Hauer #define BD_INTR 0x08 841ec1e82fSSascha Hauer #define BD_RROR 0x10 851ec1e82fSSascha Hauer #define BD_LAST 0x20 861ec1e82fSSascha Hauer #define BD_EXTD 0x80 871ec1e82fSSascha Hauer 881ec1e82fSSascha Hauer /* 891ec1e82fSSascha Hauer * Data Node descriptor status values. 901ec1e82fSSascha Hauer */ 911ec1e82fSSascha Hauer #define DND_END_OF_FRAME 0x80 921ec1e82fSSascha Hauer #define DND_END_OF_XFER 0x40 931ec1e82fSSascha Hauer #define DND_DONE 0x20 941ec1e82fSSascha Hauer #define DND_UNUSED 0x01 951ec1e82fSSascha Hauer 961ec1e82fSSascha Hauer /* 971ec1e82fSSascha Hauer * IPCV2 descriptor status values. 981ec1e82fSSascha Hauer */ 991ec1e82fSSascha Hauer #define BD_IPCV2_END_OF_FRAME 0x40 1001ec1e82fSSascha Hauer 1011ec1e82fSSascha Hauer #define IPCV2_MAX_NODES 50 1021ec1e82fSSascha Hauer /* 1031ec1e82fSSascha Hauer * Error bit set in the CCB status field by the SDMA, 1041ec1e82fSSascha Hauer * in setbd routine, in case of a transfer error 1051ec1e82fSSascha Hauer */ 1061ec1e82fSSascha Hauer #define DATA_ERROR 0x10000000 1071ec1e82fSSascha Hauer 1081ec1e82fSSascha Hauer /* 1091ec1e82fSSascha Hauer * Buffer descriptor commands. 1101ec1e82fSSascha Hauer */ 1111ec1e82fSSascha Hauer #define C0_ADDR 0x01 1121ec1e82fSSascha Hauer #define C0_LOAD 0x02 1131ec1e82fSSascha Hauer #define C0_DUMP 0x03 1141ec1e82fSSascha Hauer #define C0_SETCTX 0x07 1151ec1e82fSSascha Hauer #define C0_GETCTX 0x03 1161ec1e82fSSascha Hauer #define C0_SETDM 0x01 1171ec1e82fSSascha Hauer #define C0_SETPM 0x04 1181ec1e82fSSascha Hauer #define C0_GETDM 0x02 1191ec1e82fSSascha Hauer #define C0_GETPM 0x08 1201ec1e82fSSascha Hauer /* 1211ec1e82fSSascha Hauer * Change endianness indicator in the BD command field 1221ec1e82fSSascha Hauer */ 1231ec1e82fSSascha Hauer #define CHANGE_ENDIANNESS 0x80 1241ec1e82fSSascha Hauer 1251ec1e82fSSascha Hauer /* 1268391ecf4SShengjiu Wang * p_2_p watermark_level description 1278391ecf4SShengjiu Wang * Bits Name Description 1288391ecf4SShengjiu Wang * 0-7 Lower WML Lower watermark level 1298391ecf4SShengjiu Wang * 8 PS 1: Pad Swallowing 1308391ecf4SShengjiu Wang * 0: No Pad Swallowing 1318391ecf4SShengjiu Wang * 9 PA 1: Pad Adding 1328391ecf4SShengjiu Wang * 0: No Pad Adding 1338391ecf4SShengjiu Wang * 10 SPDIF If this bit is set both source 1348391ecf4SShengjiu Wang * and destination are on SPBA 1358391ecf4SShengjiu Wang * 11 Source Bit(SP) 1: Source on SPBA 1368391ecf4SShengjiu Wang * 0: Source on AIPS 1378391ecf4SShengjiu Wang * 12 Destination Bit(DP) 1: Destination on SPBA 1388391ecf4SShengjiu Wang * 0: Destination on AIPS 1398391ecf4SShengjiu Wang * 13-15 --------- MUST BE 0 1408391ecf4SShengjiu Wang * 16-23 Higher WML HWML 1418391ecf4SShengjiu Wang * 24-27 N Total number of samples after 1428391ecf4SShengjiu Wang * which Pad adding/Swallowing 1438391ecf4SShengjiu Wang * must be done. It must be odd. 1448391ecf4SShengjiu Wang * 28 Lower WML Event(LWE) SDMA events reg to check for 1458391ecf4SShengjiu Wang * LWML event mask 1468391ecf4SShengjiu Wang * 0: LWE in EVENTS register 1478391ecf4SShengjiu Wang * 1: LWE in EVENTS2 register 1488391ecf4SShengjiu Wang * 29 Higher WML Event(HWE) SDMA events reg to check for 1498391ecf4SShengjiu Wang * HWML event mask 1508391ecf4SShengjiu Wang * 0: HWE in EVENTS register 1518391ecf4SShengjiu Wang * 1: HWE in EVENTS2 register 1528391ecf4SShengjiu Wang * 30 --------- MUST BE 0 1538391ecf4SShengjiu Wang * 31 CONT 1: Amount of samples to be 1548391ecf4SShengjiu Wang * transferred is unknown and 1558391ecf4SShengjiu Wang * script will keep on 1568391ecf4SShengjiu Wang * transferring samples as long as 1578391ecf4SShengjiu Wang * both events are detected and 1588391ecf4SShengjiu Wang * script must be manually stopped 1598391ecf4SShengjiu Wang * by the application 1608391ecf4SShengjiu Wang * 0: The amount of samples to be 1618391ecf4SShengjiu Wang * transferred is equal to the 1628391ecf4SShengjiu Wang * count field of mode word 1638391ecf4SShengjiu Wang */ 1648391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_LWML 0xFF 1658391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_PS BIT(8) 1668391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_PA BIT(9) 1678391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) 1688391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_SP BIT(11) 1698391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_DP BIT(12) 1708391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) 1718391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_LWE BIT(28) 1728391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_HWE BIT(29) 1738391ecf4SShengjiu Wang #define SDMA_WATERMARK_LEVEL_CONT BIT(31) 1748391ecf4SShengjiu Wang 175f9d4a398SNicolin Chen #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 176f9d4a398SNicolin Chen BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 177f9d4a398SNicolin Chen BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 178f9d4a398SNicolin Chen 179f9d4a398SNicolin Chen #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ 180f9d4a398SNicolin Chen BIT(DMA_MEM_TO_DEV) | \ 181f9d4a398SNicolin Chen BIT(DMA_DEV_TO_DEV)) 182f9d4a398SNicolin Chen 1838d11cfb0SVladimir Zapolskiy /** 1848d11cfb0SVladimir Zapolskiy * struct sdma_script_start_addrs - SDMA script start pointers 1858d11cfb0SVladimir Zapolskiy * 1868d11cfb0SVladimir Zapolskiy * start addresses of the different functions in the physical 1878d11cfb0SVladimir Zapolskiy * address space of the SDMA engine. 1888d11cfb0SVladimir Zapolskiy */ 1898d11cfb0SVladimir Zapolskiy struct sdma_script_start_addrs { 1908d11cfb0SVladimir Zapolskiy s32 ap_2_ap_addr; 1918d11cfb0SVladimir Zapolskiy s32 ap_2_bp_addr; 1928d11cfb0SVladimir Zapolskiy s32 ap_2_ap_fixed_addr; 1938d11cfb0SVladimir Zapolskiy s32 bp_2_ap_addr; 1948d11cfb0SVladimir Zapolskiy s32 loopback_on_dsp_side_addr; 1958d11cfb0SVladimir Zapolskiy s32 mcu_interrupt_only_addr; 1968d11cfb0SVladimir Zapolskiy s32 firi_2_per_addr; 1978d11cfb0SVladimir Zapolskiy s32 firi_2_mcu_addr; 1988d11cfb0SVladimir Zapolskiy s32 per_2_firi_addr; 1998d11cfb0SVladimir Zapolskiy s32 mcu_2_firi_addr; 2008d11cfb0SVladimir Zapolskiy s32 uart_2_per_addr; 2018d11cfb0SVladimir Zapolskiy s32 uart_2_mcu_addr; 2028d11cfb0SVladimir Zapolskiy s32 per_2_app_addr; 2038d11cfb0SVladimir Zapolskiy s32 mcu_2_app_addr; 2048d11cfb0SVladimir Zapolskiy s32 per_2_per_addr; 2058d11cfb0SVladimir Zapolskiy s32 uartsh_2_per_addr; 2068d11cfb0SVladimir Zapolskiy s32 uartsh_2_mcu_addr; 2078d11cfb0SVladimir Zapolskiy s32 per_2_shp_addr; 2088d11cfb0SVladimir Zapolskiy s32 mcu_2_shp_addr; 2098d11cfb0SVladimir Zapolskiy s32 ata_2_mcu_addr; 2108d11cfb0SVladimir Zapolskiy s32 mcu_2_ata_addr; 2118d11cfb0SVladimir Zapolskiy s32 app_2_per_addr; 2128d11cfb0SVladimir Zapolskiy s32 app_2_mcu_addr; 2138d11cfb0SVladimir Zapolskiy s32 shp_2_per_addr; 2148d11cfb0SVladimir Zapolskiy s32 shp_2_mcu_addr; 2158d11cfb0SVladimir Zapolskiy s32 mshc_2_mcu_addr; 2168d11cfb0SVladimir Zapolskiy s32 mcu_2_mshc_addr; 2178d11cfb0SVladimir Zapolskiy s32 spdif_2_mcu_addr; 2188d11cfb0SVladimir Zapolskiy s32 mcu_2_spdif_addr; 2198d11cfb0SVladimir Zapolskiy s32 asrc_2_mcu_addr; 2208d11cfb0SVladimir Zapolskiy s32 ext_mem_2_ipu_addr; 2218d11cfb0SVladimir Zapolskiy s32 descrambler_addr; 2228d11cfb0SVladimir Zapolskiy s32 dptc_dvfs_addr; 2238d11cfb0SVladimir Zapolskiy s32 utra_addr; 2248d11cfb0SVladimir Zapolskiy s32 ram_code_start_addr; 2258d11cfb0SVladimir Zapolskiy /* End of v1 array */ 2268d11cfb0SVladimir Zapolskiy s32 mcu_2_ssish_addr; 2278d11cfb0SVladimir Zapolskiy s32 ssish_2_mcu_addr; 2288d11cfb0SVladimir Zapolskiy s32 hdmi_dma_addr; 2298d11cfb0SVladimir Zapolskiy /* End of v2 array */ 2308d11cfb0SVladimir Zapolskiy s32 zcanfd_2_mcu_addr; 2318d11cfb0SVladimir Zapolskiy s32 zqspi_2_mcu_addr; 2328d11cfb0SVladimir Zapolskiy s32 mcu_2_ecspi_addr; 2338d11cfb0SVladimir Zapolskiy /* End of v3 array */ 2348d11cfb0SVladimir Zapolskiy s32 mcu_2_zqspi_addr; 2358d11cfb0SVladimir Zapolskiy /* End of v4 array */ 2368d11cfb0SVladimir Zapolskiy }; 2378d11cfb0SVladimir Zapolskiy 2388391ecf4SShengjiu Wang /* 2391ec1e82fSSascha Hauer * Mode/Count of data node descriptors - IPCv2 2401ec1e82fSSascha Hauer */ 2411ec1e82fSSascha Hauer struct sdma_mode_count { 2424a6b2e8aSRobin Gong #define SDMA_BD_MAX_CNT 0xffff 2431ec1e82fSSascha Hauer u32 count : 16; /* size of the buffer pointed by this BD */ 2441ec1e82fSSascha Hauer u32 status : 8; /* E,R,I,C,W,D status bits stored here */ 245e4b75760SMartin Kaiser u32 command : 8; /* command mostly used for channel 0 */ 2461ec1e82fSSascha Hauer }; 2471ec1e82fSSascha Hauer 2481ec1e82fSSascha Hauer /* 2491ec1e82fSSascha Hauer * Buffer descriptor 2501ec1e82fSSascha Hauer */ 2511ec1e82fSSascha Hauer struct sdma_buffer_descriptor { 2521ec1e82fSSascha Hauer struct sdma_mode_count mode; 2531ec1e82fSSascha Hauer u32 buffer_addr; /* address of the buffer described */ 2541ec1e82fSSascha Hauer u32 ext_buffer_addr; /* extended buffer address */ 2551ec1e82fSSascha Hauer } __attribute__ ((packed)); 2561ec1e82fSSascha Hauer 2571ec1e82fSSascha Hauer /** 2581ec1e82fSSascha Hauer * struct sdma_channel_control - Channel control Block 2591ec1e82fSSascha Hauer * 26024ca312dSRobin Gong * @current_bd_ptr: current buffer descriptor processed 26124ca312dSRobin Gong * @base_bd_ptr: first element of buffer descriptor array 26224ca312dSRobin Gong * @unused: padding. The SDMA engine expects an array of 128 byte 2631ec1e82fSSascha Hauer * control blocks 2641ec1e82fSSascha Hauer */ 2651ec1e82fSSascha Hauer struct sdma_channel_control { 2661ec1e82fSSascha Hauer u32 current_bd_ptr; 2671ec1e82fSSascha Hauer u32 base_bd_ptr; 2681ec1e82fSSascha Hauer u32 unused[2]; 2691ec1e82fSSascha Hauer } __attribute__ ((packed)); 2701ec1e82fSSascha Hauer 2711ec1e82fSSascha Hauer /** 2721ec1e82fSSascha Hauer * struct sdma_state_registers - SDMA context for a channel 2731ec1e82fSSascha Hauer * 2741ec1e82fSSascha Hauer * @pc: program counter 27524ca312dSRobin Gong * @unused1: unused 2761ec1e82fSSascha Hauer * @t: test bit: status of arithmetic & test instruction 2771ec1e82fSSascha Hauer * @rpc: return program counter 27824ca312dSRobin Gong * @unused0: unused 2791ec1e82fSSascha Hauer * @sf: source fault while loading data 2801ec1e82fSSascha Hauer * @spc: loop start program counter 28124ca312dSRobin Gong * @unused2: unused 2821ec1e82fSSascha Hauer * @df: destination fault while storing data 2831ec1e82fSSascha Hauer * @epc: loop end program counter 2841ec1e82fSSascha Hauer * @lm: loop mode 2851ec1e82fSSascha Hauer */ 2861ec1e82fSSascha Hauer struct sdma_state_registers { 2871ec1e82fSSascha Hauer u32 pc :14; 2881ec1e82fSSascha Hauer u32 unused1: 1; 2891ec1e82fSSascha Hauer u32 t : 1; 2901ec1e82fSSascha Hauer u32 rpc :14; 2911ec1e82fSSascha Hauer u32 unused0: 1; 2921ec1e82fSSascha Hauer u32 sf : 1; 2931ec1e82fSSascha Hauer u32 spc :14; 2941ec1e82fSSascha Hauer u32 unused2: 1; 2951ec1e82fSSascha Hauer u32 df : 1; 2961ec1e82fSSascha Hauer u32 epc :14; 2971ec1e82fSSascha Hauer u32 lm : 2; 2981ec1e82fSSascha Hauer } __attribute__ ((packed)); 2991ec1e82fSSascha Hauer 3001ec1e82fSSascha Hauer /** 3011ec1e82fSSascha Hauer * struct sdma_context_data - sdma context specific to a channel 3021ec1e82fSSascha Hauer * 3031ec1e82fSSascha Hauer * @channel_state: channel state bits 3041ec1e82fSSascha Hauer * @gReg: general registers 3051ec1e82fSSascha Hauer * @mda: burst dma destination address register 3061ec1e82fSSascha Hauer * @msa: burst dma source address register 3071ec1e82fSSascha Hauer * @ms: burst dma status register 3081ec1e82fSSascha Hauer * @md: burst dma data register 3091ec1e82fSSascha Hauer * @pda: peripheral dma destination address register 3101ec1e82fSSascha Hauer * @psa: peripheral dma source address register 3111ec1e82fSSascha Hauer * @ps: peripheral dma status register 3121ec1e82fSSascha Hauer * @pd: peripheral dma data register 3131ec1e82fSSascha Hauer * @ca: CRC polynomial register 3141ec1e82fSSascha Hauer * @cs: CRC accumulator register 3151ec1e82fSSascha Hauer * @dda: dedicated core destination address register 3161ec1e82fSSascha Hauer * @dsa: dedicated core source address register 3171ec1e82fSSascha Hauer * @ds: dedicated core status register 3181ec1e82fSSascha Hauer * @dd: dedicated core data register 31924ca312dSRobin Gong * @scratch0: 1st word of dedicated ram for context switch 32024ca312dSRobin Gong * @scratch1: 2nd word of dedicated ram for context switch 32124ca312dSRobin Gong * @scratch2: 3rd word of dedicated ram for context switch 32224ca312dSRobin Gong * @scratch3: 4th word of dedicated ram for context switch 32324ca312dSRobin Gong * @scratch4: 5th word of dedicated ram for context switch 32424ca312dSRobin Gong * @scratch5: 6th word of dedicated ram for context switch 32524ca312dSRobin Gong * @scratch6: 7th word of dedicated ram for context switch 32624ca312dSRobin Gong * @scratch7: 8th word of dedicated ram for context switch 3271ec1e82fSSascha Hauer */ 3281ec1e82fSSascha Hauer struct sdma_context_data { 3291ec1e82fSSascha Hauer struct sdma_state_registers channel_state; 3301ec1e82fSSascha Hauer u32 gReg[8]; 3311ec1e82fSSascha Hauer u32 mda; 3321ec1e82fSSascha Hauer u32 msa; 3331ec1e82fSSascha Hauer u32 ms; 3341ec1e82fSSascha Hauer u32 md; 3351ec1e82fSSascha Hauer u32 pda; 3361ec1e82fSSascha Hauer u32 psa; 3371ec1e82fSSascha Hauer u32 ps; 3381ec1e82fSSascha Hauer u32 pd; 3391ec1e82fSSascha Hauer u32 ca; 3401ec1e82fSSascha Hauer u32 cs; 3411ec1e82fSSascha Hauer u32 dda; 3421ec1e82fSSascha Hauer u32 dsa; 3431ec1e82fSSascha Hauer u32 ds; 3441ec1e82fSSascha Hauer u32 dd; 3451ec1e82fSSascha Hauer u32 scratch0; 3461ec1e82fSSascha Hauer u32 scratch1; 3471ec1e82fSSascha Hauer u32 scratch2; 3481ec1e82fSSascha Hauer u32 scratch3; 3491ec1e82fSSascha Hauer u32 scratch4; 3501ec1e82fSSascha Hauer u32 scratch5; 3511ec1e82fSSascha Hauer u32 scratch6; 3521ec1e82fSSascha Hauer u32 scratch7; 3531ec1e82fSSascha Hauer } __attribute__ ((packed)); 3541ec1e82fSSascha Hauer 3551ec1e82fSSascha Hauer 3561ec1e82fSSascha Hauer struct sdma_engine; 3571ec1e82fSSascha Hauer 3581ec1e82fSSascha Hauer /** 35976c33d27SSascha Hauer * struct sdma_desc - descriptor structor for one transfer 36024ca312dSRobin Gong * @vd: descriptor for virt dma 36124ca312dSRobin Gong * @num_bd: number of descriptors currently handling 36224ca312dSRobin Gong * @bd_phys: physical address of bd 36324ca312dSRobin Gong * @buf_tail: ID of the buffer that was processed 36424ca312dSRobin Gong * @buf_ptail: ID of the previous buffer that was processed 36524ca312dSRobin Gong * @period_len: period length, used in cyclic. 36624ca312dSRobin Gong * @chn_real_count: the real count updated from bd->mode.count 36724ca312dSRobin Gong * @chn_count: the transfer count set 36824ca312dSRobin Gong * @sdmac: sdma_channel pointer 36924ca312dSRobin Gong * @bd: pointer of allocate bd 37076c33d27SSascha Hauer */ 37176c33d27SSascha Hauer struct sdma_desc { 37257b772b8SRobin Gong struct virt_dma_desc vd; 37376c33d27SSascha Hauer unsigned int num_bd; 37476c33d27SSascha Hauer dma_addr_t bd_phys; 37576c33d27SSascha Hauer unsigned int buf_tail; 37676c33d27SSascha Hauer unsigned int buf_ptail; 37776c33d27SSascha Hauer unsigned int period_len; 37876c33d27SSascha Hauer unsigned int chn_real_count; 37976c33d27SSascha Hauer unsigned int chn_count; 38076c33d27SSascha Hauer struct sdma_channel *sdmac; 38176c33d27SSascha Hauer struct sdma_buffer_descriptor *bd; 38276c33d27SSascha Hauer }; 38376c33d27SSascha Hauer 38476c33d27SSascha Hauer /** 3851ec1e82fSSascha Hauer * struct sdma_channel - housekeeping for a SDMA channel 3861ec1e82fSSascha Hauer * 38724ca312dSRobin Gong * @vc: virt_dma base structure 38824ca312dSRobin Gong * @desc: sdma description including vd and other special member 38924ca312dSRobin Gong * @sdma: pointer to the SDMA engine for this channel 39024ca312dSRobin Gong * @channel: the channel number, matches dmaengine chan_id + 1 39124ca312dSRobin Gong * @direction: transfer type. Needed for setting SDMA script 392d0c4a149SLee Jones * @slave_config: Slave configuration 39324ca312dSRobin Gong * @peripheral_type: Peripheral type. Needed for setting SDMA script 39424ca312dSRobin Gong * @event_id0: aka dma request line 39524ca312dSRobin Gong * @event_id1: for channels that use 2 events 39624ca312dSRobin Gong * @word_size: peripheral access size 39724ca312dSRobin Gong * @pc_from_device: script address for those device_2_memory 39824ca312dSRobin Gong * @pc_to_device: script address for those memory_2_device 39924ca312dSRobin Gong * @device_to_device: script address for those device_2_device 4000f06c027SRobin Gong * @pc_to_pc: script address for those memory_2_memory 40124ca312dSRobin Gong * @flags: loop mode or not 40224ca312dSRobin Gong * @per_address: peripheral source or destination address in common case 40324ca312dSRobin Gong * destination address in p_2_p case 40424ca312dSRobin Gong * @per_address2: peripheral source address in p_2_p case 40524ca312dSRobin Gong * @event_mask: event mask used in p_2_p script 40624ca312dSRobin Gong * @watermark_level: value for gReg[7], some script will extend it from 40724ca312dSRobin Gong * basic watermark such as p_2_p 40824ca312dSRobin Gong * @shp_addr: value for gReg[6] 40924ca312dSRobin Gong * @per_addr: value for gReg[2] 41024ca312dSRobin Gong * @status: status of dma channel 411d0c4a149SLee Jones * @context_loaded: ensure context is only loaded once 41224ca312dSRobin Gong * @data: specific sdma interface structure 41324ca312dSRobin Gong * @bd_pool: dma_pool for bd 414d0c4a149SLee Jones * @terminate_worker: used to call back into terminate work function 4151ec1e82fSSascha Hauer */ 4161ec1e82fSSascha Hauer struct sdma_channel { 41757b772b8SRobin Gong struct virt_dma_chan vc; 41876c33d27SSascha Hauer struct sdma_desc *desc; 4191ec1e82fSSascha Hauer struct sdma_engine *sdma; 4201ec1e82fSSascha Hauer unsigned int channel; 421db8196dfSVinod Koul enum dma_transfer_direction direction; 422107d0644SVinod Koul struct dma_slave_config slave_config; 4231ec1e82fSSascha Hauer enum sdma_peripheral_type peripheral_type; 4241ec1e82fSSascha Hauer unsigned int event_id0; 4251ec1e82fSSascha Hauer unsigned int event_id1; 4261ec1e82fSSascha Hauer enum dma_slave_buswidth word_size; 4271ec1e82fSSascha Hauer unsigned int pc_from_device, pc_to_device; 4288391ecf4SShengjiu Wang unsigned int device_to_device; 4290f06c027SRobin Gong unsigned int pc_to_pc; 4301ec1e82fSSascha Hauer unsigned long flags; 4318391ecf4SShengjiu Wang dma_addr_t per_address, per_address2; 4320bbc1413SRichard Zhao unsigned long event_mask[2]; 4330bbc1413SRichard Zhao unsigned long watermark_level; 4341ec1e82fSSascha Hauer u32 shp_addr, per_addr; 4351ec1e82fSSascha Hauer enum dma_status status; 4360b351865SNicolin Chen struct imx_dma_data data; 437b8603d2aSLucas Stach struct work_struct terminate_worker; 4381ec1e82fSSascha Hauer }; 4391ec1e82fSSascha Hauer 4400bbc1413SRichard Zhao #define IMX_DMA_SG_LOOP BIT(0) 4411ec1e82fSSascha Hauer 4421ec1e82fSSascha Hauer #define MAX_DMA_CHANNELS 32 4431ec1e82fSSascha Hauer #define MXC_SDMA_DEFAULT_PRIORITY 1 4441ec1e82fSSascha Hauer #define MXC_SDMA_MIN_PRIORITY 1 4451ec1e82fSSascha Hauer #define MXC_SDMA_MAX_PRIORITY 7 4461ec1e82fSSascha Hauer 4471ec1e82fSSascha Hauer #define SDMA_FIRMWARE_MAGIC 0x414d4453 4481ec1e82fSSascha Hauer 4491ec1e82fSSascha Hauer /** 4501ec1e82fSSascha Hauer * struct sdma_firmware_header - Layout of the firmware image 4511ec1e82fSSascha Hauer * 45224ca312dSRobin Gong * @magic: "SDMA" 45324ca312dSRobin Gong * @version_major: increased whenever layout of struct 45424ca312dSRobin Gong * sdma_script_start_addrs changes. 45524ca312dSRobin Gong * @version_minor: firmware minor version (for binary compatible changes) 45624ca312dSRobin Gong * @script_addrs_start: offset of struct sdma_script_start_addrs in this image 45724ca312dSRobin Gong * @num_script_addrs: Number of script addresses in this image 45824ca312dSRobin Gong * @ram_code_start: offset of SDMA ram image in this firmware image 45924ca312dSRobin Gong * @ram_code_size: size of SDMA ram image 46024ca312dSRobin Gong * @script_addrs: Stores the start address of the SDMA scripts 4611ec1e82fSSascha Hauer * (in SDMA memory space) 4621ec1e82fSSascha Hauer */ 4631ec1e82fSSascha Hauer struct sdma_firmware_header { 4641ec1e82fSSascha Hauer u32 magic; 4651ec1e82fSSascha Hauer u32 version_major; 4661ec1e82fSSascha Hauer u32 version_minor; 4671ec1e82fSSascha Hauer u32 script_addrs_start; 4681ec1e82fSSascha Hauer u32 num_script_addrs; 4691ec1e82fSSascha Hauer u32 ram_code_start; 4701ec1e82fSSascha Hauer u32 ram_code_size; 4711ec1e82fSSascha Hauer }; 4721ec1e82fSSascha Hauer 47317bba72fSSascha Hauer struct sdma_driver_data { 47417bba72fSSascha Hauer int chnenbl0; 47517bba72fSSascha Hauer int num_events; 476dcfec3c0SSascha Hauer struct sdma_script_start_addrs *script_addrs; 477941acd56SAngus Ainslie (Purism) bool check_ratio; 47862550cd7SShawn Guo }; 47962550cd7SShawn Guo 4801ec1e82fSSascha Hauer struct sdma_engine { 4811ec1e82fSSascha Hauer struct device *dev; 4821ec1e82fSSascha Hauer struct sdma_channel channel[MAX_DMA_CHANNELS]; 4831ec1e82fSSascha Hauer struct sdma_channel_control *channel_control; 4841ec1e82fSSascha Hauer void __iomem *regs; 4851ec1e82fSSascha Hauer struct sdma_context_data *context; 4861ec1e82fSSascha Hauer dma_addr_t context_phys; 4871ec1e82fSSascha Hauer struct dma_device dma_device; 4887560e3f3SSascha Hauer struct clk *clk_ipg; 4897560e3f3SSascha Hauer struct clk *clk_ahb; 4902ccaef05SRichard Zhao spinlock_t channel_0_lock; 491cd72b846SNicolin Chen u32 script_number; 4921ec1e82fSSascha Hauer struct sdma_script_start_addrs *script_addrs; 49317bba72fSSascha Hauer const struct sdma_driver_data *drvdata; 4948391ecf4SShengjiu Wang u32 spba_start_addr; 4958391ecf4SShengjiu Wang u32 spba_end_addr; 4965bb9dbb5SVinod Koul unsigned int irq; 49776c33d27SSascha Hauer dma_addr_t bd0_phys; 49876c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0; 49925aaa75dSAngus Ainslie (Purism) /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ 50025aaa75dSAngus Ainslie (Purism) bool clk_ratio; 50117bba72fSSascha Hauer }; 50217bba72fSSascha Hauer 503107d0644SVinod Koul static int sdma_config_write(struct dma_chan *chan, 504107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg, 505107d0644SVinod Koul enum dma_transfer_direction direction); 506107d0644SVinod Koul 507e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx31 = { 50817bba72fSSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX31, 50917bba72fSSascha Hauer .num_events = 32, 51017bba72fSSascha Hauer }; 51117bba72fSSascha Hauer 512dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx25 = { 513dcfec3c0SSascha Hauer .ap_2_ap_addr = 729, 514dcfec3c0SSascha Hauer .uart_2_mcu_addr = 904, 515dcfec3c0SSascha Hauer .per_2_app_addr = 1255, 516dcfec3c0SSascha Hauer .mcu_2_app_addr = 834, 517dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1120, 518dcfec3c0SSascha Hauer .per_2_shp_addr = 1329, 519dcfec3c0SSascha Hauer .mcu_2_shp_addr = 1048, 520dcfec3c0SSascha Hauer .ata_2_mcu_addr = 1560, 521dcfec3c0SSascha Hauer .mcu_2_ata_addr = 1479, 522dcfec3c0SSascha Hauer .app_2_per_addr = 1189, 523dcfec3c0SSascha Hauer .app_2_mcu_addr = 770, 524dcfec3c0SSascha Hauer .shp_2_per_addr = 1407, 525dcfec3c0SSascha Hauer .shp_2_mcu_addr = 979, 526dcfec3c0SSascha Hauer }; 527dcfec3c0SSascha Hauer 528e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx25 = { 529dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35, 530dcfec3c0SSascha Hauer .num_events = 48, 531dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx25, 532dcfec3c0SSascha Hauer }; 533dcfec3c0SSascha Hauer 534e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx35 = { 53517bba72fSSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35, 53617bba72fSSascha Hauer .num_events = 48, 5371ec1e82fSSascha Hauer }; 5381ec1e82fSSascha Hauer 539dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx51 = { 540dcfec3c0SSascha Hauer .ap_2_ap_addr = 642, 541dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817, 542dcfec3c0SSascha Hauer .mcu_2_app_addr = 747, 543dcfec3c0SSascha Hauer .mcu_2_shp_addr = 961, 544dcfec3c0SSascha Hauer .ata_2_mcu_addr = 1473, 545dcfec3c0SSascha Hauer .mcu_2_ata_addr = 1392, 546dcfec3c0SSascha Hauer .app_2_per_addr = 1033, 547dcfec3c0SSascha Hauer .app_2_mcu_addr = 683, 548dcfec3c0SSascha Hauer .shp_2_per_addr = 1251, 549dcfec3c0SSascha Hauer .shp_2_mcu_addr = 892, 550dcfec3c0SSascha Hauer }; 551dcfec3c0SSascha Hauer 552e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx51 = { 553dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35, 554dcfec3c0SSascha Hauer .num_events = 48, 555dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx51, 556dcfec3c0SSascha Hauer }; 557dcfec3c0SSascha Hauer 558dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx53 = { 559dcfec3c0SSascha Hauer .ap_2_ap_addr = 642, 560dcfec3c0SSascha Hauer .app_2_mcu_addr = 683, 561dcfec3c0SSascha Hauer .mcu_2_app_addr = 747, 562dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817, 563dcfec3c0SSascha Hauer .shp_2_mcu_addr = 891, 564dcfec3c0SSascha Hauer .mcu_2_shp_addr = 960, 565dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1032, 566dcfec3c0SSascha Hauer .spdif_2_mcu_addr = 1100, 567dcfec3c0SSascha Hauer .mcu_2_spdif_addr = 1134, 568dcfec3c0SSascha Hauer .firi_2_mcu_addr = 1193, 569dcfec3c0SSascha Hauer .mcu_2_firi_addr = 1290, 570dcfec3c0SSascha Hauer }; 571dcfec3c0SSascha Hauer 572e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx53 = { 573dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35, 574dcfec3c0SSascha Hauer .num_events = 48, 575dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx53, 576dcfec3c0SSascha Hauer }; 577dcfec3c0SSascha Hauer 578dcfec3c0SSascha Hauer static struct sdma_script_start_addrs sdma_script_imx6q = { 579dcfec3c0SSascha Hauer .ap_2_ap_addr = 642, 580dcfec3c0SSascha Hauer .uart_2_mcu_addr = 817, 581dcfec3c0SSascha Hauer .mcu_2_app_addr = 747, 582dcfec3c0SSascha Hauer .per_2_per_addr = 6331, 583dcfec3c0SSascha Hauer .uartsh_2_mcu_addr = 1032, 584dcfec3c0SSascha Hauer .mcu_2_shp_addr = 960, 585dcfec3c0SSascha Hauer .app_2_mcu_addr = 683, 586dcfec3c0SSascha Hauer .shp_2_mcu_addr = 891, 587dcfec3c0SSascha Hauer .spdif_2_mcu_addr = 1100, 588dcfec3c0SSascha Hauer .mcu_2_spdif_addr = 1134, 589dcfec3c0SSascha Hauer }; 590dcfec3c0SSascha Hauer 591e9fd58deSFabio Estevam static struct sdma_driver_data sdma_imx6q = { 592dcfec3c0SSascha Hauer .chnenbl0 = SDMA_CHNENBL0_IMX35, 593dcfec3c0SSascha Hauer .num_events = 48, 594dcfec3c0SSascha Hauer .script_addrs = &sdma_script_imx6q, 595dcfec3c0SSascha Hauer }; 596dcfec3c0SSascha Hauer 597b7d2648aSFabio Estevam static struct sdma_script_start_addrs sdma_script_imx7d = { 598b7d2648aSFabio Estevam .ap_2_ap_addr = 644, 599b7d2648aSFabio Estevam .uart_2_mcu_addr = 819, 600b7d2648aSFabio Estevam .mcu_2_app_addr = 749, 601b7d2648aSFabio Estevam .uartsh_2_mcu_addr = 1034, 602b7d2648aSFabio Estevam .mcu_2_shp_addr = 962, 603b7d2648aSFabio Estevam .app_2_mcu_addr = 685, 604b7d2648aSFabio Estevam .shp_2_mcu_addr = 893, 605b7d2648aSFabio Estevam .spdif_2_mcu_addr = 1102, 606b7d2648aSFabio Estevam .mcu_2_spdif_addr = 1136, 607b7d2648aSFabio Estevam }; 608b7d2648aSFabio Estevam 609b7d2648aSFabio Estevam static struct sdma_driver_data sdma_imx7d = { 610b7d2648aSFabio Estevam .chnenbl0 = SDMA_CHNENBL0_IMX35, 611b7d2648aSFabio Estevam .num_events = 48, 612b7d2648aSFabio Estevam .script_addrs = &sdma_script_imx7d, 613b7d2648aSFabio Estevam }; 614b7d2648aSFabio Estevam 615941acd56SAngus Ainslie (Purism) static struct sdma_driver_data sdma_imx8mq = { 616941acd56SAngus Ainslie (Purism) .chnenbl0 = SDMA_CHNENBL0_IMX35, 617941acd56SAngus Ainslie (Purism) .num_events = 48, 618941acd56SAngus Ainslie (Purism) .script_addrs = &sdma_script_imx7d, 619941acd56SAngus Ainslie (Purism) .check_ratio = 1, 620941acd56SAngus Ainslie (Purism) }; 621941acd56SAngus Ainslie (Purism) 622580975d7SShawn Guo static const struct of_device_id sdma_dt_ids[] = { 623dcfec3c0SSascha Hauer { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, 624dcfec3c0SSascha Hauer { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, }, 625dcfec3c0SSascha Hauer { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, }, 62617bba72fSSascha Hauer { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, }, 627dcfec3c0SSascha Hauer { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, 62863edea16SMarkus Pargmann { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, 629b7d2648aSFabio Estevam { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, 630941acd56SAngus Ainslie (Purism) { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, 631580975d7SShawn Guo { /* sentinel */ } 632580975d7SShawn Guo }; 633580975d7SShawn Guo MODULE_DEVICE_TABLE(of, sdma_dt_ids); 634580975d7SShawn Guo 6350bbc1413SRichard Zhao #define SDMA_H_CONFIG_DSPDMA BIT(12) /* indicates if the DSPDMA is used */ 6360bbc1413SRichard Zhao #define SDMA_H_CONFIG_RTD_PINS BIT(11) /* indicates if Real-Time Debug pins are enabled */ 6370bbc1413SRichard Zhao #define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */ 6381ec1e82fSSascha Hauer #define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/ 6391ec1e82fSSascha Hauer 6401ec1e82fSSascha Hauer static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event) 6411ec1e82fSSascha Hauer { 64217bba72fSSascha Hauer u32 chnenbl0 = sdma->drvdata->chnenbl0; 6431ec1e82fSSascha Hauer return chnenbl0 + event * 4; 6441ec1e82fSSascha Hauer } 6451ec1e82fSSascha Hauer 6461ec1e82fSSascha Hauer static int sdma_config_ownership(struct sdma_channel *sdmac, 6471ec1e82fSSascha Hauer bool event_override, bool mcu_override, bool dsp_override) 6481ec1e82fSSascha Hauer { 6491ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 6501ec1e82fSSascha Hauer int channel = sdmac->channel; 6510bbc1413SRichard Zhao unsigned long evt, mcu, dsp; 6521ec1e82fSSascha Hauer 6531ec1e82fSSascha Hauer if (event_override && mcu_override && dsp_override) 6541ec1e82fSSascha Hauer return -EINVAL; 6551ec1e82fSSascha Hauer 656c4b56857SRichard Zhao evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR); 657c4b56857SRichard Zhao mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR); 658c4b56857SRichard Zhao dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR); 6591ec1e82fSSascha Hauer 6601ec1e82fSSascha Hauer if (dsp_override) 6610bbc1413SRichard Zhao __clear_bit(channel, &dsp); 6621ec1e82fSSascha Hauer else 6630bbc1413SRichard Zhao __set_bit(channel, &dsp); 6641ec1e82fSSascha Hauer 6651ec1e82fSSascha Hauer if (event_override) 6660bbc1413SRichard Zhao __clear_bit(channel, &evt); 6671ec1e82fSSascha Hauer else 6680bbc1413SRichard Zhao __set_bit(channel, &evt); 6691ec1e82fSSascha Hauer 6701ec1e82fSSascha Hauer if (mcu_override) 6710bbc1413SRichard Zhao __clear_bit(channel, &mcu); 6721ec1e82fSSascha Hauer else 6730bbc1413SRichard Zhao __set_bit(channel, &mcu); 6741ec1e82fSSascha Hauer 675c4b56857SRichard Zhao writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR); 676c4b56857SRichard Zhao writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR); 677c4b56857SRichard Zhao writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR); 6781ec1e82fSSascha Hauer 6791ec1e82fSSascha Hauer return 0; 6801ec1e82fSSascha Hauer } 6811ec1e82fSSascha Hauer 682b9a59166SRichard Zhao static void sdma_enable_channel(struct sdma_engine *sdma, int channel) 683b9a59166SRichard Zhao { 6840bbc1413SRichard Zhao writel(BIT(channel), sdma->regs + SDMA_H_START); 685b9a59166SRichard Zhao } 686b9a59166SRichard Zhao 6871ec1e82fSSascha Hauer /* 6882ccaef05SRichard Zhao * sdma_run_channel0 - run a channel and wait till it's done 6891ec1e82fSSascha Hauer */ 6902ccaef05SRichard Zhao static int sdma_run_channel0(struct sdma_engine *sdma) 6911ec1e82fSSascha Hauer { 6921ec1e82fSSascha Hauer int ret; 6931d069bfaSMichael Olbrich u32 reg; 6941ec1e82fSSascha Hauer 6952ccaef05SRichard Zhao sdma_enable_channel(sdma, 0); 6961ec1e82fSSascha Hauer 6971d069bfaSMichael Olbrich ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP, 6981d069bfaSMichael Olbrich reg, !(reg & 1), 1, 500); 6991d069bfaSMichael Olbrich if (ret) 7002ccaef05SRichard Zhao dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); 7011ec1e82fSSascha Hauer 702855832e4SRobin Gong /* Set bits of CONFIG register with dynamic context switching */ 70325aaa75dSAngus Ainslie (Purism) reg = readl(sdma->regs + SDMA_H_CONFIG); 70425aaa75dSAngus Ainslie (Purism) if ((reg & SDMA_H_CONFIG_CSM) == 0) { 70525aaa75dSAngus Ainslie (Purism) reg |= SDMA_H_CONFIG_CSM; 70625aaa75dSAngus Ainslie (Purism) writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG); 70725aaa75dSAngus Ainslie (Purism) } 708855832e4SRobin Gong 7091d069bfaSMichael Olbrich return ret; 7101ec1e82fSSascha Hauer } 7111ec1e82fSSascha Hauer 7121ec1e82fSSascha Hauer static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, 7131ec1e82fSSascha Hauer u32 address) 7141ec1e82fSSascha Hauer { 71576c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0 = sdma->bd0; 7161ec1e82fSSascha Hauer void *buf_virt; 7171ec1e82fSSascha Hauer dma_addr_t buf_phys; 7181ec1e82fSSascha Hauer int ret; 7192ccaef05SRichard Zhao unsigned long flags; 72073eab978SSascha Hauer 721ceaf5226SAndy Duan buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); 72273eab978SSascha Hauer if (!buf_virt) { 7232ccaef05SRichard Zhao return -ENOMEM; 72473eab978SSascha Hauer } 7251ec1e82fSSascha Hauer 7262ccaef05SRichard Zhao spin_lock_irqsave(&sdma->channel_0_lock, flags); 7272ccaef05SRichard Zhao 7281ec1e82fSSascha Hauer bd0->mode.command = C0_SETPM; 7293f93a4f2SRobin Gong bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; 7301ec1e82fSSascha Hauer bd0->mode.count = size / 2; 7311ec1e82fSSascha Hauer bd0->buffer_addr = buf_phys; 7321ec1e82fSSascha Hauer bd0->ext_buffer_addr = address; 7331ec1e82fSSascha Hauer 7341ec1e82fSSascha Hauer memcpy(buf_virt, buf, size); 7351ec1e82fSSascha Hauer 7362ccaef05SRichard Zhao ret = sdma_run_channel0(sdma); 7372ccaef05SRichard Zhao 7382ccaef05SRichard Zhao spin_unlock_irqrestore(&sdma->channel_0_lock, flags); 7391ec1e82fSSascha Hauer 740ceaf5226SAndy Duan dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); 7411ec1e82fSSascha Hauer 7421ec1e82fSSascha Hauer return ret; 7431ec1e82fSSascha Hauer } 7441ec1e82fSSascha Hauer 7451ec1e82fSSascha Hauer static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) 7461ec1e82fSSascha Hauer { 7471ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 7481ec1e82fSSascha Hauer int channel = sdmac->channel; 7490bbc1413SRichard Zhao unsigned long val; 7501ec1e82fSSascha Hauer u32 chnenbl = chnenbl_ofs(sdma, event); 7511ec1e82fSSascha Hauer 752c4b56857SRichard Zhao val = readl_relaxed(sdma->regs + chnenbl); 7530bbc1413SRichard Zhao __set_bit(channel, &val); 754c4b56857SRichard Zhao writel_relaxed(val, sdma->regs + chnenbl); 7551ec1e82fSSascha Hauer } 7561ec1e82fSSascha Hauer 7571ec1e82fSSascha Hauer static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) 7581ec1e82fSSascha Hauer { 7591ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 7601ec1e82fSSascha Hauer int channel = sdmac->channel; 7611ec1e82fSSascha Hauer u32 chnenbl = chnenbl_ofs(sdma, event); 7620bbc1413SRichard Zhao unsigned long val; 7631ec1e82fSSascha Hauer 764c4b56857SRichard Zhao val = readl_relaxed(sdma->regs + chnenbl); 7650bbc1413SRichard Zhao __clear_bit(channel, &val); 766c4b56857SRichard Zhao writel_relaxed(val, sdma->regs + chnenbl); 7671ec1e82fSSascha Hauer } 7681ec1e82fSSascha Hauer 76957b772b8SRobin Gong static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t) 77057b772b8SRobin Gong { 77157b772b8SRobin Gong return container_of(t, struct sdma_desc, vd.tx); 77257b772b8SRobin Gong } 77357b772b8SRobin Gong 77457b772b8SRobin Gong static void sdma_start_desc(struct sdma_channel *sdmac) 77557b772b8SRobin Gong { 77657b772b8SRobin Gong struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc); 77757b772b8SRobin Gong struct sdma_desc *desc; 77857b772b8SRobin Gong struct sdma_engine *sdma = sdmac->sdma; 77957b772b8SRobin Gong int channel = sdmac->channel; 78057b772b8SRobin Gong 78157b772b8SRobin Gong if (!vd) { 78257b772b8SRobin Gong sdmac->desc = NULL; 78357b772b8SRobin Gong return; 78457b772b8SRobin Gong } 78557b772b8SRobin Gong sdmac->desc = desc = to_sdma_desc(&vd->tx); 78602939cd1SSascha Hauer 78757b772b8SRobin Gong list_del(&vd->node); 78857b772b8SRobin Gong 78957b772b8SRobin Gong sdma->channel_control[channel].base_bd_ptr = desc->bd_phys; 79057b772b8SRobin Gong sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; 79157b772b8SRobin Gong sdma_enable_channel(sdma, sdmac->channel); 79257b772b8SRobin Gong } 79357b772b8SRobin Gong 794d1a792f3SRussell King - ARM Linux static void sdma_update_channel_loop(struct sdma_channel *sdmac) 795d1a792f3SRussell King - ARM Linux { 7961ec1e82fSSascha Hauer struct sdma_buffer_descriptor *bd; 7975881826dSNandor Han int error = 0; 7985881826dSNandor Han enum dma_status old_status = sdmac->status; 7991ec1e82fSSascha Hauer 8001ec1e82fSSascha Hauer /* 8011ec1e82fSSascha Hauer * loop mode. Iterate over descriptors, re-setup them and 8021ec1e82fSSascha Hauer * call callback function. 8031ec1e82fSSascha Hauer */ 80457b772b8SRobin Gong while (sdmac->desc) { 80576c33d27SSascha Hauer struct sdma_desc *desc = sdmac->desc; 80676c33d27SSascha Hauer 80776c33d27SSascha Hauer bd = &desc->bd[desc->buf_tail]; 8081ec1e82fSSascha Hauer 8091ec1e82fSSascha Hauer if (bd->mode.status & BD_DONE) 8101ec1e82fSSascha Hauer break; 8111ec1e82fSSascha Hauer 8125881826dSNandor Han if (bd->mode.status & BD_RROR) { 8135881826dSNandor Han bd->mode.status &= ~BD_RROR; 8141ec1e82fSSascha Hauer sdmac->status = DMA_ERROR; 8155881826dSNandor Han error = -EIO; 8165881826dSNandor Han } 8171ec1e82fSSascha Hauer 8185881826dSNandor Han /* 8195881826dSNandor Han * We use bd->mode.count to calculate the residue, since contains 8205881826dSNandor Han * the number of bytes present in the current buffer descriptor. 8215881826dSNandor Han */ 8225881826dSNandor Han 82376c33d27SSascha Hauer desc->chn_real_count = bd->mode.count; 8241ec1e82fSSascha Hauer bd->mode.status |= BD_DONE; 82576c33d27SSascha Hauer bd->mode.count = desc->period_len; 82676c33d27SSascha Hauer desc->buf_ptail = desc->buf_tail; 82776c33d27SSascha Hauer desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; 82815f30f51SNandor Han 82915f30f51SNandor Han /* 83015f30f51SNandor Han * The callback is called from the interrupt context in order 83115f30f51SNandor Han * to reduce latency and to avoid the risk of altering the 83215f30f51SNandor Han * SDMA transaction status by the time the client tasklet is 83315f30f51SNandor Han * executed. 83415f30f51SNandor Han */ 83557b772b8SRobin Gong spin_unlock(&sdmac->vc.lock); 83657b772b8SRobin Gong dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL); 83757b772b8SRobin Gong spin_lock(&sdmac->vc.lock); 83815f30f51SNandor Han 8395881826dSNandor Han if (error) 8405881826dSNandor Han sdmac->status = old_status; 8411ec1e82fSSascha Hauer } 8421ec1e82fSSascha Hauer } 8431ec1e82fSSascha Hauer 84457b772b8SRobin Gong static void mxc_sdma_handle_channel_normal(struct sdma_channel *data) 8451ec1e82fSSascha Hauer { 84615f30f51SNandor Han struct sdma_channel *sdmac = (struct sdma_channel *) data; 8471ec1e82fSSascha Hauer struct sdma_buffer_descriptor *bd; 8481ec1e82fSSascha Hauer int i, error = 0; 8491ec1e82fSSascha Hauer 85076c33d27SSascha Hauer sdmac->desc->chn_real_count = 0; 8511ec1e82fSSascha Hauer /* 8521ec1e82fSSascha Hauer * non loop mode. Iterate over all descriptors, collect 8531ec1e82fSSascha Hauer * errors and call callback function 8541ec1e82fSSascha Hauer */ 85576c33d27SSascha Hauer for (i = 0; i < sdmac->desc->num_bd; i++) { 85676c33d27SSascha Hauer bd = &sdmac->desc->bd[i]; 8571ec1e82fSSascha Hauer 8581ec1e82fSSascha Hauer if (bd->mode.status & (BD_DONE | BD_RROR)) 8591ec1e82fSSascha Hauer error = -EIO; 86076c33d27SSascha Hauer sdmac->desc->chn_real_count += bd->mode.count; 8611ec1e82fSSascha Hauer } 8621ec1e82fSSascha Hauer 8631ec1e82fSSascha Hauer if (error) 8641ec1e82fSSascha Hauer sdmac->status = DMA_ERROR; 8651ec1e82fSSascha Hauer else 866409bff6aSVinod Koul sdmac->status = DMA_COMPLETE; 8671ec1e82fSSascha Hauer } 8681ec1e82fSSascha Hauer 8691ec1e82fSSascha Hauer static irqreturn_t sdma_int_handler(int irq, void *dev_id) 8701ec1e82fSSascha Hauer { 8711ec1e82fSSascha Hauer struct sdma_engine *sdma = dev_id; 8720bbc1413SRichard Zhao unsigned long stat; 8731ec1e82fSSascha Hauer 874c4b56857SRichard Zhao stat = readl_relaxed(sdma->regs + SDMA_H_INTR); 875c4b56857SRichard Zhao writel_relaxed(stat, sdma->regs + SDMA_H_INTR); 8761d069bfaSMichael Olbrich /* channel 0 is special and not handled here, see run_channel0() */ 8771d069bfaSMichael Olbrich stat &= ~1; 8781ec1e82fSSascha Hauer 8791ec1e82fSSascha Hauer while (stat) { 8801ec1e82fSSascha Hauer int channel = fls(stat) - 1; 8811ec1e82fSSascha Hauer struct sdma_channel *sdmac = &sdma->channel[channel]; 88257b772b8SRobin Gong struct sdma_desc *desc; 8831ec1e82fSSascha Hauer 88457b772b8SRobin Gong spin_lock(&sdmac->vc.lock); 88557b772b8SRobin Gong desc = sdmac->desc; 88657b772b8SRobin Gong if (desc) { 88757b772b8SRobin Gong if (sdmac->flags & IMX_DMA_SG_LOOP) { 888d1a792f3SRussell King - ARM Linux sdma_update_channel_loop(sdmac); 88957b772b8SRobin Gong } else { 89057b772b8SRobin Gong mxc_sdma_handle_channel_normal(sdmac); 89157b772b8SRobin Gong vchan_cookie_complete(&desc->vd); 89257b772b8SRobin Gong sdma_start_desc(sdmac); 89357b772b8SRobin Gong } 89457b772b8SRobin Gong } 8951ec1e82fSSascha Hauer 89657b772b8SRobin Gong spin_unlock(&sdmac->vc.lock); 8970bbc1413SRichard Zhao __clear_bit(channel, &stat); 8981ec1e82fSSascha Hauer } 8991ec1e82fSSascha Hauer 9001ec1e82fSSascha Hauer return IRQ_HANDLED; 9011ec1e82fSSascha Hauer } 9021ec1e82fSSascha Hauer 9031ec1e82fSSascha Hauer /* 9041ec1e82fSSascha Hauer * sets the pc of SDMA script according to the peripheral type 9051ec1e82fSSascha Hauer */ 9061ec1e82fSSascha Hauer static void sdma_get_pc(struct sdma_channel *sdmac, 9071ec1e82fSSascha Hauer enum sdma_peripheral_type peripheral_type) 9081ec1e82fSSascha Hauer { 9091ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 9101ec1e82fSSascha Hauer int per_2_emi = 0, emi_2_per = 0; 9111ec1e82fSSascha Hauer /* 9121ec1e82fSSascha Hauer * These are needed once we start to support transfers between 9131ec1e82fSSascha Hauer * two peripherals or memory-to-memory transfers 9141ec1e82fSSascha Hauer */ 9150f06c027SRobin Gong int per_2_per = 0, emi_2_emi = 0; 9161ec1e82fSSascha Hauer 9171ec1e82fSSascha Hauer sdmac->pc_from_device = 0; 9181ec1e82fSSascha Hauer sdmac->pc_to_device = 0; 9198391ecf4SShengjiu Wang sdmac->device_to_device = 0; 9200f06c027SRobin Gong sdmac->pc_to_pc = 0; 9211ec1e82fSSascha Hauer 9221ec1e82fSSascha Hauer switch (peripheral_type) { 9231ec1e82fSSascha Hauer case IMX_DMATYPE_MEMORY: 9240f06c027SRobin Gong emi_2_emi = sdma->script_addrs->ap_2_ap_addr; 9251ec1e82fSSascha Hauer break; 9261ec1e82fSSascha Hauer case IMX_DMATYPE_DSP: 9271ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->bp_2_ap_addr; 9281ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->ap_2_bp_addr; 9291ec1e82fSSascha Hauer break; 9301ec1e82fSSascha Hauer case IMX_DMATYPE_FIRI: 9311ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->firi_2_mcu_addr; 9321ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_firi_addr; 9331ec1e82fSSascha Hauer break; 9341ec1e82fSSascha Hauer case IMX_DMATYPE_UART: 9351ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->uart_2_mcu_addr; 9361ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_app_addr; 9371ec1e82fSSascha Hauer break; 9381ec1e82fSSascha Hauer case IMX_DMATYPE_UART_SP: 9391ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr; 9401ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_shp_addr; 9411ec1e82fSSascha Hauer break; 9421ec1e82fSSascha Hauer case IMX_DMATYPE_ATA: 9431ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->ata_2_mcu_addr; 9441ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_ata_addr; 9451ec1e82fSSascha Hauer break; 9461ec1e82fSSascha Hauer case IMX_DMATYPE_CSPI: 9471ec1e82fSSascha Hauer case IMX_DMATYPE_EXT: 9481ec1e82fSSascha Hauer case IMX_DMATYPE_SSI: 94929aebfdeSNicolin Chen case IMX_DMATYPE_SAI: 9501ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->app_2_mcu_addr; 9511ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_app_addr; 9521ec1e82fSSascha Hauer break; 9531a895578SNicolin Chen case IMX_DMATYPE_SSI_DUAL: 9541a895578SNicolin Chen per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; 9551a895578SNicolin Chen emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; 9561a895578SNicolin Chen break; 9571ec1e82fSSascha Hauer case IMX_DMATYPE_SSI_SP: 9581ec1e82fSSascha Hauer case IMX_DMATYPE_MMC: 9591ec1e82fSSascha Hauer case IMX_DMATYPE_SDHC: 9601ec1e82fSSascha Hauer case IMX_DMATYPE_CSPI_SP: 9611ec1e82fSSascha Hauer case IMX_DMATYPE_ESAI: 9621ec1e82fSSascha Hauer case IMX_DMATYPE_MSHC_SP: 9631ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->shp_2_mcu_addr; 9641ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_shp_addr; 9651ec1e82fSSascha Hauer break; 9661ec1e82fSSascha Hauer case IMX_DMATYPE_ASRC: 9671ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->asrc_2_mcu_addr; 9681ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; 9691ec1e82fSSascha Hauer per_2_per = sdma->script_addrs->per_2_per_addr; 9701ec1e82fSSascha Hauer break; 971f892afb0SNicolin Chen case IMX_DMATYPE_ASRC_SP: 972f892afb0SNicolin Chen per_2_emi = sdma->script_addrs->shp_2_mcu_addr; 973f892afb0SNicolin Chen emi_2_per = sdma->script_addrs->mcu_2_shp_addr; 974f892afb0SNicolin Chen per_2_per = sdma->script_addrs->per_2_per_addr; 975f892afb0SNicolin Chen break; 9761ec1e82fSSascha Hauer case IMX_DMATYPE_MSHC: 9771ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->mshc_2_mcu_addr; 9781ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_mshc_addr; 9791ec1e82fSSascha Hauer break; 9801ec1e82fSSascha Hauer case IMX_DMATYPE_CCM: 9811ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->dptc_dvfs_addr; 9821ec1e82fSSascha Hauer break; 9831ec1e82fSSascha Hauer case IMX_DMATYPE_SPDIF: 9841ec1e82fSSascha Hauer per_2_emi = sdma->script_addrs->spdif_2_mcu_addr; 9851ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->mcu_2_spdif_addr; 9861ec1e82fSSascha Hauer break; 9871ec1e82fSSascha Hauer case IMX_DMATYPE_IPU_MEMORY: 9881ec1e82fSSascha Hauer emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; 9891ec1e82fSSascha Hauer break; 9901ec1e82fSSascha Hauer default: 9911ec1e82fSSascha Hauer break; 9921ec1e82fSSascha Hauer } 9931ec1e82fSSascha Hauer 9941ec1e82fSSascha Hauer sdmac->pc_from_device = per_2_emi; 9951ec1e82fSSascha Hauer sdmac->pc_to_device = emi_2_per; 9968391ecf4SShengjiu Wang sdmac->device_to_device = per_2_per; 9970f06c027SRobin Gong sdmac->pc_to_pc = emi_2_emi; 9981ec1e82fSSascha Hauer } 9991ec1e82fSSascha Hauer 10001ec1e82fSSascha Hauer static int sdma_load_context(struct sdma_channel *sdmac) 10011ec1e82fSSascha Hauer { 10021ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 10031ec1e82fSSascha Hauer int channel = sdmac->channel; 10041ec1e82fSSascha Hauer int load_address; 10051ec1e82fSSascha Hauer struct sdma_context_data *context = sdma->context; 100676c33d27SSascha Hauer struct sdma_buffer_descriptor *bd0 = sdma->bd0; 10071ec1e82fSSascha Hauer int ret; 10082ccaef05SRichard Zhao unsigned long flags; 10091ec1e82fSSascha Hauer 10108391ecf4SShengjiu Wang if (sdmac->direction == DMA_DEV_TO_MEM) 10111ec1e82fSSascha Hauer load_address = sdmac->pc_from_device; 10128391ecf4SShengjiu Wang else if (sdmac->direction == DMA_DEV_TO_DEV) 10138391ecf4SShengjiu Wang load_address = sdmac->device_to_device; 10140f06c027SRobin Gong else if (sdmac->direction == DMA_MEM_TO_MEM) 10150f06c027SRobin Gong load_address = sdmac->pc_to_pc; 10168391ecf4SShengjiu Wang else 10171ec1e82fSSascha Hauer load_address = sdmac->pc_to_device; 10181ec1e82fSSascha Hauer 10191ec1e82fSSascha Hauer if (load_address < 0) 10201ec1e82fSSascha Hauer return load_address; 10211ec1e82fSSascha Hauer 10221ec1e82fSSascha Hauer dev_dbg(sdma->dev, "load_address = %d\n", load_address); 10230bbc1413SRichard Zhao dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level); 10241ec1e82fSSascha Hauer dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr); 10251ec1e82fSSascha Hauer dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr); 10260bbc1413SRichard Zhao dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]); 10270bbc1413SRichard Zhao dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]); 10281ec1e82fSSascha Hauer 10292ccaef05SRichard Zhao spin_lock_irqsave(&sdma->channel_0_lock, flags); 103073eab978SSascha Hauer 10311ec1e82fSSascha Hauer memset(context, 0, sizeof(*context)); 10321ec1e82fSSascha Hauer context->channel_state.pc = load_address; 10331ec1e82fSSascha Hauer 10341ec1e82fSSascha Hauer /* Send by context the event mask,base address for peripheral 10351ec1e82fSSascha Hauer * and watermark level 10361ec1e82fSSascha Hauer */ 10370bbc1413SRichard Zhao context->gReg[0] = sdmac->event_mask[1]; 10380bbc1413SRichard Zhao context->gReg[1] = sdmac->event_mask[0]; 10391ec1e82fSSascha Hauer context->gReg[2] = sdmac->per_addr; 10401ec1e82fSSascha Hauer context->gReg[6] = sdmac->shp_addr; 10411ec1e82fSSascha Hauer context->gReg[7] = sdmac->watermark_level; 10421ec1e82fSSascha Hauer 10431ec1e82fSSascha Hauer bd0->mode.command = C0_SETDM; 10443f93a4f2SRobin Gong bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; 10451ec1e82fSSascha Hauer bd0->mode.count = sizeof(*context) / 4; 10461ec1e82fSSascha Hauer bd0->buffer_addr = sdma->context_phys; 10471ec1e82fSSascha Hauer bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel; 10482ccaef05SRichard Zhao ret = sdma_run_channel0(sdma); 10491ec1e82fSSascha Hauer 10502ccaef05SRichard Zhao spin_unlock_irqrestore(&sdma->channel_0_lock, flags); 105173eab978SSascha Hauer 10521ec1e82fSSascha Hauer return ret; 10531ec1e82fSSascha Hauer } 10541ec1e82fSSascha Hauer 10557b350ab0SMaxime Ripard static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) 10561ec1e82fSSascha Hauer { 105757b772b8SRobin Gong return container_of(chan, struct sdma_channel, vc.chan); 10587b350ab0SMaxime Ripard } 10597b350ab0SMaxime Ripard 10607b350ab0SMaxime Ripard static int sdma_disable_channel(struct dma_chan *chan) 10617b350ab0SMaxime Ripard { 10627b350ab0SMaxime Ripard struct sdma_channel *sdmac = to_sdma_chan(chan); 10631ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 10641ec1e82fSSascha Hauer int channel = sdmac->channel; 10651ec1e82fSSascha Hauer 10660bbc1413SRichard Zhao writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP); 10671ec1e82fSSascha Hauer sdmac->status = DMA_ERROR; 10687b350ab0SMaxime Ripard 10697b350ab0SMaxime Ripard return 0; 10701ec1e82fSSascha Hauer } 1071b8603d2aSLucas Stach static void sdma_channel_terminate_work(struct work_struct *work) 10727f3ff14bSJiada Wang { 1073b8603d2aSLucas Stach struct sdma_channel *sdmac = container_of(work, struct sdma_channel, 1074b8603d2aSLucas Stach terminate_worker); 107557b772b8SRobin Gong unsigned long flags; 107657b772b8SRobin Gong LIST_HEAD(head); 107757b772b8SRobin Gong 10787f3ff14bSJiada Wang /* 10797f3ff14bSJiada Wang * According to NXP R&D team a delay of one BD SDMA cost time 10807f3ff14bSJiada Wang * (maximum is 1ms) should be added after disable of the channel 10817f3ff14bSJiada Wang * bit, to ensure SDMA core has really been stopped after SDMA 10827f3ff14bSJiada Wang * clients call .device_terminate_all. 10837f3ff14bSJiada Wang */ 1084b8603d2aSLucas Stach usleep_range(1000, 2000); 1085b8603d2aSLucas Stach 1086b8603d2aSLucas Stach spin_lock_irqsave(&sdmac->vc.lock, flags); 1087b8603d2aSLucas Stach vchan_get_all_descriptors(&sdmac->vc, &head); 1088b8603d2aSLucas Stach spin_unlock_irqrestore(&sdmac->vc.lock, flags); 1089b8603d2aSLucas Stach vchan_dma_desc_free_list(&sdmac->vc, &head); 1090b8603d2aSLucas Stach } 1091b8603d2aSLucas Stach 1092a80f2787SSascha Hauer static int sdma_terminate_all(struct dma_chan *chan) 1093b8603d2aSLucas Stach { 1094b8603d2aSLucas Stach struct sdma_channel *sdmac = to_sdma_chan(chan); 109502939cd1SSascha Hauer unsigned long flags; 109602939cd1SSascha Hauer 109702939cd1SSascha Hauer spin_lock_irqsave(&sdmac->vc.lock, flags); 1098b8603d2aSLucas Stach 1099b8603d2aSLucas Stach sdma_disable_channel(chan); 1100b8603d2aSLucas Stach 110102939cd1SSascha Hauer if (sdmac->desc) { 110202939cd1SSascha Hauer vchan_terminate_vdesc(&sdmac->desc->vd); 110302939cd1SSascha Hauer sdmac->desc = NULL; 1104b8603d2aSLucas Stach schedule_work(&sdmac->terminate_worker); 110502939cd1SSascha Hauer } 110602939cd1SSascha Hauer 110702939cd1SSascha Hauer spin_unlock_irqrestore(&sdmac->vc.lock, flags); 11087f3ff14bSJiada Wang 11097f3ff14bSJiada Wang return 0; 11107f3ff14bSJiada Wang } 11117f3ff14bSJiada Wang 1112b8603d2aSLucas Stach static void sdma_channel_synchronize(struct dma_chan *chan) 1113b8603d2aSLucas Stach { 1114b8603d2aSLucas Stach struct sdma_channel *sdmac = to_sdma_chan(chan); 1115b8603d2aSLucas Stach 1116b8603d2aSLucas Stach vchan_synchronize(&sdmac->vc); 1117b8603d2aSLucas Stach 1118b8603d2aSLucas Stach flush_work(&sdmac->terminate_worker); 1119b8603d2aSLucas Stach } 1120b8603d2aSLucas Stach 11218391ecf4SShengjiu Wang static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) 11228391ecf4SShengjiu Wang { 11238391ecf4SShengjiu Wang struct sdma_engine *sdma = sdmac->sdma; 11248391ecf4SShengjiu Wang 11258391ecf4SShengjiu Wang int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML; 11268391ecf4SShengjiu Wang int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16; 11278391ecf4SShengjiu Wang 11288391ecf4SShengjiu Wang set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]); 11298391ecf4SShengjiu Wang set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]); 11308391ecf4SShengjiu Wang 11318391ecf4SShengjiu Wang if (sdmac->event_id0 > 31) 11328391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE; 11338391ecf4SShengjiu Wang 11348391ecf4SShengjiu Wang if (sdmac->event_id1 > 31) 11358391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE; 11368391ecf4SShengjiu Wang 11378391ecf4SShengjiu Wang /* 11388391ecf4SShengjiu Wang * If LWML(src_maxburst) > HWML(dst_maxburst), we need 11398391ecf4SShengjiu Wang * swap LWML and HWML of INFO(A.3.2.5.1), also need swap 11408391ecf4SShengjiu Wang * r0(event_mask[1]) and r1(event_mask[0]). 11418391ecf4SShengjiu Wang */ 11428391ecf4SShengjiu Wang if (lwml > hwml) { 11438391ecf4SShengjiu Wang sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML | 11448391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_HWML); 11458391ecf4SShengjiu Wang sdmac->watermark_level |= hwml; 11468391ecf4SShengjiu Wang sdmac->watermark_level |= lwml << 16; 11478391ecf4SShengjiu Wang swap(sdmac->event_mask[0], sdmac->event_mask[1]); 11488391ecf4SShengjiu Wang } 11498391ecf4SShengjiu Wang 11508391ecf4SShengjiu Wang if (sdmac->per_address2 >= sdma->spba_start_addr && 11518391ecf4SShengjiu Wang sdmac->per_address2 <= sdma->spba_end_addr) 11528391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP; 11538391ecf4SShengjiu Wang 11548391ecf4SShengjiu Wang if (sdmac->per_address >= sdma->spba_start_addr && 11558391ecf4SShengjiu Wang sdmac->per_address <= sdma->spba_end_addr) 11568391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; 11578391ecf4SShengjiu Wang 11588391ecf4SShengjiu Wang sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; 11598391ecf4SShengjiu Wang } 11608391ecf4SShengjiu Wang 11617b350ab0SMaxime Ripard static int sdma_config_channel(struct dma_chan *chan) 11621ec1e82fSSascha Hauer { 11637b350ab0SMaxime Ripard struct sdma_channel *sdmac = to_sdma_chan(chan); 11641ec1e82fSSascha Hauer 11657b350ab0SMaxime Ripard sdma_disable_channel(chan); 11661ec1e82fSSascha Hauer 11670bbc1413SRichard Zhao sdmac->event_mask[0] = 0; 11680bbc1413SRichard Zhao sdmac->event_mask[1] = 0; 11691ec1e82fSSascha Hauer sdmac->shp_addr = 0; 11701ec1e82fSSascha Hauer sdmac->per_addr = 0; 11711ec1e82fSSascha Hauer 11721ec1e82fSSascha Hauer switch (sdmac->peripheral_type) { 11731ec1e82fSSascha Hauer case IMX_DMATYPE_DSP: 11741ec1e82fSSascha Hauer sdma_config_ownership(sdmac, false, true, true); 11751ec1e82fSSascha Hauer break; 11761ec1e82fSSascha Hauer case IMX_DMATYPE_MEMORY: 11771ec1e82fSSascha Hauer sdma_config_ownership(sdmac, false, true, false); 11781ec1e82fSSascha Hauer break; 11791ec1e82fSSascha Hauer default: 11801ec1e82fSSascha Hauer sdma_config_ownership(sdmac, true, true, false); 11811ec1e82fSSascha Hauer break; 11821ec1e82fSSascha Hauer } 11831ec1e82fSSascha Hauer 11841ec1e82fSSascha Hauer sdma_get_pc(sdmac, sdmac->peripheral_type); 11851ec1e82fSSascha Hauer 11861ec1e82fSSascha Hauer if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) && 11871ec1e82fSSascha Hauer (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { 11881ec1e82fSSascha Hauer /* Handle multiple event channels differently */ 11891ec1e82fSSascha Hauer if (sdmac->event_id1) { 11908391ecf4SShengjiu Wang if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || 11918391ecf4SShengjiu Wang sdmac->peripheral_type == IMX_DMATYPE_ASRC) 11928391ecf4SShengjiu Wang sdma_set_watermarklevel_for_p2p(sdmac); 11938391ecf4SShengjiu Wang } else 11940bbc1413SRichard Zhao __set_bit(sdmac->event_id0, sdmac->event_mask); 11958391ecf4SShengjiu Wang 11961ec1e82fSSascha Hauer /* Address */ 11971ec1e82fSSascha Hauer sdmac->shp_addr = sdmac->per_address; 11988391ecf4SShengjiu Wang sdmac->per_addr = sdmac->per_address2; 11991ec1e82fSSascha Hauer } else { 12001ec1e82fSSascha Hauer sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ 12011ec1e82fSSascha Hauer } 12021ec1e82fSSascha Hauer 1203*e555a03bSRobin Gong return 0; 12041ec1e82fSSascha Hauer } 12051ec1e82fSSascha Hauer 12061ec1e82fSSascha Hauer static int sdma_set_channel_priority(struct sdma_channel *sdmac, 12071ec1e82fSSascha Hauer unsigned int priority) 12081ec1e82fSSascha Hauer { 12091ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 12101ec1e82fSSascha Hauer int channel = sdmac->channel; 12111ec1e82fSSascha Hauer 12121ec1e82fSSascha Hauer if (priority < MXC_SDMA_MIN_PRIORITY 12131ec1e82fSSascha Hauer || priority > MXC_SDMA_MAX_PRIORITY) { 12141ec1e82fSSascha Hauer return -EINVAL; 12151ec1e82fSSascha Hauer } 12161ec1e82fSSascha Hauer 1217c4b56857SRichard Zhao writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel); 12181ec1e82fSSascha Hauer 12191ec1e82fSSascha Hauer return 0; 12201ec1e82fSSascha Hauer } 12211ec1e82fSSascha Hauer 122257b772b8SRobin Gong static int sdma_request_channel0(struct sdma_engine *sdma) 12231ec1e82fSSascha Hauer { 12241ec1e82fSSascha Hauer int ret = -EBUSY; 12251ec1e82fSSascha Hauer 122631ef489aSLinus Torvalds sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, 122757b772b8SRobin Gong GFP_NOWAIT); 122857b772b8SRobin Gong if (!sdma->bd0) { 12291ec1e82fSSascha Hauer ret = -ENOMEM; 12301ec1e82fSSascha Hauer goto out; 12311ec1e82fSSascha Hauer } 12321ec1e82fSSascha Hauer 123357b772b8SRobin Gong sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys; 123457b772b8SRobin Gong sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys; 12351ec1e82fSSascha Hauer 123657b772b8SRobin Gong sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY); 12371ec1e82fSSascha Hauer return 0; 12381ec1e82fSSascha Hauer out: 12391ec1e82fSSascha Hauer 12401ec1e82fSSascha Hauer return ret; 12411ec1e82fSSascha Hauer } 12421ec1e82fSSascha Hauer 124357b772b8SRobin Gong 124457b772b8SRobin Gong static int sdma_alloc_bd(struct sdma_desc *desc) 12451ec1e82fSSascha Hauer { 1246ebb853b1SLucas Stach u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); 124757b772b8SRobin Gong int ret = 0; 12481ec1e82fSSascha Hauer 124931ef489aSLinus Torvalds desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, 1250ceaf5226SAndy Duan &desc->bd_phys, GFP_NOWAIT); 125157b772b8SRobin Gong if (!desc->bd) { 125257b772b8SRobin Gong ret = -ENOMEM; 125357b772b8SRobin Gong goto out; 125457b772b8SRobin Gong } 125557b772b8SRobin Gong out: 125657b772b8SRobin Gong return ret; 125757b772b8SRobin Gong } 12581ec1e82fSSascha Hauer 125957b772b8SRobin Gong static void sdma_free_bd(struct sdma_desc *desc) 126057b772b8SRobin Gong { 1261ebb853b1SLucas Stach u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); 1262ebb853b1SLucas Stach 1263ceaf5226SAndy Duan dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, 1264ceaf5226SAndy Duan desc->bd_phys); 126557b772b8SRobin Gong } 12661ec1e82fSSascha Hauer 126757b772b8SRobin Gong static void sdma_desc_free(struct virt_dma_desc *vd) 126857b772b8SRobin Gong { 126957b772b8SRobin Gong struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd); 127057b772b8SRobin Gong 127157b772b8SRobin Gong sdma_free_bd(desc); 127257b772b8SRobin Gong kfree(desc); 12731ec1e82fSSascha Hauer } 12741ec1e82fSSascha Hauer 12751ec1e82fSSascha Hauer static int sdma_alloc_chan_resources(struct dma_chan *chan) 12761ec1e82fSSascha Hauer { 12771ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 12781ec1e82fSSascha Hauer struct imx_dma_data *data = chan->private; 12790f06c027SRobin Gong struct imx_dma_data mem_data; 12801ec1e82fSSascha Hauer int prio, ret; 12811ec1e82fSSascha Hauer 12820f06c027SRobin Gong /* 12830f06c027SRobin Gong * MEMCPY may never setup chan->private by filter function such as 12840f06c027SRobin Gong * dmatest, thus create 'struct imx_dma_data mem_data' for this case. 12850f06c027SRobin Gong * Please note in any other slave case, you have to setup chan->private 12860f06c027SRobin Gong * with 'struct imx_dma_data' in your own filter function if you want to 12870f06c027SRobin Gong * request dma channel by dma_request_channel() rather than 12880f06c027SRobin Gong * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear 12890f06c027SRobin Gong * to warn you to correct your filter function. 12900f06c027SRobin Gong */ 12910f06c027SRobin Gong if (!data) { 12920f06c027SRobin Gong dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n"); 12930f06c027SRobin Gong mem_data.priority = 2; 12940f06c027SRobin Gong mem_data.peripheral_type = IMX_DMATYPE_MEMORY; 12950f06c027SRobin Gong mem_data.dma_request = 0; 12960f06c027SRobin Gong mem_data.dma_request2 = 0; 12970f06c027SRobin Gong data = &mem_data; 12980f06c027SRobin Gong 12990f06c027SRobin Gong sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY); 13000f06c027SRobin Gong } 13011ec1e82fSSascha Hauer 13021ec1e82fSSascha Hauer switch (data->priority) { 13031ec1e82fSSascha Hauer case DMA_PRIO_HIGH: 13041ec1e82fSSascha Hauer prio = 3; 13051ec1e82fSSascha Hauer break; 13061ec1e82fSSascha Hauer case DMA_PRIO_MEDIUM: 13071ec1e82fSSascha Hauer prio = 2; 13081ec1e82fSSascha Hauer break; 13091ec1e82fSSascha Hauer case DMA_PRIO_LOW: 13101ec1e82fSSascha Hauer default: 13111ec1e82fSSascha Hauer prio = 1; 13121ec1e82fSSascha Hauer break; 13131ec1e82fSSascha Hauer } 13141ec1e82fSSascha Hauer 13151ec1e82fSSascha Hauer sdmac->peripheral_type = data->peripheral_type; 13161ec1e82fSSascha Hauer sdmac->event_id0 = data->dma_request; 13178391ecf4SShengjiu Wang sdmac->event_id1 = data->dma_request2; 1318c2c744d3SRichard Zhao 1319b93edcddSFabio Estevam ret = clk_enable(sdmac->sdma->clk_ipg); 1320b93edcddSFabio Estevam if (ret) 1321b93edcddSFabio Estevam return ret; 1322b93edcddSFabio Estevam ret = clk_enable(sdmac->sdma->clk_ahb); 1323b93edcddSFabio Estevam if (ret) 1324b93edcddSFabio Estevam goto disable_clk_ipg; 1325c2c744d3SRichard Zhao 13263bb5e7caSRichard Zhao ret = sdma_set_channel_priority(sdmac, prio); 13271ec1e82fSSascha Hauer if (ret) 1328b93edcddSFabio Estevam goto disable_clk_ahb; 13291ec1e82fSSascha Hauer 13301ec1e82fSSascha Hauer return 0; 1331b93edcddSFabio Estevam 1332b93edcddSFabio Estevam disable_clk_ahb: 1333b93edcddSFabio Estevam clk_disable(sdmac->sdma->clk_ahb); 1334b93edcddSFabio Estevam disable_clk_ipg: 1335b93edcddSFabio Estevam clk_disable(sdmac->sdma->clk_ipg); 1336b93edcddSFabio Estevam return ret; 13371ec1e82fSSascha Hauer } 13381ec1e82fSSascha Hauer 13391ec1e82fSSascha Hauer static void sdma_free_chan_resources(struct dma_chan *chan) 13401ec1e82fSSascha Hauer { 13411ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 13421ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 13431ec1e82fSSascha Hauer 1344a80f2787SSascha Hauer sdma_terminate_all(chan); 1345b8603d2aSLucas Stach 1346b8603d2aSLucas Stach sdma_channel_synchronize(chan); 13471ec1e82fSSascha Hauer 13481ec1e82fSSascha Hauer sdma_event_disable(sdmac, sdmac->event_id0); 13491ec1e82fSSascha Hauer if (sdmac->event_id1) 13501ec1e82fSSascha Hauer sdma_event_disable(sdmac, sdmac->event_id1); 13511ec1e82fSSascha Hauer 13521ec1e82fSSascha Hauer sdmac->event_id0 = 0; 13531ec1e82fSSascha Hauer sdmac->event_id1 = 0; 13541ec1e82fSSascha Hauer 13551ec1e82fSSascha Hauer sdma_set_channel_priority(sdmac, 0); 13561ec1e82fSSascha Hauer 13577560e3f3SSascha Hauer clk_disable(sdma->clk_ipg); 13587560e3f3SSascha Hauer clk_disable(sdma->clk_ahb); 13591ec1e82fSSascha Hauer } 13601ec1e82fSSascha Hauer 136121420841SRobin Gong static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, 136221420841SRobin Gong enum dma_transfer_direction direction, u32 bds) 136321420841SRobin Gong { 136421420841SRobin Gong struct sdma_desc *desc; 136521420841SRobin Gong 136621420841SRobin Gong desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); 136721420841SRobin Gong if (!desc) 136821420841SRobin Gong goto err_out; 136921420841SRobin Gong 137021420841SRobin Gong sdmac->status = DMA_IN_PROGRESS; 137121420841SRobin Gong sdmac->direction = direction; 137221420841SRobin Gong sdmac->flags = 0; 137321420841SRobin Gong 137421420841SRobin Gong desc->chn_count = 0; 137521420841SRobin Gong desc->chn_real_count = 0; 137621420841SRobin Gong desc->buf_tail = 0; 137721420841SRobin Gong desc->buf_ptail = 0; 137821420841SRobin Gong desc->sdmac = sdmac; 137921420841SRobin Gong desc->num_bd = bds; 138021420841SRobin Gong 138121420841SRobin Gong if (sdma_alloc_bd(desc)) 138221420841SRobin Gong goto err_desc_out; 138321420841SRobin Gong 13840f06c027SRobin Gong /* No slave_config called in MEMCPY case, so do here */ 13850f06c027SRobin Gong if (direction == DMA_MEM_TO_MEM) 13860f06c027SRobin Gong sdma_config_ownership(sdmac, false, true, false); 13870f06c027SRobin Gong 138821420841SRobin Gong if (sdma_load_context(sdmac)) 138921420841SRobin Gong goto err_desc_out; 139021420841SRobin Gong 139121420841SRobin Gong return desc; 139221420841SRobin Gong 139321420841SRobin Gong err_desc_out: 139421420841SRobin Gong kfree(desc); 139521420841SRobin Gong err_out: 139621420841SRobin Gong return NULL; 139721420841SRobin Gong } 139821420841SRobin Gong 13990f06c027SRobin Gong static struct dma_async_tx_descriptor *sdma_prep_memcpy( 14000f06c027SRobin Gong struct dma_chan *chan, dma_addr_t dma_dst, 14010f06c027SRobin Gong dma_addr_t dma_src, size_t len, unsigned long flags) 14020f06c027SRobin Gong { 14030f06c027SRobin Gong struct sdma_channel *sdmac = to_sdma_chan(chan); 14040f06c027SRobin Gong struct sdma_engine *sdma = sdmac->sdma; 14050f06c027SRobin Gong int channel = sdmac->channel; 14060f06c027SRobin Gong size_t count; 14070f06c027SRobin Gong int i = 0, param; 14080f06c027SRobin Gong struct sdma_buffer_descriptor *bd; 14090f06c027SRobin Gong struct sdma_desc *desc; 14100f06c027SRobin Gong 14110f06c027SRobin Gong if (!chan || !len) 14120f06c027SRobin Gong return NULL; 14130f06c027SRobin Gong 14140f06c027SRobin Gong dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n", 14150f06c027SRobin Gong &dma_src, &dma_dst, len, channel); 14160f06c027SRobin Gong 14170f06c027SRobin Gong desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM, 14180f06c027SRobin Gong len / SDMA_BD_MAX_CNT + 1); 14190f06c027SRobin Gong if (!desc) 14200f06c027SRobin Gong return NULL; 14210f06c027SRobin Gong 14220f06c027SRobin Gong do { 14230f06c027SRobin Gong count = min_t(size_t, len, SDMA_BD_MAX_CNT); 14240f06c027SRobin Gong bd = &desc->bd[i]; 14250f06c027SRobin Gong bd->buffer_addr = dma_src; 14260f06c027SRobin Gong bd->ext_buffer_addr = dma_dst; 14270f06c027SRobin Gong bd->mode.count = count; 14280f06c027SRobin Gong desc->chn_count += count; 14290f06c027SRobin Gong bd->mode.command = 0; 14300f06c027SRobin Gong 14310f06c027SRobin Gong dma_src += count; 14320f06c027SRobin Gong dma_dst += count; 14330f06c027SRobin Gong len -= count; 14340f06c027SRobin Gong i++; 14350f06c027SRobin Gong 14360f06c027SRobin Gong param = BD_DONE | BD_EXTD | BD_CONT; 14370f06c027SRobin Gong /* last bd */ 14380f06c027SRobin Gong if (!len) { 14390f06c027SRobin Gong param |= BD_INTR; 14400f06c027SRobin Gong param |= BD_LAST; 14410f06c027SRobin Gong param &= ~BD_CONT; 14420f06c027SRobin Gong } 14430f06c027SRobin Gong 14440f06c027SRobin Gong dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n", 14450f06c027SRobin Gong i, count, bd->buffer_addr, 14460f06c027SRobin Gong param & BD_WRAP ? "wrap" : "", 14470f06c027SRobin Gong param & BD_INTR ? " intr" : ""); 14480f06c027SRobin Gong 14490f06c027SRobin Gong bd->mode.status = param; 14500f06c027SRobin Gong } while (len); 14510f06c027SRobin Gong 14520f06c027SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); 14530f06c027SRobin Gong } 14540f06c027SRobin Gong 14551ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_slave_sg( 14561ec1e82fSSascha Hauer struct dma_chan *chan, struct scatterlist *sgl, 1457db8196dfSVinod Koul unsigned int sg_len, enum dma_transfer_direction direction, 1458185ecb5fSAlexandre Bounine unsigned long flags, void *context) 14591ec1e82fSSascha Hauer { 14601ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 14611ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 1462ad78b000SVinod Koul int i, count; 146323889c63SSascha Hauer int channel = sdmac->channel; 14641ec1e82fSSascha Hauer struct scatterlist *sg; 146557b772b8SRobin Gong struct sdma_desc *desc; 14661ec1e82fSSascha Hauer 1467107d0644SVinod Koul sdma_config_write(chan, &sdmac->slave_config, direction); 1468107d0644SVinod Koul 146921420841SRobin Gong desc = sdma_transfer_init(sdmac, direction, sg_len); 147057b772b8SRobin Gong if (!desc) 147157b772b8SRobin Gong goto err_out; 147257b772b8SRobin Gong 14731ec1e82fSSascha Hauer dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", 14741ec1e82fSSascha Hauer sg_len, channel); 14751ec1e82fSSascha Hauer 14761ec1e82fSSascha Hauer for_each_sg(sgl, sg, sg_len, i) { 147776c33d27SSascha Hauer struct sdma_buffer_descriptor *bd = &desc->bd[i]; 14781ec1e82fSSascha Hauer int param; 14791ec1e82fSSascha Hauer 1480d2f5c276SAnatolij Gustschin bd->buffer_addr = sg->dma_address; 14811ec1e82fSSascha Hauer 1482fdaf9c4bSLars-Peter Clausen count = sg_dma_len(sg); 14831ec1e82fSSascha Hauer 14844a6b2e8aSRobin Gong if (count > SDMA_BD_MAX_CNT) { 14851ec1e82fSSascha Hauer dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n", 14864a6b2e8aSRobin Gong channel, count, SDMA_BD_MAX_CNT); 148757b772b8SRobin Gong goto err_bd_out; 14881ec1e82fSSascha Hauer } 14891ec1e82fSSascha Hauer 14901ec1e82fSSascha Hauer bd->mode.count = count; 149176c33d27SSascha Hauer desc->chn_count += count; 14921ec1e82fSSascha Hauer 1493ad78b000SVinod Koul if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) 149457b772b8SRobin Gong goto err_bd_out; 14951fa81c27SSascha Hauer 14961fa81c27SSascha Hauer switch (sdmac->word_size) { 14971fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_4_BYTES: 14981ec1e82fSSascha Hauer bd->mode.command = 0; 14991fa81c27SSascha Hauer if (count & 3 || sg->dma_address & 3) 150057b772b8SRobin Gong goto err_bd_out; 15011fa81c27SSascha Hauer break; 15021fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_2_BYTES: 15031fa81c27SSascha Hauer bd->mode.command = 2; 15041fa81c27SSascha Hauer if (count & 1 || sg->dma_address & 1) 150557b772b8SRobin Gong goto err_bd_out; 15061fa81c27SSascha Hauer break; 15071fa81c27SSascha Hauer case DMA_SLAVE_BUSWIDTH_1_BYTE: 15081fa81c27SSascha Hauer bd->mode.command = 1; 15091fa81c27SSascha Hauer break; 15101fa81c27SSascha Hauer default: 151157b772b8SRobin Gong goto err_bd_out; 15121fa81c27SSascha Hauer } 15131ec1e82fSSascha Hauer 15141ec1e82fSSascha Hauer param = BD_DONE | BD_EXTD | BD_CONT; 15151ec1e82fSSascha Hauer 1516341b9419SShawn Guo if (i + 1 == sg_len) { 15171ec1e82fSSascha Hauer param |= BD_INTR; 1518341b9419SShawn Guo param |= BD_LAST; 1519341b9419SShawn Guo param &= ~BD_CONT; 15201ec1e82fSSascha Hauer } 15211ec1e82fSSascha Hauer 1522c3cc74b2SOlof Johansson dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n", 1523c3cc74b2SOlof Johansson i, count, (u64)sg->dma_address, 15241ec1e82fSSascha Hauer param & BD_WRAP ? "wrap" : "", 15251ec1e82fSSascha Hauer param & BD_INTR ? " intr" : ""); 15261ec1e82fSSascha Hauer 15271ec1e82fSSascha Hauer bd->mode.status = param; 15281ec1e82fSSascha Hauer } 15291ec1e82fSSascha Hauer 153057b772b8SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); 153157b772b8SRobin Gong err_bd_out: 153257b772b8SRobin Gong sdma_free_bd(desc); 153357b772b8SRobin Gong kfree(desc); 15341ec1e82fSSascha Hauer err_out: 15354b2ce9ddSShawn Guo sdmac->status = DMA_ERROR; 15361ec1e82fSSascha Hauer return NULL; 15371ec1e82fSSascha Hauer } 15381ec1e82fSSascha Hauer 15391ec1e82fSSascha Hauer static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( 15401ec1e82fSSascha Hauer struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, 1541185ecb5fSAlexandre Bounine size_t period_len, enum dma_transfer_direction direction, 154231c1e5a1SLaurent Pinchart unsigned long flags) 15431ec1e82fSSascha Hauer { 15441ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 15451ec1e82fSSascha Hauer struct sdma_engine *sdma = sdmac->sdma; 15461ec1e82fSSascha Hauer int num_periods = buf_len / period_len; 154723889c63SSascha Hauer int channel = sdmac->channel; 154821420841SRobin Gong int i = 0, buf = 0; 154957b772b8SRobin Gong struct sdma_desc *desc; 15501ec1e82fSSascha Hauer 15511ec1e82fSSascha Hauer dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); 15521ec1e82fSSascha Hauer 1553107d0644SVinod Koul sdma_config_write(chan, &sdmac->slave_config, direction); 1554107d0644SVinod Koul 155521420841SRobin Gong desc = sdma_transfer_init(sdmac, direction, num_periods); 155657b772b8SRobin Gong if (!desc) 155757b772b8SRobin Gong goto err_out; 155857b772b8SRobin Gong 155976c33d27SSascha Hauer desc->period_len = period_len; 15608e2e27c7SRichard Zhao 15611ec1e82fSSascha Hauer sdmac->flags |= IMX_DMA_SG_LOOP; 15621ec1e82fSSascha Hauer 15634a6b2e8aSRobin Gong if (period_len > SDMA_BD_MAX_CNT) { 1564ba6ab3b3SArvind Yadav dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", 15654a6b2e8aSRobin Gong channel, period_len, SDMA_BD_MAX_CNT); 156657b772b8SRobin Gong goto err_bd_out; 15671ec1e82fSSascha Hauer } 15681ec1e82fSSascha Hauer 15691ec1e82fSSascha Hauer while (buf < buf_len) { 157076c33d27SSascha Hauer struct sdma_buffer_descriptor *bd = &desc->bd[i]; 15711ec1e82fSSascha Hauer int param; 15721ec1e82fSSascha Hauer 15731ec1e82fSSascha Hauer bd->buffer_addr = dma_addr; 15741ec1e82fSSascha Hauer 15751ec1e82fSSascha Hauer bd->mode.count = period_len; 15761ec1e82fSSascha Hauer 15771ec1e82fSSascha Hauer if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) 157857b772b8SRobin Gong goto err_bd_out; 15791ec1e82fSSascha Hauer if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES) 15801ec1e82fSSascha Hauer bd->mode.command = 0; 15811ec1e82fSSascha Hauer else 15821ec1e82fSSascha Hauer bd->mode.command = sdmac->word_size; 15831ec1e82fSSascha Hauer 15841ec1e82fSSascha Hauer param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR; 15851ec1e82fSSascha Hauer if (i + 1 == num_periods) 15861ec1e82fSSascha Hauer param |= BD_WRAP; 15871ec1e82fSSascha Hauer 1588ba6ab3b3SArvind Yadav dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n", 1589c3cc74b2SOlof Johansson i, period_len, (u64)dma_addr, 15901ec1e82fSSascha Hauer param & BD_WRAP ? "wrap" : "", 15911ec1e82fSSascha Hauer param & BD_INTR ? " intr" : ""); 15921ec1e82fSSascha Hauer 15931ec1e82fSSascha Hauer bd->mode.status = param; 15941ec1e82fSSascha Hauer 15951ec1e82fSSascha Hauer dma_addr += period_len; 15961ec1e82fSSascha Hauer buf += period_len; 15971ec1e82fSSascha Hauer 15981ec1e82fSSascha Hauer i++; 15991ec1e82fSSascha Hauer } 16001ec1e82fSSascha Hauer 160157b772b8SRobin Gong return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); 160257b772b8SRobin Gong err_bd_out: 160357b772b8SRobin Gong sdma_free_bd(desc); 160457b772b8SRobin Gong kfree(desc); 16051ec1e82fSSascha Hauer err_out: 16061ec1e82fSSascha Hauer sdmac->status = DMA_ERROR; 16071ec1e82fSSascha Hauer return NULL; 16081ec1e82fSSascha Hauer } 16091ec1e82fSSascha Hauer 1610107d0644SVinod Koul static int sdma_config_write(struct dma_chan *chan, 1611107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg, 1612107d0644SVinod Koul enum dma_transfer_direction direction) 16131ec1e82fSSascha Hauer { 16141ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 16151ec1e82fSSascha Hauer 1616107d0644SVinod Koul if (direction == DMA_DEV_TO_MEM) { 16171ec1e82fSSascha Hauer sdmac->per_address = dmaengine_cfg->src_addr; 161894ac27a5SPhilippe Rétornaz sdmac->watermark_level = dmaengine_cfg->src_maxburst * 161994ac27a5SPhilippe Rétornaz dmaengine_cfg->src_addr_width; 16201ec1e82fSSascha Hauer sdmac->word_size = dmaengine_cfg->src_addr_width; 1621107d0644SVinod Koul } else if (direction == DMA_DEV_TO_DEV) { 16228391ecf4SShengjiu Wang sdmac->per_address2 = dmaengine_cfg->src_addr; 16238391ecf4SShengjiu Wang sdmac->per_address = dmaengine_cfg->dst_addr; 16248391ecf4SShengjiu Wang sdmac->watermark_level = dmaengine_cfg->src_maxburst & 16258391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_LWML; 16268391ecf4SShengjiu Wang sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & 16278391ecf4SShengjiu Wang SDMA_WATERMARK_LEVEL_HWML; 16288391ecf4SShengjiu Wang sdmac->word_size = dmaengine_cfg->dst_addr_width; 16291ec1e82fSSascha Hauer } else { 16301ec1e82fSSascha Hauer sdmac->per_address = dmaengine_cfg->dst_addr; 163194ac27a5SPhilippe Rétornaz sdmac->watermark_level = dmaengine_cfg->dst_maxburst * 163294ac27a5SPhilippe Rétornaz dmaengine_cfg->dst_addr_width; 16331ec1e82fSSascha Hauer sdmac->word_size = dmaengine_cfg->dst_addr_width; 16341ec1e82fSSascha Hauer } 1635107d0644SVinod Koul sdmac->direction = direction; 16367b350ab0SMaxime Ripard return sdma_config_channel(chan); 16371ec1e82fSSascha Hauer } 16381ec1e82fSSascha Hauer 1639107d0644SVinod Koul static int sdma_config(struct dma_chan *chan, 1640107d0644SVinod Koul struct dma_slave_config *dmaengine_cfg) 1641107d0644SVinod Koul { 1642107d0644SVinod Koul struct sdma_channel *sdmac = to_sdma_chan(chan); 1643107d0644SVinod Koul 1644107d0644SVinod Koul memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); 1645107d0644SVinod Koul 1646107d0644SVinod Koul /* Set ENBLn earlier to make sure dma request triggered after that */ 1647107d0644SVinod Koul if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) 1648107d0644SVinod Koul return -EINVAL; 1649107d0644SVinod Koul sdma_event_enable(sdmac, sdmac->event_id0); 1650107d0644SVinod Koul 1651107d0644SVinod Koul if (sdmac->event_id1) { 1652107d0644SVinod Koul if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) 1653107d0644SVinod Koul return -EINVAL; 1654107d0644SVinod Koul sdma_event_enable(sdmac, sdmac->event_id1); 1655107d0644SVinod Koul } 1656107d0644SVinod Koul 1657107d0644SVinod Koul return 0; 1658107d0644SVinod Koul } 1659107d0644SVinod Koul 16601ec1e82fSSascha Hauer static enum dma_status sdma_tx_status(struct dma_chan *chan, 16611ec1e82fSSascha Hauer dma_cookie_t cookie, 16621ec1e82fSSascha Hauer struct dma_tx_state *txstate) 16631ec1e82fSSascha Hauer { 16641ec1e82fSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 1665a1ff6a07SSascha Hauer struct sdma_desc *desc = NULL; 1666d1a792f3SRussell King - ARM Linux u32 residue; 166757b772b8SRobin Gong struct virt_dma_desc *vd; 166857b772b8SRobin Gong enum dma_status ret; 166957b772b8SRobin Gong unsigned long flags; 1670d1a792f3SRussell King - ARM Linux 167157b772b8SRobin Gong ret = dma_cookie_status(chan, cookie, txstate); 167257b772b8SRobin Gong if (ret == DMA_COMPLETE || !txstate) 167357b772b8SRobin Gong return ret; 167457b772b8SRobin Gong 167557b772b8SRobin Gong spin_lock_irqsave(&sdmac->vc.lock, flags); 1676a1ff6a07SSascha Hauer 167757b772b8SRobin Gong vd = vchan_find_desc(&sdmac->vc, cookie); 1678a1ff6a07SSascha Hauer if (vd) 167957b772b8SRobin Gong desc = to_sdma_desc(&vd->tx); 1680a1ff6a07SSascha Hauer else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) 1681a1ff6a07SSascha Hauer desc = sdmac->desc; 1682a1ff6a07SSascha Hauer 1683a1ff6a07SSascha Hauer if (desc) { 1684d1a792f3SRussell King - ARM Linux if (sdmac->flags & IMX_DMA_SG_LOOP) 168576c33d27SSascha Hauer residue = (desc->num_bd - desc->buf_ptail) * 168676c33d27SSascha Hauer desc->period_len - desc->chn_real_count; 1687d1a792f3SRussell King - ARM Linux else 168876c33d27SSascha Hauer residue = desc->chn_count - desc->chn_real_count; 168957b772b8SRobin Gong } else { 169057b772b8SRobin Gong residue = 0; 169157b772b8SRobin Gong } 1692a1ff6a07SSascha Hauer 169357b772b8SRobin Gong spin_unlock_irqrestore(&sdmac->vc.lock, flags); 16941ec1e82fSSascha Hauer 1695e8e3a790SAndy Shevchenko dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, 1696d1a792f3SRussell King - ARM Linux residue); 16971ec1e82fSSascha Hauer 16988a965911SShawn Guo return sdmac->status; 16991ec1e82fSSascha Hauer } 17001ec1e82fSSascha Hauer 17011ec1e82fSSascha Hauer static void sdma_issue_pending(struct dma_chan *chan) 17021ec1e82fSSascha Hauer { 17032b4f130eSSascha Hauer struct sdma_channel *sdmac = to_sdma_chan(chan); 170457b772b8SRobin Gong unsigned long flags; 17052b4f130eSSascha Hauer 170657b772b8SRobin Gong spin_lock_irqsave(&sdmac->vc.lock, flags); 170757b772b8SRobin Gong if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc) 170857b772b8SRobin Gong sdma_start_desc(sdmac); 170957b772b8SRobin Gong spin_unlock_irqrestore(&sdmac->vc.lock, flags); 17101ec1e82fSSascha Hauer } 17111ec1e82fSSascha Hauer 17125b28aa31SSascha Hauer #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 1713cd72b846SNicolin Chen #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 1714a572460bSFabio Estevam #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 1715b7d2648aSFabio Estevam #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42 17165b28aa31SSascha Hauer 17175b28aa31SSascha Hauer static void sdma_add_scripts(struct sdma_engine *sdma, 17185b28aa31SSascha Hauer const struct sdma_script_start_addrs *addr) 17195b28aa31SSascha Hauer { 17205b28aa31SSascha Hauer s32 *addr_arr = (u32 *)addr; 17215b28aa31SSascha Hauer s32 *saddr_arr = (u32 *)sdma->script_addrs; 17225b28aa31SSascha Hauer int i; 17235b28aa31SSascha Hauer 172470dabaedSNicolin Chen /* use the default firmware in ROM if missing external firmware */ 172570dabaedSNicolin Chen if (!sdma->script_number) 172670dabaedSNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; 172770dabaedSNicolin Chen 1728bd73dfabSRobin Gong if (sdma->script_number > sizeof(struct sdma_script_start_addrs) 1729bd73dfabSRobin Gong / sizeof(s32)) { 1730bd73dfabSRobin Gong dev_err(sdma->dev, 1731bd73dfabSRobin Gong "SDMA script number %d not match with firmware.\n", 1732bd73dfabSRobin Gong sdma->script_number); 1733bd73dfabSRobin Gong return; 1734bd73dfabSRobin Gong } 1735bd73dfabSRobin Gong 1736cd72b846SNicolin Chen for (i = 0; i < sdma->script_number; i++) 17375b28aa31SSascha Hauer if (addr_arr[i] > 0) 17385b28aa31SSascha Hauer saddr_arr[i] = addr_arr[i]; 17395b28aa31SSascha Hauer } 17405b28aa31SSascha Hauer 17417b4b88e0SSascha Hauer static void sdma_load_firmware(const struct firmware *fw, void *context) 17425b28aa31SSascha Hauer { 17437b4b88e0SSascha Hauer struct sdma_engine *sdma = context; 17445b28aa31SSascha Hauer const struct sdma_firmware_header *header; 17455b28aa31SSascha Hauer const struct sdma_script_start_addrs *addr; 17465b28aa31SSascha Hauer unsigned short *ram_code; 17475b28aa31SSascha Hauer 17487b4b88e0SSascha Hauer if (!fw) { 17490f927a11SSascha Hauer dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); 17500f927a11SSascha Hauer /* In this case we just use the ROM firmware. */ 17517b4b88e0SSascha Hauer return; 17527b4b88e0SSascha Hauer } 17535b28aa31SSascha Hauer 17545b28aa31SSascha Hauer if (fw->size < sizeof(*header)) 17555b28aa31SSascha Hauer goto err_firmware; 17565b28aa31SSascha Hauer 17575b28aa31SSascha Hauer header = (struct sdma_firmware_header *)fw->data; 17585b28aa31SSascha Hauer 17595b28aa31SSascha Hauer if (header->magic != SDMA_FIRMWARE_MAGIC) 17605b28aa31SSascha Hauer goto err_firmware; 17615b28aa31SSascha Hauer if (header->ram_code_start + header->ram_code_size > fw->size) 17625b28aa31SSascha Hauer goto err_firmware; 1763cd72b846SNicolin Chen switch (header->version_major) { 1764cd72b846SNicolin Chen case 1: 1765cd72b846SNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; 1766cd72b846SNicolin Chen break; 1767cd72b846SNicolin Chen case 2: 1768cd72b846SNicolin Chen sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; 1769cd72b846SNicolin Chen break; 1770a572460bSFabio Estevam case 3: 1771a572460bSFabio Estevam sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3; 1772a572460bSFabio Estevam break; 1773b7d2648aSFabio Estevam case 4: 1774b7d2648aSFabio Estevam sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4; 1775b7d2648aSFabio Estevam break; 1776cd72b846SNicolin Chen default: 1777cd72b846SNicolin Chen dev_err(sdma->dev, "unknown firmware version\n"); 1778cd72b846SNicolin Chen goto err_firmware; 1779cd72b846SNicolin Chen } 17805b28aa31SSascha Hauer 17815b28aa31SSascha Hauer addr = (void *)header + header->script_addrs_start; 17825b28aa31SSascha Hauer ram_code = (void *)header + header->ram_code_start; 17835b28aa31SSascha Hauer 17847560e3f3SSascha Hauer clk_enable(sdma->clk_ipg); 17857560e3f3SSascha Hauer clk_enable(sdma->clk_ahb); 17865b28aa31SSascha Hauer /* download the RAM image for SDMA */ 17875b28aa31SSascha Hauer sdma_load_script(sdma, ram_code, 17885b28aa31SSascha Hauer header->ram_code_size, 17896866fd3bSSascha Hauer addr->ram_code_start_addr); 17907560e3f3SSascha Hauer clk_disable(sdma->clk_ipg); 17917560e3f3SSascha Hauer clk_disable(sdma->clk_ahb); 17925b28aa31SSascha Hauer 17935b28aa31SSascha Hauer sdma_add_scripts(sdma, addr); 17945b28aa31SSascha Hauer 17955b28aa31SSascha Hauer dev_info(sdma->dev, "loaded firmware %d.%d\n", 17965b28aa31SSascha Hauer header->version_major, 17975b28aa31SSascha Hauer header->version_minor); 17985b28aa31SSascha Hauer 17995b28aa31SSascha Hauer err_firmware: 18005b28aa31SSascha Hauer release_firmware(fw); 18017b4b88e0SSascha Hauer } 18027b4b88e0SSascha Hauer 1803d078cd1bSZidan Wang #define EVENT_REMAP_CELLS 3 1804d078cd1bSZidan Wang 180529f493daSJason Liu static int sdma_event_remap(struct sdma_engine *sdma) 1806d078cd1bSZidan Wang { 1807d078cd1bSZidan Wang struct device_node *np = sdma->dev->of_node; 1808d078cd1bSZidan Wang struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0); 1809d078cd1bSZidan Wang struct property *event_remap; 1810d078cd1bSZidan Wang struct regmap *gpr; 1811d078cd1bSZidan Wang char propname[] = "fsl,sdma-event-remap"; 1812d078cd1bSZidan Wang u32 reg, val, shift, num_map, i; 1813d078cd1bSZidan Wang int ret = 0; 1814d078cd1bSZidan Wang 1815d078cd1bSZidan Wang if (IS_ERR(np) || IS_ERR(gpr_np)) 1816d078cd1bSZidan Wang goto out; 1817d078cd1bSZidan Wang 1818d078cd1bSZidan Wang event_remap = of_find_property(np, propname, NULL); 1819d078cd1bSZidan Wang num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0; 1820d078cd1bSZidan Wang if (!num_map) { 1821ce078af7SFabio Estevam dev_dbg(sdma->dev, "no event needs to be remapped\n"); 1822d078cd1bSZidan Wang goto out; 1823d078cd1bSZidan Wang } else if (num_map % EVENT_REMAP_CELLS) { 1824d078cd1bSZidan Wang dev_err(sdma->dev, "the property %s must modulo %d\n", 1825d078cd1bSZidan Wang propname, EVENT_REMAP_CELLS); 1826d078cd1bSZidan Wang ret = -EINVAL; 1827d078cd1bSZidan Wang goto out; 1828d078cd1bSZidan Wang } 1829d078cd1bSZidan Wang 1830d078cd1bSZidan Wang gpr = syscon_node_to_regmap(gpr_np); 1831d078cd1bSZidan Wang if (IS_ERR(gpr)) { 1832d078cd1bSZidan Wang dev_err(sdma->dev, "failed to get gpr regmap\n"); 1833d078cd1bSZidan Wang ret = PTR_ERR(gpr); 1834d078cd1bSZidan Wang goto out; 1835d078cd1bSZidan Wang } 1836d078cd1bSZidan Wang 1837d078cd1bSZidan Wang for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) { 1838d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i, ®); 1839d078cd1bSZidan Wang if (ret) { 1840d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n", 1841d078cd1bSZidan Wang propname, i); 1842d078cd1bSZidan Wang goto out; 1843d078cd1bSZidan Wang } 1844d078cd1bSZidan Wang 1845d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i + 1, &shift); 1846d078cd1bSZidan Wang if (ret) { 1847d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n", 1848d078cd1bSZidan Wang propname, i + 1); 1849d078cd1bSZidan Wang goto out; 1850d078cd1bSZidan Wang } 1851d078cd1bSZidan Wang 1852d078cd1bSZidan Wang ret = of_property_read_u32_index(np, propname, i + 2, &val); 1853d078cd1bSZidan Wang if (ret) { 1854d078cd1bSZidan Wang dev_err(sdma->dev, "failed to read property %s index %d\n", 1855d078cd1bSZidan Wang propname, i + 2); 1856d078cd1bSZidan Wang goto out; 1857d078cd1bSZidan Wang } 1858d078cd1bSZidan Wang 1859d078cd1bSZidan Wang regmap_update_bits(gpr, reg, BIT(shift), val << shift); 1860d078cd1bSZidan Wang } 1861d078cd1bSZidan Wang 1862d078cd1bSZidan Wang out: 1863d078cd1bSZidan Wang if (!IS_ERR(gpr_np)) 1864d078cd1bSZidan Wang of_node_put(gpr_np); 1865d078cd1bSZidan Wang 1866d078cd1bSZidan Wang return ret; 1867d078cd1bSZidan Wang } 1868d078cd1bSZidan Wang 1869fe6cf289SArnd Bergmann static int sdma_get_firmware(struct sdma_engine *sdma, 18707b4b88e0SSascha Hauer const char *fw_name) 18717b4b88e0SSascha Hauer { 18727b4b88e0SSascha Hauer int ret; 18737b4b88e0SSascha Hauer 18747b4b88e0SSascha Hauer ret = request_firmware_nowait(THIS_MODULE, 18750733d839SShawn Guo FW_ACTION_UEVENT, fw_name, sdma->dev, 18767b4b88e0SSascha Hauer GFP_KERNEL, sdma, sdma_load_firmware); 18775b28aa31SSascha Hauer 18785b28aa31SSascha Hauer return ret; 18795b28aa31SSascha Hauer } 18805b28aa31SSascha Hauer 188119bfc772SJingoo Han static int sdma_init(struct sdma_engine *sdma) 18821ec1e82fSSascha Hauer { 18831ec1e82fSSascha Hauer int i, ret; 18841ec1e82fSSascha Hauer dma_addr_t ccb_phys; 18851ec1e82fSSascha Hauer 1886b93edcddSFabio Estevam ret = clk_enable(sdma->clk_ipg); 1887b93edcddSFabio Estevam if (ret) 1888b93edcddSFabio Estevam return ret; 1889b93edcddSFabio Estevam ret = clk_enable(sdma->clk_ahb); 1890b93edcddSFabio Estevam if (ret) 1891b93edcddSFabio Estevam goto disable_clk_ipg; 18921ec1e82fSSascha Hauer 1893941acd56SAngus Ainslie (Purism) if (sdma->drvdata->check_ratio && 1894941acd56SAngus Ainslie (Purism) (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg))) 189525aaa75dSAngus Ainslie (Purism) sdma->clk_ratio = 1; 189625aaa75dSAngus Ainslie (Purism) 18971ec1e82fSSascha Hauer /* Be sure SDMA has not started yet */ 1898c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); 18991ec1e82fSSascha Hauer 1900ceaf5226SAndy Duan sdma->channel_control = dma_alloc_coherent(sdma->dev, 19011ec1e82fSSascha Hauer MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + 19021ec1e82fSSascha Hauer sizeof(struct sdma_context_data), 19031ec1e82fSSascha Hauer &ccb_phys, GFP_KERNEL); 19041ec1e82fSSascha Hauer 19051ec1e82fSSascha Hauer if (!sdma->channel_control) { 19061ec1e82fSSascha Hauer ret = -ENOMEM; 19071ec1e82fSSascha Hauer goto err_dma_alloc; 19081ec1e82fSSascha Hauer } 19091ec1e82fSSascha Hauer 19101ec1e82fSSascha Hauer sdma->context = (void *)sdma->channel_control + 19111ec1e82fSSascha Hauer MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); 19121ec1e82fSSascha Hauer sdma->context_phys = ccb_phys + 19131ec1e82fSSascha Hauer MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); 19141ec1e82fSSascha Hauer 19151ec1e82fSSascha Hauer /* disable all channels */ 191617bba72fSSascha Hauer for (i = 0; i < sdma->drvdata->num_events; i++) 1917c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); 19181ec1e82fSSascha Hauer 19191ec1e82fSSascha Hauer /* All channels have priority 0 */ 19201ec1e82fSSascha Hauer for (i = 0; i < MAX_DMA_CHANNELS; i++) 1921c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); 19221ec1e82fSSascha Hauer 192357b772b8SRobin Gong ret = sdma_request_channel0(sdma); 19241ec1e82fSSascha Hauer if (ret) 19251ec1e82fSSascha Hauer goto err_dma_alloc; 19261ec1e82fSSascha Hauer 19271ec1e82fSSascha Hauer sdma_config_ownership(&sdma->channel[0], false, true, false); 19281ec1e82fSSascha Hauer 19291ec1e82fSSascha Hauer /* Set Command Channel (Channel Zero) */ 1930c4b56857SRichard Zhao writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); 19311ec1e82fSSascha Hauer 19321ec1e82fSSascha Hauer /* Set bits of CONFIG register but with static context switching */ 193325aaa75dSAngus Ainslie (Purism) if (sdma->clk_ratio) 193425aaa75dSAngus Ainslie (Purism) writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); 193525aaa75dSAngus Ainslie (Purism) else 1936c4b56857SRichard Zhao writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); 19371ec1e82fSSascha Hauer 1938c4b56857SRichard Zhao writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); 19391ec1e82fSSascha Hauer 19401ec1e82fSSascha Hauer /* Initializes channel's priorities */ 19411ec1e82fSSascha Hauer sdma_set_channel_priority(&sdma->channel[0], 7); 19421ec1e82fSSascha Hauer 19437560e3f3SSascha Hauer clk_disable(sdma->clk_ipg); 19447560e3f3SSascha Hauer clk_disable(sdma->clk_ahb); 19451ec1e82fSSascha Hauer 19461ec1e82fSSascha Hauer return 0; 19471ec1e82fSSascha Hauer 19481ec1e82fSSascha Hauer err_dma_alloc: 19497560e3f3SSascha Hauer clk_disable(sdma->clk_ahb); 1950b93edcddSFabio Estevam disable_clk_ipg: 1951b93edcddSFabio Estevam clk_disable(sdma->clk_ipg); 19521ec1e82fSSascha Hauer dev_err(sdma->dev, "initialisation failed with %d\n", ret); 19531ec1e82fSSascha Hauer return ret; 19541ec1e82fSSascha Hauer } 19551ec1e82fSSascha Hauer 19569479e17cSShawn Guo static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) 19579479e17cSShawn Guo { 19580b351865SNicolin Chen struct sdma_channel *sdmac = to_sdma_chan(chan); 19599479e17cSShawn Guo struct imx_dma_data *data = fn_param; 19609479e17cSShawn Guo 19619479e17cSShawn Guo if (!imx_dma_is_general_purpose(chan)) 19629479e17cSShawn Guo return false; 19639479e17cSShawn Guo 19640b351865SNicolin Chen sdmac->data = *data; 19650b351865SNicolin Chen chan->private = &sdmac->data; 19669479e17cSShawn Guo 19679479e17cSShawn Guo return true; 19689479e17cSShawn Guo } 19699479e17cSShawn Guo 19709479e17cSShawn Guo static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, 19719479e17cSShawn Guo struct of_dma *ofdma) 19729479e17cSShawn Guo { 19739479e17cSShawn Guo struct sdma_engine *sdma = ofdma->of_dma_data; 19749479e17cSShawn Guo dma_cap_mask_t mask = sdma->dma_device.cap_mask; 19759479e17cSShawn Guo struct imx_dma_data data; 19769479e17cSShawn Guo 19779479e17cSShawn Guo if (dma_spec->args_count != 3) 19789479e17cSShawn Guo return NULL; 19799479e17cSShawn Guo 19809479e17cSShawn Guo data.dma_request = dma_spec->args[0]; 19819479e17cSShawn Guo data.peripheral_type = dma_spec->args[1]; 19829479e17cSShawn Guo data.priority = dma_spec->args[2]; 19838391ecf4SShengjiu Wang /* 19848391ecf4SShengjiu Wang * init dma_request2 to zero, which is not used by the dts. 19858391ecf4SShengjiu Wang * For P2P, dma_request2 is init from dma_request_channel(), 19868391ecf4SShengjiu Wang * chan->private will point to the imx_dma_data, and in 19878391ecf4SShengjiu Wang * device_alloc_chan_resources(), imx_dma_data.dma_request2 will 19888391ecf4SShengjiu Wang * be set to sdmac->event_id1. 19898391ecf4SShengjiu Wang */ 19908391ecf4SShengjiu Wang data.dma_request2 = 0; 19919479e17cSShawn Guo 1992990c0b53SBaolin Wang return __dma_request_channel(&mask, sdma_filter_fn, &data, 1993990c0b53SBaolin Wang ofdma->of_node); 19949479e17cSShawn Guo } 19959479e17cSShawn Guo 1996e34b731fSMark Brown static int sdma_probe(struct platform_device *pdev) 19971ec1e82fSSascha Hauer { 1998580975d7SShawn Guo struct device_node *np = pdev->dev.of_node; 19998391ecf4SShengjiu Wang struct device_node *spba_bus; 2000580975d7SShawn Guo const char *fw_name; 20011ec1e82fSSascha Hauer int ret; 20021ec1e82fSSascha Hauer int irq; 20031ec1e82fSSascha Hauer struct resource *iores; 20048391ecf4SShengjiu Wang struct resource spba_res; 20051ec1e82fSSascha Hauer int i; 20061ec1e82fSSascha Hauer struct sdma_engine *sdma; 200736e2f21aSSascha Hauer s32 *saddr_arr; 20081ec1e82fSSascha Hauer 200942536b9fSPhilippe Retornaz ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 201042536b9fSPhilippe Retornaz if (ret) 201142536b9fSPhilippe Retornaz return ret; 201242536b9fSPhilippe Retornaz 20137f24e0eeSFabio Estevam sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL); 20141ec1e82fSSascha Hauer if (!sdma) 20151ec1e82fSSascha Hauer return -ENOMEM; 20161ec1e82fSSascha Hauer 20172ccaef05SRichard Zhao spin_lock_init(&sdma->channel_0_lock); 201873eab978SSascha Hauer 20191ec1e82fSSascha Hauer sdma->dev = &pdev->dev; 202032996419SFabio Estevam sdma->drvdata = of_device_get_match_data(sdma->dev); 20211ec1e82fSSascha Hauer 20221ec1e82fSSascha Hauer irq = platform_get_irq(pdev, 0); 20237f24e0eeSFabio Estevam if (irq < 0) 202463c72e02SFabio Estevam return irq; 20251ec1e82fSSascha Hauer 20267f24e0eeSFabio Estevam iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); 20277f24e0eeSFabio Estevam sdma->regs = devm_ioremap_resource(&pdev->dev, iores); 20287f24e0eeSFabio Estevam if (IS_ERR(sdma->regs)) 20297f24e0eeSFabio Estevam return PTR_ERR(sdma->regs); 20301ec1e82fSSascha Hauer 20317560e3f3SSascha Hauer sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); 20327f24e0eeSFabio Estevam if (IS_ERR(sdma->clk_ipg)) 20337f24e0eeSFabio Estevam return PTR_ERR(sdma->clk_ipg); 20341ec1e82fSSascha Hauer 20357560e3f3SSascha Hauer sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); 20367f24e0eeSFabio Estevam if (IS_ERR(sdma->clk_ahb)) 20377f24e0eeSFabio Estevam return PTR_ERR(sdma->clk_ahb); 20387560e3f3SSascha Hauer 2039fb9caf37SArvind Yadav ret = clk_prepare(sdma->clk_ipg); 2040fb9caf37SArvind Yadav if (ret) 2041fb9caf37SArvind Yadav return ret; 2042fb9caf37SArvind Yadav 2043fb9caf37SArvind Yadav ret = clk_prepare(sdma->clk_ahb); 2044fb9caf37SArvind Yadav if (ret) 2045fb9caf37SArvind Yadav goto err_clk; 20467560e3f3SSascha Hauer 20477f24e0eeSFabio Estevam ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma", 20487f24e0eeSFabio Estevam sdma); 20491ec1e82fSSascha Hauer if (ret) 2050fb9caf37SArvind Yadav goto err_irq; 20511ec1e82fSSascha Hauer 20525bb9dbb5SVinod Koul sdma->irq = irq; 20535bb9dbb5SVinod Koul 20545b28aa31SSascha Hauer sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL); 2055fb9caf37SArvind Yadav if (!sdma->script_addrs) { 2056fb9caf37SArvind Yadav ret = -ENOMEM; 2057fb9caf37SArvind Yadav goto err_irq; 2058fb9caf37SArvind Yadav } 20591ec1e82fSSascha Hauer 206036e2f21aSSascha Hauer /* initially no scripts available */ 206136e2f21aSSascha Hauer saddr_arr = (s32 *)sdma->script_addrs; 2062be4cf718SSascha Hauer for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++) 206336e2f21aSSascha Hauer saddr_arr[i] = -EINVAL; 206436e2f21aSSascha Hauer 20657214a8b1SSascha Hauer dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask); 20667214a8b1SSascha Hauer dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask); 20670f06c027SRobin Gong dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask); 20687214a8b1SSascha Hauer 20691ec1e82fSSascha Hauer INIT_LIST_HEAD(&sdma->dma_device.channels); 20701ec1e82fSSascha Hauer /* Initialize channel parameters */ 20711ec1e82fSSascha Hauer for (i = 0; i < MAX_DMA_CHANNELS; i++) { 20721ec1e82fSSascha Hauer struct sdma_channel *sdmac = &sdma->channel[i]; 20731ec1e82fSSascha Hauer 20741ec1e82fSSascha Hauer sdmac->sdma = sdma; 20751ec1e82fSSascha Hauer 20761ec1e82fSSascha Hauer sdmac->channel = i; 207757b772b8SRobin Gong sdmac->vc.desc_free = sdma_desc_free; 2078b8603d2aSLucas Stach INIT_WORK(&sdmac->terminate_worker, 2079b8603d2aSLucas Stach sdma_channel_terminate_work); 208023889c63SSascha Hauer /* 208123889c63SSascha Hauer * Add the channel to the DMAC list. Do not add channel 0 though 208223889c63SSascha Hauer * because we need it internally in the SDMA driver. This also means 208323889c63SSascha Hauer * that channel 0 in dmaengine counting matches sdma channel 1. 208423889c63SSascha Hauer */ 208523889c63SSascha Hauer if (i) 208657b772b8SRobin Gong vchan_init(&sdmac->vc, &sdma->dma_device); 20871ec1e82fSSascha Hauer } 20881ec1e82fSSascha Hauer 20895b28aa31SSascha Hauer ret = sdma_init(sdma); 20901ec1e82fSSascha Hauer if (ret) 20911ec1e82fSSascha Hauer goto err_init; 20921ec1e82fSSascha Hauer 2093d078cd1bSZidan Wang ret = sdma_event_remap(sdma); 2094d078cd1bSZidan Wang if (ret) 2095d078cd1bSZidan Wang goto err_init; 2096d078cd1bSZidan Wang 2097dcfec3c0SSascha Hauer if (sdma->drvdata->script_addrs) 2098dcfec3c0SSascha Hauer sdma_add_scripts(sdma, sdma->drvdata->script_addrs); 20995b28aa31SSascha Hauer 21001ec1e82fSSascha Hauer sdma->dma_device.dev = &pdev->dev; 21011ec1e82fSSascha Hauer 21021ec1e82fSSascha Hauer sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources; 21031ec1e82fSSascha Hauer sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources; 21041ec1e82fSSascha Hauer sdma->dma_device.device_tx_status = sdma_tx_status; 21051ec1e82fSSascha Hauer sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg; 21061ec1e82fSSascha Hauer sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic; 21077b350ab0SMaxime Ripard sdma->dma_device.device_config = sdma_config; 2108a80f2787SSascha Hauer sdma->dma_device.device_terminate_all = sdma_terminate_all; 2109b8603d2aSLucas Stach sdma->dma_device.device_synchronize = sdma_channel_synchronize; 2110f9d4a398SNicolin Chen sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS; 2111f9d4a398SNicolin Chen sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS; 2112f9d4a398SNicolin Chen sdma->dma_device.directions = SDMA_DMA_DIRECTIONS; 21136f3125ceSLucas Stach sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 21140f06c027SRobin Gong sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy; 21151ec1e82fSSascha Hauer sdma->dma_device.device_issue_pending = sdma_issue_pending; 2116a3711d49SAngus Ainslie (Purism) sdma->dma_device.copy_align = 2; 21174a6b2e8aSRobin Gong dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT); 21181ec1e82fSSascha Hauer 211923e11811SVignesh Raman platform_set_drvdata(pdev, sdma); 212023e11811SVignesh Raman 21211ec1e82fSSascha Hauer ret = dma_async_device_register(&sdma->dma_device); 21221ec1e82fSSascha Hauer if (ret) { 21231ec1e82fSSascha Hauer dev_err(&pdev->dev, "unable to register\n"); 21241ec1e82fSSascha Hauer goto err_init; 21251ec1e82fSSascha Hauer } 21261ec1e82fSSascha Hauer 21279479e17cSShawn Guo if (np) { 21289479e17cSShawn Guo ret = of_dma_controller_register(np, sdma_xlate, sdma); 21299479e17cSShawn Guo if (ret) { 21309479e17cSShawn Guo dev_err(&pdev->dev, "failed to register controller\n"); 21319479e17cSShawn Guo goto err_register; 21329479e17cSShawn Guo } 21338391ecf4SShengjiu Wang 21348391ecf4SShengjiu Wang spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); 21358391ecf4SShengjiu Wang ret = of_address_to_resource(spba_bus, 0, &spba_res); 21368391ecf4SShengjiu Wang if (!ret) { 21378391ecf4SShengjiu Wang sdma->spba_start_addr = spba_res.start; 21388391ecf4SShengjiu Wang sdma->spba_end_addr = spba_res.end; 21398391ecf4SShengjiu Wang } 21408391ecf4SShengjiu Wang of_node_put(spba_bus); 21419479e17cSShawn Guo } 21429479e17cSShawn Guo 21432b8066c3SSven Van Asbroeck /* 21442b8066c3SSven Van Asbroeck * Because that device tree does not encode ROM script address, 21452b8066c3SSven Van Asbroeck * the RAM script in firmware is mandatory for device tree 21462b8066c3SSven Van Asbroeck * probe, otherwise it fails. 21472b8066c3SSven Van Asbroeck */ 21482b8066c3SSven Van Asbroeck ret = of_property_read_string(np, "fsl,sdma-ram-script-name", 21492b8066c3SSven Van Asbroeck &fw_name); 21502b8066c3SSven Van Asbroeck if (ret) { 21512b8066c3SSven Van Asbroeck dev_warn(&pdev->dev, "failed to get firmware name\n"); 21522b8066c3SSven Van Asbroeck } else { 21532b8066c3SSven Van Asbroeck ret = sdma_get_firmware(sdma, fw_name); 21542b8066c3SSven Van Asbroeck if (ret) 21552b8066c3SSven Van Asbroeck dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); 21562b8066c3SSven Van Asbroeck } 21572b8066c3SSven Van Asbroeck 21581ec1e82fSSascha Hauer return 0; 21591ec1e82fSSascha Hauer 21609479e17cSShawn Guo err_register: 21619479e17cSShawn Guo dma_async_device_unregister(&sdma->dma_device); 21621ec1e82fSSascha Hauer err_init: 21631ec1e82fSSascha Hauer kfree(sdma->script_addrs); 2164fb9caf37SArvind Yadav err_irq: 2165fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ahb); 2166fb9caf37SArvind Yadav err_clk: 2167fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ipg); 2168939fd4f0SShawn Guo return ret; 21691ec1e82fSSascha Hauer } 21701ec1e82fSSascha Hauer 21711d1bbd30SMaxin B. John static int sdma_remove(struct platform_device *pdev) 21721ec1e82fSSascha Hauer { 217323e11811SVignesh Raman struct sdma_engine *sdma = platform_get_drvdata(pdev); 2174c12fe497SVignesh Raman int i; 217523e11811SVignesh Raman 21765bb9dbb5SVinod Koul devm_free_irq(&pdev->dev, sdma->irq, sdma); 217723e11811SVignesh Raman dma_async_device_unregister(&sdma->dma_device); 217823e11811SVignesh Raman kfree(sdma->script_addrs); 2179fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ahb); 2180fb9caf37SArvind Yadav clk_unprepare(sdma->clk_ipg); 2181c12fe497SVignesh Raman /* Kill the tasklet */ 2182c12fe497SVignesh Raman for (i = 0; i < MAX_DMA_CHANNELS; i++) { 2183c12fe497SVignesh Raman struct sdma_channel *sdmac = &sdma->channel[i]; 2184c12fe497SVignesh Raman 218557b772b8SRobin Gong tasklet_kill(&sdmac->vc.task); 218657b772b8SRobin Gong sdma_free_chan_resources(&sdmac->vc.chan); 2187c12fe497SVignesh Raman } 218823e11811SVignesh Raman 218923e11811SVignesh Raman platform_set_drvdata(pdev, NULL); 219023e11811SVignesh Raman return 0; 21911ec1e82fSSascha Hauer } 21921ec1e82fSSascha Hauer 21931ec1e82fSSascha Hauer static struct platform_driver sdma_driver = { 21941ec1e82fSSascha Hauer .driver = { 21951ec1e82fSSascha Hauer .name = "imx-sdma", 2196580975d7SShawn Guo .of_match_table = sdma_dt_ids, 21971ec1e82fSSascha Hauer }, 21981d1bbd30SMaxin B. John .remove = sdma_remove, 219923e11811SVignesh Raman .probe = sdma_probe, 22001ec1e82fSSascha Hauer }; 22011ec1e82fSSascha Hauer 220223e11811SVignesh Raman module_platform_driver(sdma_driver); 22031ec1e82fSSascha Hauer 22041ec1e82fSSascha Hauer MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>"); 22051ec1e82fSSascha Hauer MODULE_DESCRIPTION("i.MX SDMA driver"); 2206c0879342SNicolas Chauvet #if IS_ENABLED(CONFIG_SOC_IMX6Q) 2207c0879342SNicolas Chauvet MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin"); 2208c0879342SNicolas Chauvet #endif 2209c0879342SNicolas Chauvet #if IS_ENABLED(CONFIG_SOC_IMX7D) 2210c0879342SNicolas Chauvet MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin"); 2211c0879342SNicolas Chauvet #endif 22121ec1e82fSSascha Hauer MODULE_LICENSE("GPL"); 2213