1f707079dSWolfram Sang // SPDX-License-Identifier: GPL-2.0 22a68ea78SSimon Horman /* 32a68ea78SSimon Horman * DMA support for Internal DMAC with SDHI SD/SDIO controller 42a68ea78SSimon Horman * 5f49bdcdeSWolfram Sang * Copyright (C) 2016-19 Renesas Electronics Corporation 62a68ea78SSimon Horman * Copyright (C) 2016-17 Horms Solutions, Simon Horman 7f49bdcdeSWolfram Sang * Copyright (C) 2018-19 Sang Engineering, Wolfram Sang 82a68ea78SSimon Horman */ 92a68ea78SSimon Horman 100cbc94daSWolfram Sang #include <linux/bitops.h> 112a68ea78SSimon Horman #include <linux/device.h> 122a68ea78SSimon Horman #include <linux/dma-mapping.h> 132a68ea78SSimon Horman #include <linux/io-64-nonatomic-hi-lo.h> 142a68ea78SSimon Horman #include <linux/mfd/tmio.h> 152a68ea78SSimon Horman #include <linux/mmc/host.h> 162a68ea78SSimon Horman #include <linux/mod_devicetable.h> 172a68ea78SSimon Horman #include <linux/module.h> 1871b7597cSYoshihiro Shimoda #include <linux/of_device.h> 192a68ea78SSimon Horman #include <linux/pagemap.h> 202a68ea78SSimon Horman #include <linux/scatterlist.h> 21cd09780fSSimon Horman #include <linux/sys_soc.h> 222a68ea78SSimon Horman 232a68ea78SSimon Horman #include "renesas_sdhi.h" 242a68ea78SSimon Horman #include "tmio_mmc.h" 252a68ea78SSimon Horman 262a68ea78SSimon Horman #define DM_CM_DTRAN_MODE 0x820 272a68ea78SSimon Horman #define DM_CM_DTRAN_CTRL 0x828 282a68ea78SSimon Horman #define DM_CM_RST 0x830 292a68ea78SSimon Horman #define DM_CM_INFO1 0x840 302a68ea78SSimon Horman #define DM_CM_INFO1_MASK 0x848 312a68ea78SSimon Horman #define DM_CM_INFO2 0x850 322a68ea78SSimon Horman #define DM_CM_INFO2_MASK 0x858 332a68ea78SSimon Horman #define DM_DTRAN_ADDR 0x880 342a68ea78SSimon Horman 352a68ea78SSimon Horman /* DM_CM_DTRAN_MODE */ 362a68ea78SSimon Horman #define DTRAN_MODE_CH_NUM_CH0 0 /* "downstream" = for write commands */ 37c1ec8f86SSergei Shtylyov #define DTRAN_MODE_CH_NUM_CH1 BIT(16) /* "upstream" = for read commands */ 38c1ec8f86SSergei Shtylyov #define DTRAN_MODE_BUS_WIDTH (BIT(5) | BIT(4)) 399706b472SChris Brandt #define DTRAN_MODE_ADDR_MODE BIT(0) /* 1 = Increment address, 0 = Fixed */ 402a68ea78SSimon Horman 412a68ea78SSimon Horman /* DM_CM_DTRAN_CTRL */ 422a68ea78SSimon Horman #define DTRAN_CTRL_DM_START BIT(0) 432a68ea78SSimon Horman 442a68ea78SSimon Horman /* DM_CM_RST */ 452a68ea78SSimon Horman #define RST_DTRANRST1 BIT(9) 462a68ea78SSimon Horman #define RST_DTRANRST0 BIT(8) 479faf870eSSergei Shtylyov #define RST_RESERVED_BITS GENMASK_ULL(31, 0) 482a68ea78SSimon Horman 492a68ea78SSimon Horman /* DM_CM_INFO1 and DM_CM_INFO1_MASK */ 502a68ea78SSimon Horman #define INFO1_CLEAR 0 51d2332f88SSergei Shtylyov #define INFO1_MASK_CLEAR GENMASK_ULL(31, 0) 522a68ea78SSimon Horman #define INFO1_DTRANEND1 BIT(17) 532a68ea78SSimon Horman #define INFO1_DTRANEND0 BIT(16) 542a68ea78SSimon Horman 552a68ea78SSimon Horman /* DM_CM_INFO2 and DM_CM_INFO2_MASK */ 56d2332f88SSergei Shtylyov #define INFO2_MASK_CLEAR GENMASK_ULL(31, 0) 572a68ea78SSimon Horman #define INFO2_DTRANERR1 BIT(17) 582a68ea78SSimon Horman #define INFO2_DTRANERR0 BIT(16) 592a68ea78SSimon Horman 6069e7d76aSYoshihiro Shimoda enum renesas_sdhi_dma_cookie { 6169e7d76aSYoshihiro Shimoda COOKIE_UNMAPPED, 6269e7d76aSYoshihiro Shimoda COOKIE_PRE_MAPPED, 6369e7d76aSYoshihiro Shimoda COOKIE_MAPPED, 6469e7d76aSYoshihiro Shimoda }; 6569e7d76aSYoshihiro Shimoda 662a68ea78SSimon Horman /* 672a68ea78SSimon Horman * Specification of this driver: 682a68ea78SSimon Horman * - host->chan_{rx,tx} will be used as a flag of enabling/disabling the dma 692a68ea78SSimon Horman * - Since this SDHI DMAC register set has 16 but 32-bit width, we 702a68ea78SSimon Horman * need a custom accessor. 712a68ea78SSimon Horman */ 722a68ea78SSimon Horman 730cbc94daSWolfram Sang static unsigned long global_flags; 740cbc94daSWolfram Sang /* 750cbc94daSWolfram Sang * Workaround for avoiding to use RX DMAC by multiple channels. 760cbc94daSWolfram Sang * On R-Car H3 ES1.* and M3-W ES1.0, when multiple SDHI channels use 770cbc94daSWolfram Sang * RX DMAC simultaneously, sometimes hundreds of bytes data are not 780cbc94daSWolfram Sang * stored into the system memory even if the DMAC interrupt happened. 790cbc94daSWolfram Sang * So, this driver then uses one RX DMAC channel only. 800cbc94daSWolfram Sang */ 81bcfa7f15SWolfram Sang #define SDHI_INTERNAL_DMAC_RX_IN_USE 0 820cbc94daSWolfram Sang 832a68ea78SSimon Horman /* Definitions for sampling clocks */ 842a68ea78SSimon Horman static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = { 852a68ea78SSimon Horman { 862a68ea78SSimon Horman .clk_rate = 0, 872a68ea78SSimon Horman .tap = 0x00000300, 88c1a49782SWolfram Sang .tap_hs400_4tap = 0x00000100, 892a68ea78SSimon Horman }, 902a68ea78SSimon Horman }; 912a68ea78SSimon Horman 9271b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data of_data_rza2 = { 939706b472SChris Brandt .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 949706b472SChris Brandt TMIO_MMC_HAVE_CBSY, 959706b472SChris Brandt .tmio_ocr_mask = MMC_VDD_32_33, 969706b472SChris Brandt .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 9787e985aeSWolfram Sang MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, 989706b472SChris Brandt .bus_shift = 2, 999706b472SChris Brandt .scc_offset = 0 - 0x1000, 1009706b472SChris Brandt .taps = rcar_gen3_scc_taps, 1019706b472SChris Brandt .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), 1022a55c1eaSWolfram Sang /* DMAC can handle 32bit blk count but only 1 segment */ 1032a55c1eaSWolfram Sang .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, 1049706b472SChris Brandt .max_segs = 1, 1059706b472SChris Brandt }; 1069706b472SChris Brandt 10771b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data of_data_rcar_gen3 = { 1082ad1db05SMasahiro Yamada .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 1092ad1db05SMasahiro Yamada TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, 1102a68ea78SSimon Horman .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 11187e985aeSWolfram Sang MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, 112c7d9eccbSYoshihiro Shimoda .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT | MMC_CAP2_MERGE_CAPABLE, 1132a68ea78SSimon Horman .bus_shift = 2, 1142a68ea78SSimon Horman .scc_offset = 0x1000, 1152a68ea78SSimon Horman .taps = rcar_gen3_scc_taps, 1162a68ea78SSimon Horman .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), 1172a55c1eaSWolfram Sang /* DMAC can handle 32bit blk count but only 1 segment */ 1182a55c1eaSWolfram Sang .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, 1192a68ea78SSimon Horman .max_segs = 1, 120627151b4SWolfram Sang .sdhi_flags = SDHI_FLAG_NEED_CLKH_FALLBACK, 121627151b4SWolfram Sang }; 122627151b4SWolfram Sang 1236de9727aSWolfram Sang static const struct renesas_sdhi_of_data of_data_rcar_gen3_no_sdh_fallback = { 124627151b4SWolfram Sang .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | 125627151b4SWolfram Sang TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, 126627151b4SWolfram Sang .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | 127627151b4SWolfram Sang MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, 128627151b4SWolfram Sang .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT | MMC_CAP2_MERGE_CAPABLE, 129627151b4SWolfram Sang .bus_shift = 2, 130627151b4SWolfram Sang .scc_offset = 0x1000, 131627151b4SWolfram Sang .taps = rcar_gen3_scc_taps, 132627151b4SWolfram Sang .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), 133627151b4SWolfram Sang /* DMAC can handle 32bit blk count but only 1 segment */ 134627151b4SWolfram Sang .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, 135627151b4SWolfram Sang .max_segs = 1, 1362a68ea78SSimon Horman }; 1372a68ea78SSimon Horman 13871b7597cSYoshihiro Shimoda static const u8 r8a7796_es13_calib_table[2][SDHI_CALIB_TABLE_MAX] = { 13971b7597cSYoshihiro Shimoda { 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 15, 14071b7597cSYoshihiro Shimoda 16, 16, 16, 16, 16, 16, 17, 18, 18, 19, 20, 21, 22, 23, 24, 25 }, 14171b7597cSYoshihiro Shimoda { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 11, 14271b7597cSYoshihiro Shimoda 12, 17, 18, 18, 18, 18, 18, 18, 18, 19, 20, 21, 22, 23, 25, 25 } 14371b7597cSYoshihiro Shimoda }; 14471b7597cSYoshihiro Shimoda 14571b7597cSYoshihiro Shimoda static const u8 r8a77965_calib_table[2][SDHI_CALIB_TABLE_MAX] = { 14671b7597cSYoshihiro Shimoda { 1, 2, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15, 15, 16, 14771b7597cSYoshihiro Shimoda 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 26, 27, 28, 29, 30, 31 }, 14871b7597cSYoshihiro Shimoda { 2, 3, 4, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 14971b7597cSYoshihiro Shimoda 17, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 31, 31, 31 } 15071b7597cSYoshihiro Shimoda }; 15171b7597cSYoshihiro Shimoda 15271b7597cSYoshihiro Shimoda static const u8 r8a77990_calib_table[2][SDHI_CALIB_TABLE_MAX] = { 15371b7597cSYoshihiro Shimoda { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15471b7597cSYoshihiro Shimoda 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 15571b7597cSYoshihiro Shimoda { 0, 0, 0, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 8, 9, 10, 15671b7597cSYoshihiro Shimoda 11, 12, 13, 15, 16, 17, 17, 18, 18, 19, 20, 22, 24, 25, 26, 26 } 15771b7597cSYoshihiro Shimoda }; 15871b7597cSYoshihiro Shimoda 15971b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_4tap_nohs400 = { 16071b7597cSYoshihiro Shimoda .hs400_disabled = true, 16171b7597cSYoshihiro Shimoda .hs400_4taps = true, 16271b7597cSYoshihiro Shimoda }; 16371b7597cSYoshihiro Shimoda 164bcfa7f15SWolfram Sang static const struct renesas_sdhi_quirks sdhi_quirks_4tap_nohs400_one_rx = { 165bcfa7f15SWolfram Sang .hs400_disabled = true, 166bcfa7f15SWolfram Sang .hs400_4taps = true, 167bcfa7f15SWolfram Sang .dma_one_rx_only = true, 168bcfa7f15SWolfram Sang }; 169bcfa7f15SWolfram Sang 17071b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_4tap = { 17171b7597cSYoshihiro Shimoda .hs400_4taps = true, 17271b7597cSYoshihiro Shimoda .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), 17371b7597cSYoshihiro Shimoda }; 17471b7597cSYoshihiro Shimoda 17571b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_nohs400 = { 17671b7597cSYoshihiro Shimoda .hs400_disabled = true, 17771b7597cSYoshihiro Shimoda }; 17871b7597cSYoshihiro Shimoda 179c0a43968SWolfram Sang static const struct renesas_sdhi_quirks sdhi_quirks_fixed_addr = { 180c0a43968SWolfram Sang .fixed_addr_mode = true, 181c0a43968SWolfram Sang }; 182c0a43968SWolfram Sang 18371b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_bad_taps1357 = { 18471b7597cSYoshihiro Shimoda .hs400_bad_taps = BIT(1) | BIT(3) | BIT(5) | BIT(7), 18571b7597cSYoshihiro Shimoda }; 18671b7597cSYoshihiro Shimoda 18771b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_bad_taps2367 = { 18871b7597cSYoshihiro Shimoda .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), 18971b7597cSYoshihiro Shimoda }; 19071b7597cSYoshihiro Shimoda 19171b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_r8a7796_es13 = { 19271b7597cSYoshihiro Shimoda .hs400_4taps = true, 19371b7597cSYoshihiro Shimoda .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), 19471b7597cSYoshihiro Shimoda .hs400_calib_table = r8a7796_es13_calib_table, 19571b7597cSYoshihiro Shimoda }; 19671b7597cSYoshihiro Shimoda 19771b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_r8a77965 = { 19871b7597cSYoshihiro Shimoda .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), 19971b7597cSYoshihiro Shimoda .hs400_calib_table = r8a77965_calib_table, 20071b7597cSYoshihiro Shimoda }; 20171b7597cSYoshihiro Shimoda 20271b7597cSYoshihiro Shimoda static const struct renesas_sdhi_quirks sdhi_quirks_r8a77990 = { 20371b7597cSYoshihiro Shimoda .hs400_calib_table = r8a77990_calib_table, 20471b7597cSYoshihiro Shimoda }; 20571b7597cSYoshihiro Shimoda 20671b7597cSYoshihiro Shimoda /* 20771b7597cSYoshihiro Shimoda * Note for r8a7796 / r8a774a1: we can't distinguish ES1.1 and 1.2 as of now. 20871b7597cSYoshihiro Shimoda * So, we want to treat them equally and only have a match for ES1.2 to enforce 20971b7597cSYoshihiro Shimoda * this if there ever will be a way to distinguish ES1.2. 21071b7597cSYoshihiro Shimoda */ 21171b7597cSYoshihiro Shimoda static const struct soc_device_attribute sdhi_quirks_match[] = { 21271b7597cSYoshihiro Shimoda { .soc_id = "r8a774a1", .revision = "ES1.[012]", .data = &sdhi_quirks_4tap_nohs400 }, 213bcfa7f15SWolfram Sang { .soc_id = "r8a7795", .revision = "ES1.*", .data = &sdhi_quirks_4tap_nohs400_one_rx }, 21471b7597cSYoshihiro Shimoda { .soc_id = "r8a7795", .revision = "ES2.0", .data = &sdhi_quirks_4tap }, 215bcfa7f15SWolfram Sang { .soc_id = "r8a7796", .revision = "ES1.0", .data = &sdhi_quirks_4tap_nohs400_one_rx }, 216bcfa7f15SWolfram Sang { .soc_id = "r8a7796", .revision = "ES1.[12]", .data = &sdhi_quirks_4tap_nohs400 }, 21771b7597cSYoshihiro Shimoda { .soc_id = "r8a7796", .revision = "ES1.*", .data = &sdhi_quirks_r8a7796_es13 }, 2180ffd498dSGeert Uytterhoeven { /* Sentinel. */ } 21971b7597cSYoshihiro Shimoda }; 22071b7597cSYoshihiro Shimoda 22171b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data_with_quirks of_r8a7795_compatible = { 22271b7597cSYoshihiro Shimoda .of_data = &of_data_rcar_gen3, 22371b7597cSYoshihiro Shimoda .quirks = &sdhi_quirks_bad_taps2367, 22471b7597cSYoshihiro Shimoda }; 22571b7597cSYoshihiro Shimoda 22671b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data_with_quirks of_r8a77961_compatible = { 22771b7597cSYoshihiro Shimoda .of_data = &of_data_rcar_gen3, 22871b7597cSYoshihiro Shimoda .quirks = &sdhi_quirks_bad_taps1357, 22971b7597cSYoshihiro Shimoda }; 23071b7597cSYoshihiro Shimoda 23171b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data_with_quirks of_r8a77965_compatible = { 23271b7597cSYoshihiro Shimoda .of_data = &of_data_rcar_gen3, 23371b7597cSYoshihiro Shimoda .quirks = &sdhi_quirks_r8a77965, 23471b7597cSYoshihiro Shimoda }; 23571b7597cSYoshihiro Shimoda 236627151b4SWolfram Sang static const struct renesas_sdhi_of_data_with_quirks of_r8a77970_compatible = { 2376de9727aSWolfram Sang .of_data = &of_data_rcar_gen3_no_sdh_fallback, 238*fc1fdbd9SWolfram Sang .quirks = &sdhi_quirks_nohs400, 239627151b4SWolfram Sang }; 240627151b4SWolfram Sang 24171b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data_with_quirks of_r8a77990_compatible = { 24271b7597cSYoshihiro Shimoda .of_data = &of_data_rcar_gen3, 24371b7597cSYoshihiro Shimoda .quirks = &sdhi_quirks_r8a77990, 24471b7597cSYoshihiro Shimoda }; 24571b7597cSYoshihiro Shimoda 24671b7597cSYoshihiro Shimoda static const struct renesas_sdhi_of_data_with_quirks of_rcar_gen3_compatible = { 24771b7597cSYoshihiro Shimoda .of_data = &of_data_rcar_gen3, 24871b7597cSYoshihiro Shimoda }; 24971b7597cSYoshihiro Shimoda 2506af8dd53SWolfram Sang static const struct renesas_sdhi_of_data_with_quirks of_rcar_gen3_nohs400_compatible = { 2516af8dd53SWolfram Sang .of_data = &of_data_rcar_gen3, 2526af8dd53SWolfram Sang .quirks = &sdhi_quirks_nohs400, 2536af8dd53SWolfram Sang }; 2546af8dd53SWolfram Sang 255c0a43968SWolfram Sang static const struct renesas_sdhi_of_data_with_quirks of_rza2_compatible = { 256c0a43968SWolfram Sang .of_data = &of_data_rza2, 257c0a43968SWolfram Sang .quirks = &sdhi_quirks_fixed_addr, 258c0a43968SWolfram Sang }; 259c0a43968SWolfram Sang 2602a68ea78SSimon Horman static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = { 2619706b472SChris Brandt { .compatible = "renesas,sdhi-r7s9210", .data = &of_rza2_compatible, }, 26260ab43baSFabrizio Castro { .compatible = "renesas,sdhi-mmc-r8a77470", .data = &of_rcar_gen3_compatible, }, 26371b7597cSYoshihiro Shimoda { .compatible = "renesas,sdhi-r8a7795", .data = &of_r8a7795_compatible, }, 2642c907f05SNiklas Söderlund { .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_gen3_compatible, }, 26571b7597cSYoshihiro Shimoda { .compatible = "renesas,sdhi-r8a77961", .data = &of_r8a77961_compatible, }, 26671b7597cSYoshihiro Shimoda { .compatible = "renesas,sdhi-r8a77965", .data = &of_r8a77965_compatible, }, 267627151b4SWolfram Sang { .compatible = "renesas,sdhi-r8a77970", .data = &of_r8a77970_compatible, }, 2686af8dd53SWolfram Sang { .compatible = "renesas,sdhi-r8a77980", .data = &of_rcar_gen3_nohs400_compatible, }, 26971b7597cSYoshihiro Shimoda { .compatible = "renesas,sdhi-r8a77990", .data = &of_r8a77990_compatible, }, 2706af8dd53SWolfram Sang { .compatible = "renesas,sdhi-r8a77995", .data = &of_rcar_gen3_nohs400_compatible, }, 271d6dc425aSSimon Horman { .compatible = "renesas,rcar-gen3-sdhi", .data = &of_rcar_gen3_compatible, }, 2722a68ea78SSimon Horman {}, 2732a68ea78SSimon Horman }; 2742a68ea78SSimon Horman MODULE_DEVICE_TABLE(of, renesas_sdhi_internal_dmac_of_match); 2752a68ea78SSimon Horman 2762a68ea78SSimon Horman static void 2772a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(struct tmio_mmc_host *host, 2782a68ea78SSimon Horman int addr, u64 val) 2792a68ea78SSimon Horman { 2802a68ea78SSimon Horman writeq(val, host->ctl + addr); 2812a68ea78SSimon Horman } 2822a68ea78SSimon Horman 2832a68ea78SSimon Horman static void 2842a68ea78SSimon Horman renesas_sdhi_internal_dmac_enable_dma(struct tmio_mmc_host *host, bool enable) 2852a68ea78SSimon Horman { 286058db286SMasahiro Yamada struct renesas_sdhi *priv = host_to_priv(host); 287058db286SMasahiro Yamada 2882a68ea78SSimon Horman if (!host->chan_tx || !host->chan_rx) 2892a68ea78SSimon Horman return; 2902a68ea78SSimon Horman 2912a68ea78SSimon Horman if (!enable) 2922a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1, 2932a68ea78SSimon Horman INFO1_CLEAR); 2942a68ea78SSimon Horman 295058db286SMasahiro Yamada if (priv->dma_priv.enable) 296058db286SMasahiro Yamada priv->dma_priv.enable(host, enable); 2972a68ea78SSimon Horman } 2982a68ea78SSimon Horman 2992a68ea78SSimon Horman static void 300ed9ab884SWolfram Sang renesas_sdhi_internal_dmac_abort_dma(struct tmio_mmc_host *host) 301ed9ab884SWolfram Sang { 3022a68ea78SSimon Horman u64 val = RST_DTRANRST1 | RST_DTRANRST0; 3032a68ea78SSimon Horman 3042a68ea78SSimon Horman renesas_sdhi_internal_dmac_enable_dma(host, false); 3052a68ea78SSimon Horman 3062a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST, 3072a68ea78SSimon Horman RST_RESERVED_BITS & ~val); 3082a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST, 3092a68ea78SSimon Horman RST_RESERVED_BITS | val); 3102a68ea78SSimon Horman 3110cbc94daSWolfram Sang clear_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags); 3120cbc94daSWolfram Sang 3132a68ea78SSimon Horman renesas_sdhi_internal_dmac_enable_dma(host, true); 3142a68ea78SSimon Horman } 3152a68ea78SSimon Horman 3162a68ea78SSimon Horman static void 317ed9ab884SWolfram Sang renesas_sdhi_internal_dmac_dataend_dma(struct tmio_mmc_host *host) 318ed9ab884SWolfram Sang { 31990d95106SMasahiro Yamada struct renesas_sdhi *priv = host_to_priv(host); 32090d95106SMasahiro Yamada 32190d95106SMasahiro Yamada tasklet_schedule(&priv->dma_priv.dma_complete); 3222a68ea78SSimon Horman } 3232a68ea78SSimon Horman 32469e7d76aSYoshihiro Shimoda /* 32569e7d76aSYoshihiro Shimoda * renesas_sdhi_internal_dmac_map() will be called with two difference 32669e7d76aSYoshihiro Shimoda * sg pointers in two mmc_data by .pre_req(), but tmio host can have a single 32769e7d76aSYoshihiro Shimoda * sg_ptr only. So, renesas_sdhi_internal_dmac_{un}map() should use a sg 32869e7d76aSYoshihiro Shimoda * pointer in a mmc_data instead of host->sg_ptr. 32969e7d76aSYoshihiro Shimoda */ 33069e7d76aSYoshihiro Shimoda static void 33169e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_unmap(struct tmio_mmc_host *host, 33269e7d76aSYoshihiro Shimoda struct mmc_data *data, 33369e7d76aSYoshihiro Shimoda enum renesas_sdhi_dma_cookie cookie) 33469e7d76aSYoshihiro Shimoda { 33569e7d76aSYoshihiro Shimoda bool unmap = cookie == COOKIE_UNMAPPED ? (data->host_cookie != cookie) : 33669e7d76aSYoshihiro Shimoda (data->host_cookie == cookie); 33769e7d76aSYoshihiro Shimoda 33869e7d76aSYoshihiro Shimoda if (unmap) { 33969e7d76aSYoshihiro Shimoda dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len, 34069e7d76aSYoshihiro Shimoda mmc_get_dma_dir(data)); 34169e7d76aSYoshihiro Shimoda data->host_cookie = COOKIE_UNMAPPED; 34269e7d76aSYoshihiro Shimoda } 34369e7d76aSYoshihiro Shimoda } 34469e7d76aSYoshihiro Shimoda 34569e7d76aSYoshihiro Shimoda static bool 34669e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_map(struct tmio_mmc_host *host, 34769e7d76aSYoshihiro Shimoda struct mmc_data *data, 34869e7d76aSYoshihiro Shimoda enum renesas_sdhi_dma_cookie cookie) 34969e7d76aSYoshihiro Shimoda { 35069e7d76aSYoshihiro Shimoda if (data->host_cookie == COOKIE_PRE_MAPPED) 35169e7d76aSYoshihiro Shimoda return true; 35269e7d76aSYoshihiro Shimoda 35369e7d76aSYoshihiro Shimoda if (!dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, 35469e7d76aSYoshihiro Shimoda mmc_get_dma_dir(data))) 35569e7d76aSYoshihiro Shimoda return false; 35669e7d76aSYoshihiro Shimoda 35769e7d76aSYoshihiro Shimoda data->host_cookie = cookie; 35869e7d76aSYoshihiro Shimoda 35969e7d76aSYoshihiro Shimoda /* This DMAC cannot handle if buffer is not 128-bytes alignment */ 36069e7d76aSYoshihiro Shimoda if (!IS_ALIGNED(sg_dma_address(data->sg), 128)) { 36169e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_unmap(host, data, cookie); 36269e7d76aSYoshihiro Shimoda return false; 36369e7d76aSYoshihiro Shimoda } 36469e7d76aSYoshihiro Shimoda 36569e7d76aSYoshihiro Shimoda return true; 36669e7d76aSYoshihiro Shimoda } 36769e7d76aSYoshihiro Shimoda 3682a68ea78SSimon Horman static void 3692a68ea78SSimon Horman renesas_sdhi_internal_dmac_start_dma(struct tmio_mmc_host *host, 3702a68ea78SSimon Horman struct mmc_data *data) 3712a68ea78SSimon Horman { 372c0a43968SWolfram Sang struct renesas_sdhi *priv = host_to_priv(host); 3732a68ea78SSimon Horman struct scatterlist *sg = host->sg_ptr; 3749706b472SChris Brandt u32 dtran_mode = DTRAN_MODE_BUS_WIDTH; 3759706b472SChris Brandt 376139bbdbaSGeert Uytterhoeven if (!(priv->quirks && priv->quirks->fixed_addr_mode)) 3779706b472SChris Brandt dtran_mode |= DTRAN_MODE_ADDR_MODE; 3782a68ea78SSimon Horman 37969e7d76aSYoshihiro Shimoda if (!renesas_sdhi_internal_dmac_map(host, data, COOKIE_MAPPED)) 38048e1dc10SYoshihiro Shimoda goto force_pio; 3812a68ea78SSimon Horman 3822a68ea78SSimon Horman if (data->flags & MMC_DATA_READ) { 3832a68ea78SSimon Horman dtran_mode |= DTRAN_MODE_CH_NUM_CH1; 384139bbdbaSGeert Uytterhoeven if (priv->quirks && priv->quirks->dma_one_rx_only && 3850cbc94daSWolfram Sang test_and_set_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags)) 386fe6e0494SYoshihiro Shimoda goto force_pio_with_unmap; 3872a68ea78SSimon Horman } else { 3882a68ea78SSimon Horman dtran_mode |= DTRAN_MODE_CH_NUM_CH0; 3892a68ea78SSimon Horman } 3902a68ea78SSimon Horman 3912a68ea78SSimon Horman renesas_sdhi_internal_dmac_enable_dma(host, true); 3922a68ea78SSimon Horman 3932a68ea78SSimon Horman /* set dma parameters */ 3942a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_MODE, 3952a68ea78SSimon Horman dtran_mode); 3962a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_DTRAN_ADDR, 397a028b435SNiklas Söderlund sg_dma_address(sg)); 39848e1dc10SYoshihiro Shimoda 399d3dd5db0SMasahiro Yamada host->dma_on = true; 400d3dd5db0SMasahiro Yamada 40148e1dc10SYoshihiro Shimoda return; 40248e1dc10SYoshihiro Shimoda 403fe6e0494SYoshihiro Shimoda force_pio_with_unmap: 40469e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_unmap(host, data, COOKIE_UNMAPPED); 405fe6e0494SYoshihiro Shimoda 40648e1dc10SYoshihiro Shimoda force_pio: 40748e1dc10SYoshihiro Shimoda renesas_sdhi_internal_dmac_enable_dma(host, false); 4082a68ea78SSimon Horman } 4092a68ea78SSimon Horman 4102a68ea78SSimon Horman static void renesas_sdhi_internal_dmac_issue_tasklet_fn(unsigned long arg) 4112a68ea78SSimon Horman { 4122a68ea78SSimon Horman struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg; 4132a68ea78SSimon Horman 4142a68ea78SSimon Horman tmio_mmc_enable_mmc_irqs(host, TMIO_STAT_DATAEND); 4152a68ea78SSimon Horman 4162a68ea78SSimon Horman /* start the DMAC */ 4172a68ea78SSimon Horman renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_CTRL, 4182a68ea78SSimon Horman DTRAN_CTRL_DM_START); 4192a68ea78SSimon Horman } 4202a68ea78SSimon Horman 4212b26e34eSYoshihiro Shimoda static bool renesas_sdhi_internal_dmac_complete(struct tmio_mmc_host *host) 4222a68ea78SSimon Horman { 4232a68ea78SSimon Horman enum dma_data_direction dir; 4242a68ea78SSimon Horman 42558a91d96SYoshihiro Shimoda if (!host->dma_on) 42658a91d96SYoshihiro Shimoda return false; 42758a91d96SYoshihiro Shimoda 4282a68ea78SSimon Horman if (!host->data) 4292b26e34eSYoshihiro Shimoda return false; 4302a68ea78SSimon Horman 4312a68ea78SSimon Horman if (host->data->flags & MMC_DATA_READ) 4322a68ea78SSimon Horman dir = DMA_FROM_DEVICE; 4332a68ea78SSimon Horman else 4342a68ea78SSimon Horman dir = DMA_TO_DEVICE; 4352a68ea78SSimon Horman 4362a68ea78SSimon Horman renesas_sdhi_internal_dmac_enable_dma(host, false); 43769e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_unmap(host, host->data, COOKIE_MAPPED); 4382a68ea78SSimon Horman 4390cbc94daSWolfram Sang if (dir == DMA_FROM_DEVICE) 4400cbc94daSWolfram Sang clear_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags); 4410cbc94daSWolfram Sang 44258a91d96SYoshihiro Shimoda host->dma_on = false; 44358a91d96SYoshihiro Shimoda 4442b26e34eSYoshihiro Shimoda return true; 4452b26e34eSYoshihiro Shimoda } 4462b26e34eSYoshihiro Shimoda 4472b26e34eSYoshihiro Shimoda static void renesas_sdhi_internal_dmac_complete_tasklet_fn(unsigned long arg) 4482b26e34eSYoshihiro Shimoda { 4492b26e34eSYoshihiro Shimoda struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg; 4502b26e34eSYoshihiro Shimoda 4512b26e34eSYoshihiro Shimoda spin_lock_irq(&host->lock); 4522b26e34eSYoshihiro Shimoda if (!renesas_sdhi_internal_dmac_complete(host)) 4532b26e34eSYoshihiro Shimoda goto out; 4542b26e34eSYoshihiro Shimoda 4552a68ea78SSimon Horman tmio_mmc_do_data_irq(host); 4562a68ea78SSimon Horman out: 4572a68ea78SSimon Horman spin_unlock_irq(&host->lock); 4582a68ea78SSimon Horman } 4592a68ea78SSimon Horman 46058a91d96SYoshihiro Shimoda static void renesas_sdhi_internal_dmac_end_dma(struct tmio_mmc_host *host) 46158a91d96SYoshihiro Shimoda { 46258a91d96SYoshihiro Shimoda if (host->data) 46358a91d96SYoshihiro Shimoda renesas_sdhi_internal_dmac_complete(host); 46458a91d96SYoshihiro Shimoda } 46558a91d96SYoshihiro Shimoda 46669e7d76aSYoshihiro Shimoda static void renesas_sdhi_internal_dmac_post_req(struct mmc_host *mmc, 46769e7d76aSYoshihiro Shimoda struct mmc_request *mrq, 46869e7d76aSYoshihiro Shimoda int err) 46969e7d76aSYoshihiro Shimoda { 47069e7d76aSYoshihiro Shimoda struct tmio_mmc_host *host = mmc_priv(mmc); 47169e7d76aSYoshihiro Shimoda struct mmc_data *data = mrq->data; 47269e7d76aSYoshihiro Shimoda 47369e7d76aSYoshihiro Shimoda if (!data) 47469e7d76aSYoshihiro Shimoda return; 47569e7d76aSYoshihiro Shimoda 47669e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_unmap(host, data, COOKIE_UNMAPPED); 47769e7d76aSYoshihiro Shimoda } 47869e7d76aSYoshihiro Shimoda 47969e7d76aSYoshihiro Shimoda static void renesas_sdhi_internal_dmac_pre_req(struct mmc_host *mmc, 48069e7d76aSYoshihiro Shimoda struct mmc_request *mrq) 48169e7d76aSYoshihiro Shimoda { 48269e7d76aSYoshihiro Shimoda struct tmio_mmc_host *host = mmc_priv(mmc); 48369e7d76aSYoshihiro Shimoda struct mmc_data *data = mrq->data; 48469e7d76aSYoshihiro Shimoda 48569e7d76aSYoshihiro Shimoda if (!data) 48669e7d76aSYoshihiro Shimoda return; 48769e7d76aSYoshihiro Shimoda 48869e7d76aSYoshihiro Shimoda data->host_cookie = COOKIE_UNMAPPED; 48969e7d76aSYoshihiro Shimoda renesas_sdhi_internal_dmac_map(host, data, COOKIE_PRE_MAPPED); 49069e7d76aSYoshihiro Shimoda } 49169e7d76aSYoshihiro Shimoda 4922a68ea78SSimon Horman static void 4932a68ea78SSimon Horman renesas_sdhi_internal_dmac_request_dma(struct tmio_mmc_host *host, 4942a68ea78SSimon Horman struct tmio_mmc_data *pdata) 4952a68ea78SSimon Horman { 49690d95106SMasahiro Yamada struct renesas_sdhi *priv = host_to_priv(host); 49790d95106SMasahiro Yamada 498d2332f88SSergei Shtylyov /* Disable DMAC interrupts, we don't use them */ 499d2332f88SSergei Shtylyov renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1_MASK, 500d2332f88SSergei Shtylyov INFO1_MASK_CLEAR); 501d2332f88SSergei Shtylyov renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO2_MASK, 502d2332f88SSergei Shtylyov INFO2_MASK_CLEAR); 503d2332f88SSergei Shtylyov 5042a68ea78SSimon Horman /* Each value is set to non-zero to assume "enabling" each DMA */ 5052a68ea78SSimon Horman host->chan_rx = host->chan_tx = (void *)0xdeadbeaf; 5062a68ea78SSimon Horman 50790d95106SMasahiro Yamada tasklet_init(&priv->dma_priv.dma_complete, 5082a68ea78SSimon Horman renesas_sdhi_internal_dmac_complete_tasklet_fn, 5092a68ea78SSimon Horman (unsigned long)host); 5102a68ea78SSimon Horman tasklet_init(&host->dma_issue, 5112a68ea78SSimon Horman renesas_sdhi_internal_dmac_issue_tasklet_fn, 5122a68ea78SSimon Horman (unsigned long)host); 51369e7d76aSYoshihiro Shimoda 51469e7d76aSYoshihiro Shimoda /* Add pre_req and post_req */ 51569e7d76aSYoshihiro Shimoda host->ops.pre_req = renesas_sdhi_internal_dmac_pre_req; 51669e7d76aSYoshihiro Shimoda host->ops.post_req = renesas_sdhi_internal_dmac_post_req; 5172a68ea78SSimon Horman } 5182a68ea78SSimon Horman 5192a68ea78SSimon Horman static void 5202a68ea78SSimon Horman renesas_sdhi_internal_dmac_release_dma(struct tmio_mmc_host *host) 5212a68ea78SSimon Horman { 5222a68ea78SSimon Horman /* Each value is set to zero to assume "disabling" each DMA */ 5232a68ea78SSimon Horman host->chan_rx = host->chan_tx = NULL; 5242a68ea78SSimon Horman } 5252a68ea78SSimon Horman 52610154068SJulia Lawall static const struct tmio_mmc_dma_ops renesas_sdhi_internal_dmac_dma_ops = { 5272a68ea78SSimon Horman .start = renesas_sdhi_internal_dmac_start_dma, 5282a68ea78SSimon Horman .enable = renesas_sdhi_internal_dmac_enable_dma, 5292a68ea78SSimon Horman .request = renesas_sdhi_internal_dmac_request_dma, 5302a68ea78SSimon Horman .release = renesas_sdhi_internal_dmac_release_dma, 5312a68ea78SSimon Horman .abort = renesas_sdhi_internal_dmac_abort_dma, 5322a68ea78SSimon Horman .dataend = renesas_sdhi_internal_dmac_dataend_dma, 53358a91d96SYoshihiro Shimoda .end = renesas_sdhi_internal_dmac_end_dma, 5342a68ea78SSimon Horman }; 5352a68ea78SSimon Horman 5362a68ea78SSimon Horman static int renesas_sdhi_internal_dmac_probe(struct platform_device *pdev) 5372a68ea78SSimon Horman { 53871b7597cSYoshihiro Shimoda const struct soc_device_attribute *attr; 53971b7597cSYoshihiro Shimoda const struct renesas_sdhi_of_data_with_quirks *of_data_quirks; 54071b7597cSYoshihiro Shimoda const struct renesas_sdhi_quirks *quirks; 54154541815SNiklas Söderlund struct device *dev = &pdev->dev; 5420cbc94daSWolfram Sang 54371b7597cSYoshihiro Shimoda of_data_quirks = of_device_get_match_data(&pdev->dev); 54471b7597cSYoshihiro Shimoda quirks = of_data_quirks->quirks; 54571b7597cSYoshihiro Shimoda 54671b7597cSYoshihiro Shimoda attr = soc_device_match(sdhi_quirks_match); 54771b7597cSYoshihiro Shimoda if (attr) 54871b7597cSYoshihiro Shimoda quirks = attr->data; 5490cbc94daSWolfram Sang 55054541815SNiklas Söderlund /* value is max of SD_SECCNT. Confirmed by HW engineers */ 55154541815SNiklas Söderlund dma_set_max_seg_size(dev, 0xffffffff); 55254541815SNiklas Söderlund 55371b7597cSYoshihiro Shimoda return renesas_sdhi_probe(pdev, &renesas_sdhi_internal_dmac_dma_ops, 55471b7597cSYoshihiro Shimoda of_data_quirks->of_data, quirks); 5552a68ea78SSimon Horman } 5562a68ea78SSimon Horman 5572a68ea78SSimon Horman static const struct dev_pm_ops renesas_sdhi_internal_dmac_dev_pm_ops = { 5582a68ea78SSimon Horman SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 5592a68ea78SSimon Horman pm_runtime_force_resume) 5602a68ea78SSimon Horman SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend, 5612a68ea78SSimon Horman tmio_mmc_host_runtime_resume, 5622a68ea78SSimon Horman NULL) 5632a68ea78SSimon Horman }; 5642a68ea78SSimon Horman 5652a68ea78SSimon Horman static struct platform_driver renesas_internal_dmac_sdhi_driver = { 5662a68ea78SSimon Horman .driver = { 5672a68ea78SSimon Horman .name = "renesas_sdhi_internal_dmac", 5687320915cSDouglas Anderson .probe_type = PROBE_PREFER_ASYNCHRONOUS, 5692a68ea78SSimon Horman .pm = &renesas_sdhi_internal_dmac_dev_pm_ops, 5702a68ea78SSimon Horman .of_match_table = renesas_sdhi_internal_dmac_of_match, 5712a68ea78SSimon Horman }, 5722a68ea78SSimon Horman .probe = renesas_sdhi_internal_dmac_probe, 5732a68ea78SSimon Horman .remove = renesas_sdhi_remove, 5742a68ea78SSimon Horman }; 5752a68ea78SSimon Horman 5762a68ea78SSimon Horman module_platform_driver(renesas_internal_dmac_sdhi_driver); 5772a68ea78SSimon Horman 5782a68ea78SSimon Horman MODULE_DESCRIPTION("Renesas SDHI driver for internal DMAC"); 5792a68ea78SSimon Horman MODULE_AUTHOR("Yoshihiro Shimoda"); 5802a68ea78SSimon Horman MODULE_LICENSE("GPL v2"); 581