72 	void (*intctrl)(struct spi_imx_data *spi_imx, int enable);
73 	int (*prepare_message)(struct spi_imx_data *spi_imx, struct spi_message *msg);
74 	int (*prepare_transfer)(struct spi_imx_data *spi_imx, struct spi_device *spi);
75 	void (*trigger)(struct spi_imx_data *spi_imx);
76 	int (*rx_available)(struct spi_imx_data *spi_imx);
77 	void (*reset)(struct spi_imx_data *spi_imx);
78 	void (*setup_wml)(struct spi_imx_data *spi_imx);
79 	void (*disable)(struct spi_imx_data *spi_imx);
109 	void (*tx)(struct spi_imx_data *spi_imx);
110 	void (*rx)(struct spi_imx_data *spi_imx);
152 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)		\
154 	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);	\
156 	if (spi_imx->rx_buf) {						\
157 		*(type *)spi_imx->rx_buf = val;				\
158 		spi_imx->rx_buf += sizeof(type);			\
161 	spi_imx->remainder -= sizeof(type);				\
165 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)		\
169 	if (spi_imx->tx_buf) {						\
170 		val = *(type *)spi_imx->tx_buf;				\
171 		spi_imx->tx_buf += sizeof(type);			\
174 	spi_imx->count -= sizeof(type);					\
176 	writel(val, spi_imx->base + MXC_CSPITXDATA);			\
236 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_can_dma()  local
244 	if (spi_imx->target_mode)  in spi_imx_can_dma()
247 	if (transfer->len < spi_imx->devtype_data->fifo_size)  in spi_imx_can_dma()
250 	spi_imx->dynamic_burst = 0;  in spi_imx_can_dma()
307 static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)  in spi_imx_buf_rx_swap_u32()  argument
309 	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);  in spi_imx_buf_rx_swap_u32()
311 	if (spi_imx->rx_buf) {  in spi_imx_buf_rx_swap_u32()
315 		bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);  in spi_imx_buf_rx_swap_u32()
321 		*(u32 *)spi_imx->rx_buf = val;  in spi_imx_buf_rx_swap_u32()
322 		spi_imx->rx_buf += sizeof(u32);  in spi_imx_buf_rx_swap_u32()
325 	spi_imx->remainder -= sizeof(u32);  in spi_imx_buf_rx_swap_u32()
328 static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)  in spi_imx_buf_rx_swap()  argument
333 	unaligned = spi_imx->remainder % 4;  in spi_imx_buf_rx_swap()
336 		spi_imx_buf_rx_swap_u32(spi_imx);  in spi_imx_buf_rx_swap()
340 	if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {  in spi_imx_buf_rx_swap()
341 		spi_imx_buf_rx_u16(spi_imx);  in spi_imx_buf_rx_swap()
345 	val = readl(spi_imx->base + MXC_CSPIRXDATA);  in spi_imx_buf_rx_swap()
348 		if (spi_imx->rx_buf) {  in spi_imx_buf_rx_swap()
349 			*(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;  in spi_imx_buf_rx_swap()
350 			spi_imx->rx_buf++;  in spi_imx_buf_rx_swap()
352 		spi_imx->remainder--;  in spi_imx_buf_rx_swap()
356 static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)  in spi_imx_buf_tx_swap_u32()  argument
363 	if (spi_imx->tx_buf) {  in spi_imx_buf_tx_swap_u32()
364 		val = *(u32 *)spi_imx->tx_buf;  in spi_imx_buf_tx_swap_u32()
365 		spi_imx->tx_buf += sizeof(u32);  in spi_imx_buf_tx_swap_u32()
368 	spi_imx->count -= sizeof(u32);  in spi_imx_buf_tx_swap_u32()
370 	bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);  in spi_imx_buf_tx_swap_u32()
377 	writel(val, spi_imx->base + MXC_CSPITXDATA);  in spi_imx_buf_tx_swap_u32()
380 static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)  in spi_imx_buf_tx_swap()  argument
385 	unaligned = spi_imx->count % 4;  in spi_imx_buf_tx_swap()
388 		spi_imx_buf_tx_swap_u32(spi_imx);  in spi_imx_buf_tx_swap()
392 	if (spi_imx_bytes_per_word(spi_imx->bits_per_word) == 2) {  in spi_imx_buf_tx_swap()
393 		spi_imx_buf_tx_u16(spi_imx);  in spi_imx_buf_tx_swap()
398 		if (spi_imx->tx_buf) {  in spi_imx_buf_tx_swap()
399 			val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);  in spi_imx_buf_tx_swap()
400 			spi_imx->tx_buf++;  in spi_imx_buf_tx_swap()
402 		spi_imx->count--;  in spi_imx_buf_tx_swap()
405 	writel(val, spi_imx->base + MXC_CSPITXDATA);  in spi_imx_buf_tx_swap()
408 static void mx53_ecspi_rx_target(struct spi_imx_data *spi_imx)  in mx53_ecspi_rx_target()  argument
410 	u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));  in mx53_ecspi_rx_target()
412 	if (spi_imx->rx_buf) {  in mx53_ecspi_rx_target()
413 		int n_bytes = spi_imx->target_burst % sizeof(val);  in mx53_ecspi_rx_target()
418 		memcpy(spi_imx->rx_buf,  in mx53_ecspi_rx_target()
421 		spi_imx->rx_buf += n_bytes;  in mx53_ecspi_rx_target()
422 		spi_imx->target_burst -= n_bytes;  in mx53_ecspi_rx_target()
425 	spi_imx->remainder -= sizeof(u32);  in mx53_ecspi_rx_target()
428 static void mx53_ecspi_tx_target(struct spi_imx_data *spi_imx)  in mx53_ecspi_tx_target()  argument
431 	int n_bytes = spi_imx->count % sizeof(val);  in mx53_ecspi_tx_target()
436 	if (spi_imx->tx_buf) {  in mx53_ecspi_tx_target()
438 		       spi_imx->tx_buf, n_bytes);  in mx53_ecspi_tx_target()
440 		spi_imx->tx_buf += n_bytes;  in mx53_ecspi_tx_target()
443 	spi_imx->count -= n_bytes;  in mx53_ecspi_tx_target()
445 	writel(val, spi_imx->base + MXC_CSPITXDATA);  in mx53_ecspi_tx_target()
449 static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,  in mx51_ecspi_clkdiv()  argument
457 	unsigned int fin = spi_imx->spi_clk;  in mx51_ecspi_clkdiv()
469 		dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",  in mx51_ecspi_clkdiv()
476 	dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",  in mx51_ecspi_clkdiv()
486 static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx51_ecspi_intctrl()  argument
499 	writel(val, spi_imx->base + MX51_ECSPI_INT);  in mx51_ecspi_intctrl()
502 static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)  in mx51_ecspi_trigger()  argument
506 	reg = readl(spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
508 	writel(reg, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
511 static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)  in mx51_ecspi_disable()  argument
515 	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_disable()
517 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_disable()
527 static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,  in mx51_ecspi_prepare_message()  argument
535 	u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_ecspi_prepare_message()
540 	if (spi_imx->target_mode)  in mx51_ecspi_prepare_message()
549 		ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);  in mx51_ecspi_prepare_message()
558 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_prepare_message()
560 	testreg = readl(spi_imx->base + MX51_ECSPI_TESTREG);  in mx51_ecspi_prepare_message()
565 	writel(testreg, spi_imx->base + MX51_ECSPI_TESTREG);  in mx51_ecspi_prepare_message()
572 	if (spi_imx->target_mode && is_imx53_ecspi(spi_imx))  in mx51_ecspi_prepare_message()
598 	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_ecspi_prepare_message()
632 static void mx51_configure_cpha(struct spi_imx_data *spi_imx,  in mx51_configure_cpha()  argument
636 	bool flip_cpha = (spi->mode & SPI_RX_CPHA_FLIP) && spi_imx->rx_only;  in mx51_configure_cpha()
637 	u32 cfg = readl(spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_configure_cpha()
648 	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_configure_cpha()
651 static int mx51_ecspi_prepare_transfer(struct spi_imx_data *spi_imx,  in mx51_ecspi_prepare_transfer()  argument
654 	u32 ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_prepare_transfer()
659 	if (spi_imx->target_mode && is_imx53_ecspi(spi_imx))  in mx51_ecspi_prepare_transfer()
660 		ctrl |= (spi_imx->target_burst * 8 - 1)  in mx51_ecspi_prepare_transfer()
663 		ctrl |= (spi_imx->bits_per_word - 1)  in mx51_ecspi_prepare_transfer()
670 	ctrl |= mx51_ecspi_clkdiv(spi_imx, spi_imx->spi_bus_clk, &clk);  in mx51_ecspi_prepare_transfer()
671 	spi_imx->spi_bus_clk = clk;  in mx51_ecspi_prepare_transfer()
673 	mx51_configure_cpha(spi_imx, spi);  in mx51_ecspi_prepare_transfer()
679 	if (spi_imx->usedma && spi_imx->devtype_data->tx_glitch_fixed)  in mx51_ecspi_prepare_transfer()
684 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_prepare_transfer()
689 static void mx51_setup_wml(struct spi_imx_data *spi_imx)  in mx51_setup_wml()  argument
693 	if (spi_imx->devtype_data->tx_glitch_fixed)  in mx51_setup_wml()
694 		tx_wml = spi_imx->wml;  in mx51_setup_wml()
699 	writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) |  in mx51_setup_wml()
701 		MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |  in mx51_setup_wml()
703 		MX51_ECSPI_DMA_RXTDEN, spi_imx->base + MX51_ECSPI_DMA);  in mx51_setup_wml()
706 static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)  in mx51_ecspi_rx_available()  argument
708 	return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;  in mx51_ecspi_rx_available()
711 static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)  in mx51_ecspi_reset()  argument
714 	while (mx51_ecspi_rx_available(spi_imx))  in mx51_ecspi_reset()
715 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx51_ecspi_reset()
749 static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx31_intctrl()  argument
758 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx31_intctrl()
761 static void mx31_trigger(struct spi_imx_data *spi_imx)  in mx31_trigger()  argument
765 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
767 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
770 static int mx31_prepare_message(struct spi_imx_data *spi_imx,  in mx31_prepare_message()  argument
776 static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,  in mx31_prepare_transfer()  argument
782 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<  in mx31_prepare_transfer()
784 	spi_imx->spi_bus_clk = clk;  in mx31_prepare_transfer()
786 	if (is_imx35_cspi(spi_imx)) {  in mx31_prepare_transfer()
787 		reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;  in mx31_prepare_transfer()
790 		reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;  in mx31_prepare_transfer()
801 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :  in mx31_prepare_transfer()
804 	if (spi_imx->usedma)  in mx31_prepare_transfer()
807 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_prepare_transfer()
809 	reg = readl(spi_imx->base + MX31_CSPI_TESTREG);  in mx31_prepare_transfer()
814 	writel(reg, spi_imx->base + MX31_CSPI_TESTREG);  in mx31_prepare_transfer()
816 	if (spi_imx->usedma) {  in mx31_prepare_transfer()
822 			spi_imx->base + MX31_CSPI_DMAREG);  in mx31_prepare_transfer()
828 static int mx31_rx_available(struct spi_imx_data *spi_imx)  in mx31_rx_available()  argument
830 	return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;  in mx31_rx_available()
833 static void mx31_reset(struct spi_imx_data *spi_imx)  in mx31_reset()  argument
836 	while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)  in mx31_reset()
837 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx31_reset()
853 static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx21_intctrl()  argument
862 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx21_intctrl()
865 static void mx21_trigger(struct spi_imx_data *spi_imx)  in mx21_trigger()  argument
869 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
871 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
874 static int mx21_prepare_message(struct spi_imx_data *spi_imx,  in mx21_prepare_message()  argument
880 static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,  in mx21_prepare_transfer()  argument
884 	unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;  in mx21_prepare_transfer()
887 	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, spi_imx->spi_bus_clk, max, &clk)  in mx21_prepare_transfer()
889 	spi_imx->spi_bus_clk = clk;  in mx21_prepare_transfer()
891 	reg |= spi_imx->bits_per_word - 1;  in mx21_prepare_transfer()
902 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_prepare_transfer()
907 static int mx21_rx_available(struct spi_imx_data *spi_imx)  in mx21_rx_available()  argument
909 	return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;  in mx21_rx_available()
912 static void mx21_reset(struct spi_imx_data *spi_imx)  in mx21_reset()  argument
914 	writel(1, spi_imx->base + MXC_RESET);  in mx21_reset()
928 static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx1_intctrl()  argument
937 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx1_intctrl()
940 static void mx1_trigger(struct spi_imx_data *spi_imx)  in mx1_trigger()  argument
944 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
946 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
949 static int mx1_prepare_message(struct spi_imx_data *spi_imx,  in mx1_prepare_message()  argument
955 static int mx1_prepare_transfer(struct spi_imx_data *spi_imx,  in mx1_prepare_transfer()  argument
961 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, spi_imx->spi_bus_clk, &clk) <<  in mx1_prepare_transfer()
963 	spi_imx->spi_bus_clk = clk;  in mx1_prepare_transfer()
965 	reg |= spi_imx->bits_per_word - 1;  in mx1_prepare_transfer()
972 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_prepare_transfer()
977 static int mx1_rx_available(struct spi_imx_data *spi_imx)  in mx1_rx_available()  argument
979 	return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;  in mx1_rx_available()
982 static void mx1_reset(struct spi_imx_data *spi_imx)  in mx1_reset()  argument
984 	writel(1, spi_imx->base + MXC_RESET);  in mx1_reset()
1119 static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)  in spi_imx_set_burst_len()  argument
1123 	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);  in spi_imx_set_burst_len()
1126 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in spi_imx_set_burst_len()
1129 static void spi_imx_push(struct spi_imx_data *spi_imx)  in spi_imx_push()  argument
1138 	if (!spi_imx->remainder) {  in spi_imx_push()
1139 		if (spi_imx->dynamic_burst) {  in spi_imx_push()
1142 			burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;  in spi_imx_push()
1147 			spi_imx_set_burst_len(spi_imx, burst_len * 8);  in spi_imx_push()
1149 			spi_imx->remainder = burst_len;  in spi_imx_push()
1151 			spi_imx->remainder = spi_imx_bytes_per_word(spi_imx->bits_per_word);  in spi_imx_push()
1155 	while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {  in spi_imx_push()
1156 		if (!spi_imx->count)  in spi_imx_push()
1158 		if (spi_imx->dynamic_burst &&  in spi_imx_push()
1159 		    spi_imx->txfifo >= DIV_ROUND_UP(spi_imx->remainder, 4))  in spi_imx_push()
1161 		spi_imx->tx(spi_imx);  in spi_imx_push()
1162 		spi_imx->txfifo++;  in spi_imx_push()
1165 	if (!spi_imx->target_mode)  in spi_imx_push()
1166 		spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_push()
1171 	struct spi_imx_data *spi_imx = dev_id;  in spi_imx_isr()  local
1173 	while (spi_imx->txfifo &&  in spi_imx_isr()
1174 	       spi_imx->devtype_data->rx_available(spi_imx)) {  in spi_imx_isr()
1175 		spi_imx->rx(spi_imx);  in spi_imx_isr()
1176 		spi_imx->txfifo--;  in spi_imx_isr()
1179 	if (spi_imx->count) {  in spi_imx_isr()
1180 		spi_imx_push(spi_imx);  in spi_imx_isr()
1184 	if (spi_imx->txfifo) {  in spi_imx_isr()
1188 		spi_imx->devtype_data->intctrl(  in spi_imx_isr()
1189 				spi_imx, MXC_INT_RR);  in spi_imx_isr()
1193 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_isr()
1194 	complete(&spi_imx->xfer_done);  in spi_imx_isr()
1204 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_dma_configure()  local
1206 	switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {  in spi_imx_dma_configure()
1221 	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;  in spi_imx_dma_configure()
1223 	tx.dst_maxburst = spi_imx->wml;  in spi_imx_dma_configure()
1226 		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);  in spi_imx_dma_configure()
1231 	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;  in spi_imx_dma_configure()
1233 	rx.src_maxburst = spi_imx->wml;  in spi_imx_dma_configure()
1236 		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);  in spi_imx_dma_configure()
1246 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);  in spi_imx_setupxfer()  local
1257 		spi_imx->spi_bus_clk = spi->max_speed_hz;  in spi_imx_setupxfer()
1259 		spi_imx->spi_bus_clk = t->speed_hz;  in spi_imx_setupxfer()
1261 	spi_imx->bits_per_word = t->bits_per_word;  in spi_imx_setupxfer()
1262 	spi_imx->count = t->len;  in spi_imx_setupxfer()
1269 	if (spi_imx->devtype_data->dynamic_burst && !spi_imx->target_mode &&  in spi_imx_setupxfer()
1271 	    (spi_imx->bits_per_word == 8 ||  in spi_imx_setupxfer()
1272 	    spi_imx->bits_per_word == 16 ||  in spi_imx_setupxfer()
1273 	    spi_imx->bits_per_word == 32)) {  in spi_imx_setupxfer()
1275 		spi_imx->rx = spi_imx_buf_rx_swap;  in spi_imx_setupxfer()
1276 		spi_imx->tx = spi_imx_buf_tx_swap;  in spi_imx_setupxfer()
1277 		spi_imx->dynamic_burst = 1;  in spi_imx_setupxfer()
1280 		if (spi_imx->bits_per_word <= 8) {  in spi_imx_setupxfer()
1281 			spi_imx->rx = spi_imx_buf_rx_u8;  in spi_imx_setupxfer()
1282 			spi_imx->tx = spi_imx_buf_tx_u8;  in spi_imx_setupxfer()
1283 		} else if (spi_imx->bits_per_word <= 16) {  in spi_imx_setupxfer()
1284 			spi_imx->rx = spi_imx_buf_rx_u16;  in spi_imx_setupxfer()
1285 			spi_imx->tx = spi_imx_buf_tx_u16;  in spi_imx_setupxfer()
1287 			spi_imx->rx = spi_imx_buf_rx_u32;  in spi_imx_setupxfer()
1288 			spi_imx->tx = spi_imx_buf_tx_u32;  in spi_imx_setupxfer()
1290 		spi_imx->dynamic_burst = 0;  in spi_imx_setupxfer()
1293 	if (spi_imx_can_dma(spi_imx->controller, spi, t))  in spi_imx_setupxfer()
1294 		spi_imx->usedma = true;  in spi_imx_setupxfer()
1296 		spi_imx->usedma = false;  in spi_imx_setupxfer()
1298 	spi_imx->rx_only = ((t->tx_buf == NULL)  in spi_imx_setupxfer()
1301 	if (is_imx53_ecspi(spi_imx) && spi_imx->target_mode) {  in spi_imx_setupxfer()
1302 		spi_imx->rx = mx53_ecspi_rx_target;  in spi_imx_setupxfer()
1303 		spi_imx->tx = mx53_ecspi_tx_target;  in spi_imx_setupxfer()
1304 		spi_imx->target_burst = t->len;  in spi_imx_setupxfer()
1307 	spi_imx->devtype_data->prepare_transfer(spi_imx, spi);  in spi_imx_setupxfer()
1312 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)  in spi_imx_sdma_exit()  argument
1314 	struct spi_controller *controller = spi_imx->controller;  in spi_imx_sdma_exit()
1327 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,  in spi_imx_sdma_init()  argument
1332 	spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;  in spi_imx_sdma_init()
1352 	init_completion(&spi_imx->dma_rx_completion);  in spi_imx_sdma_init()
1353 	init_completion(&spi_imx->dma_tx_completion);  in spi_imx_sdma_init()
1356 	spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |  in spi_imx_sdma_init()
1361 	spi_imx_sdma_exit(spi_imx);  in spi_imx_sdma_init()
1367 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_rx_callback()  local
1369 	complete(&spi_imx->dma_rx_completion);  in spi_imx_dma_rx_callback()
1374 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_tx_callback()  local
1376 	complete(&spi_imx->dma_tx_completion);  in spi_imx_dma_tx_callback()
1379 static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)  in spi_imx_calculate_timeout()  argument
1384 	timeout = (8 + 4) * size / spi_imx->spi_bus_clk;  in spi_imx_calculate_timeout()
1393 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,  in spi_imx_dma_transfer()  argument
1399 	struct spi_controller *controller = spi_imx->controller;  in spi_imx_dma_transfer()
1407 	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {  in spi_imx_dma_transfer()
1415 	spi_imx->wml =  i;  in spi_imx_dma_transfer()
1421 	if (!spi_imx->devtype_data->setup_wml) {  in spi_imx_dma_transfer()
1422 		dev_err(spi_imx->dev, "No setup_wml()?\n");  in spi_imx_dma_transfer()
1426 	spi_imx->devtype_data->setup_wml(spi_imx);  in spi_imx_dma_transfer()
1441 	desc_rx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
1443 	reinit_completion(&spi_imx->dma_rx_completion);  in spi_imx_dma_transfer()
1456 	desc_tx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
1458 	reinit_completion(&spi_imx->dma_tx_completion);  in spi_imx_dma_transfer()
1461 	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);  in spi_imx_dma_transfer()
1464 	time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,  in spi_imx_dma_transfer()
1467 		dev_err(spi_imx->dev, "I/O Error in DMA TX\n");  in spi_imx_dma_transfer()
1473 	time_left = wait_for_completion_timeout(&spi_imx->dma_rx_completion,  in spi_imx_dma_transfer()
1477 		spi_imx->devtype_data->reset(spi_imx);  in spi_imx_dma_transfer()
1492 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);  in spi_imx_pio_transfer()  local
1496 	spi_imx->tx_buf = transfer->tx_buf;  in spi_imx_pio_transfer()
1497 	spi_imx->rx_buf = transfer->rx_buf;  in spi_imx_pio_transfer()
1498 	spi_imx->count = transfer->len;  in spi_imx_pio_transfer()
1499 	spi_imx->txfifo = 0;  in spi_imx_pio_transfer()
1500 	spi_imx->remainder = 0;  in spi_imx_pio_transfer()
1502 	reinit_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer()
1504 	spi_imx_push(spi_imx);  in spi_imx_pio_transfer()
1506 	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);  in spi_imx_pio_transfer()
1508 	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);  in spi_imx_pio_transfer()
1510 	time_left = wait_for_completion_timeout(&spi_imx->xfer_done,  in spi_imx_pio_transfer()
1514 		spi_imx->devtype_data->reset(spi_imx);  in spi_imx_pio_transfer()
1524 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);  in spi_imx_poll_transfer()  local
1527 	spi_imx->tx_buf = transfer->tx_buf;  in spi_imx_poll_transfer()
1528 	spi_imx->rx_buf = transfer->rx_buf;  in spi_imx_poll_transfer()
1529 	spi_imx->count = transfer->len;  in spi_imx_poll_transfer()
1530 	spi_imx->txfifo = 0;  in spi_imx_poll_transfer()
1531 	spi_imx->remainder = 0;  in spi_imx_poll_transfer()
1537 	spi_imx_push(spi_imx);  in spi_imx_poll_transfer()
1539 	timeout = spi_imx_calculate_timeout(spi_imx, transfer->len) + jiffies;  in spi_imx_poll_transfer()
1540 	while (spi_imx->txfifo) {  in spi_imx_poll_transfer()
1542 		while (spi_imx->txfifo &&  in spi_imx_poll_transfer()
1543 		       spi_imx->devtype_data->rx_available(spi_imx)) {  in spi_imx_poll_transfer()
1544 			spi_imx->rx(spi_imx);  in spi_imx_poll_transfer()
1545 			spi_imx->txfifo--;  in spi_imx_poll_transfer()
1549 		if (spi_imx->count) {  in spi_imx_poll_transfer()
1550 			spi_imx_push(spi_imx);  in spi_imx_poll_transfer()
1554 		if (spi_imx->txfifo &&  in spi_imx_poll_transfer()
1572 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);  in spi_imx_pio_transfer_target()  local
1575 	if (is_imx53_ecspi(spi_imx) &&  in spi_imx_pio_transfer_target()
1582 	spi_imx->tx_buf = transfer->tx_buf;  in spi_imx_pio_transfer_target()
1583 	spi_imx->rx_buf = transfer->rx_buf;  in spi_imx_pio_transfer_target()
1584 	spi_imx->count = transfer->len;  in spi_imx_pio_transfer_target()
1585 	spi_imx->txfifo = 0;  in spi_imx_pio_transfer_target()
1586 	spi_imx->remainder = 0;  in spi_imx_pio_transfer_target()
1588 	reinit_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer_target()
1589 	spi_imx->target_aborted = false;  in spi_imx_pio_transfer_target()
1591 	spi_imx_push(spi_imx);  in spi_imx_pio_transfer_target()
1593 	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);  in spi_imx_pio_transfer_target()
1595 	if (wait_for_completion_interruptible(&spi_imx->xfer_done) ||  in spi_imx_pio_transfer_target()
1596 	    spi_imx->target_aborted) {  in spi_imx_pio_transfer_target()
1607 	if (spi_imx->devtype_data->disable)  in spi_imx_pio_transfer_target()
1608 		spi_imx->devtype_data->disable(spi_imx);  in spi_imx_pio_transfer_target()
1617 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(spi->controller);  in spi_imx_transfer_one()  local
1621 	transfer->effective_speed_hz = spi_imx->spi_bus_clk;  in spi_imx_transfer_one()
1624 	while (spi_imx->devtype_data->rx_available(spi_imx))  in spi_imx_transfer_one()
1625 		readl(spi_imx->base + MXC_CSPIRXDATA);  in spi_imx_transfer_one()
1627 	if (spi_imx->target_mode)  in spi_imx_transfer_one()
1635 	if (spi_imx->usedma)  in spi_imx_transfer_one()
1636 		return spi_imx_dma_transfer(spi_imx, transfer);  in spi_imx_transfer_one()
1662 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_prepare_message()  local
1665 	ret = pm_runtime_resume_and_get(spi_imx->dev);  in spi_imx_prepare_message()
1667 		dev_err(spi_imx->dev, "failed to enable clock\n");  in spi_imx_prepare_message()
1671 	ret = spi_imx->devtype_data->prepare_message(spi_imx, msg);  in spi_imx_prepare_message()
1673 		pm_runtime_mark_last_busy(spi_imx->dev);  in spi_imx_prepare_message()
1674 		pm_runtime_put_autosuspend(spi_imx->dev);  in spi_imx_prepare_message()
1683 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_unprepare_message()  local
1685 	pm_runtime_mark_last_busy(spi_imx->dev);  in spi_imx_unprepare_message()
1686 	pm_runtime_put_autosuspend(spi_imx->dev);  in spi_imx_unprepare_message()
1692 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_target_abort()  local
1694 	spi_imx->target_aborted = true;  in spi_imx_target_abort()
1695 	complete(&spi_imx->xfer_done);  in spi_imx_target_abort()
1704 	struct spi_imx_data *spi_imx;  in spi_imx_probe()  local
1735 	spi_imx = spi_controller_get_devdata(controller);  in spi_imx_probe()
1736 	spi_imx->controller = controller;  in spi_imx_probe()
1737 	spi_imx->dev = &pdev->dev;  in spi_imx_probe()
1738 	spi_imx->target_mode = target_mode;  in spi_imx_probe()
1740 	spi_imx->devtype_data = devtype_data;  in spi_imx_probe()
1761 	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||  in spi_imx_probe()
1762 	    is_imx53_ecspi(spi_imx))  in spi_imx_probe()
1765 	if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx))  in spi_imx_probe()
1768 	if (is_imx51_ecspi(spi_imx) &&  in spi_imx_probe()
1777 	if (is_imx51_ecspi(spi_imx) || is_imx53_ecspi(spi_imx)) {  in spi_imx_probe()
1782 	spi_imx->spi_drctl = spi_drctl;  in spi_imx_probe()
1784 	init_completion(&spi_imx->xfer_done);  in spi_imx_probe()
1786 	spi_imx->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);  in spi_imx_probe()
1787 	if (IS_ERR(spi_imx->base)) {  in spi_imx_probe()
1788 		ret = PTR_ERR(spi_imx->base);  in spi_imx_probe()
1791 	spi_imx->base_phys = res->start;  in spi_imx_probe()
1800 			       dev_name(&pdev->dev), spi_imx);  in spi_imx_probe()
1806 	spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");  in spi_imx_probe()
1807 	if (IS_ERR(spi_imx->clk_ipg)) {  in spi_imx_probe()
1808 		ret = PTR_ERR(spi_imx->clk_ipg);  in spi_imx_probe()
1812 	spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");  in spi_imx_probe()
1813 	if (IS_ERR(spi_imx->clk_per)) {  in spi_imx_probe()
1814 		ret = PTR_ERR(spi_imx->clk_per);  in spi_imx_probe()
1818 	ret = clk_prepare_enable(spi_imx->clk_per);  in spi_imx_probe()
1822 	ret = clk_prepare_enable(spi_imx->clk_ipg);  in spi_imx_probe()
1826 	pm_runtime_set_autosuspend_delay(spi_imx->dev, MXC_RPM_TIMEOUT);  in spi_imx_probe()
1827 	pm_runtime_use_autosuspend(spi_imx->dev);  in spi_imx_probe()
1828 	pm_runtime_get_noresume(spi_imx->dev);  in spi_imx_probe()
1829 	pm_runtime_set_active(spi_imx->dev);  in spi_imx_probe()
1830 	pm_runtime_enable(spi_imx->dev);  in spi_imx_probe()
1832 	spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);  in spi_imx_probe()
1837 	if (spi_imx->devtype_data->has_dmamode) {  in spi_imx_probe()
1838 		ret = spi_imx_sdma_init(&pdev->dev, spi_imx, controller);  in spi_imx_probe()
1847 	spi_imx->devtype_data->reset(spi_imx);  in spi_imx_probe()
1849 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_probe()
1858 	pm_runtime_mark_last_busy(spi_imx->dev);  in spi_imx_probe()
1859 	pm_runtime_put_autosuspend(spi_imx->dev);  in spi_imx_probe()
1864 	if (spi_imx->devtype_data->has_dmamode)  in spi_imx_probe()
1865 		spi_imx_sdma_exit(spi_imx);  in spi_imx_probe()
1867 	pm_runtime_dont_use_autosuspend(spi_imx->dev);  in spi_imx_probe()
1868 	pm_runtime_disable(spi_imx->dev);  in spi_imx_probe()
1871 	clk_disable_unprepare(spi_imx->clk_ipg);  in spi_imx_probe()
1873 	clk_disable_unprepare(spi_imx->clk_per);  in spi_imx_probe()
1883 	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);  in spi_imx_remove()  local
1888 	ret = pm_runtime_get_sync(spi_imx->dev);  in spi_imx_remove()
1890 		writel(0, spi_imx->base + MXC_CSPICTRL);  in spi_imx_remove()
1892 		dev_warn(spi_imx->dev, "failed to enable clock, skip hw disable\n");  in spi_imx_remove()
1894 	pm_runtime_dont_use_autosuspend(spi_imx->dev);  in spi_imx_remove()
1895 	pm_runtime_put_sync(spi_imx->dev);  in spi_imx_remove()
1896 	pm_runtime_disable(spi_imx->dev);  in spi_imx_remove()
1898 	spi_imx_sdma_exit(spi_imx);  in spi_imx_remove()
1904 	struct spi_imx_data *spi_imx;  in spi_imx_runtime_resume()  local
1907 	spi_imx = spi_controller_get_devdata(controller);  in spi_imx_runtime_resume()
1909 	ret = clk_prepare_enable(spi_imx->clk_per);  in spi_imx_runtime_resume()
1913 	ret = clk_prepare_enable(spi_imx->clk_ipg);  in spi_imx_runtime_resume()
1915 		clk_disable_unprepare(spi_imx->clk_per);  in spi_imx_runtime_resume()
1925 	struct spi_imx_data *spi_imx;  in spi_imx_runtime_suspend()  local
1927 	spi_imx = spi_controller_get_devdata(controller);  in spi_imx_runtime_suspend()
1929 	clk_disable_unprepare(spi_imx->clk_per);  in spi_imx_runtime_suspend()
1930 	clk_disable_unprepare(spi_imx->clk_ipg);  in spi_imx_runtime_suspend()