xref: /linux/drivers/mmc/host/rtsx_pci_sdmmc.c (revision a6021aa24f6417416d93318bbfa022ab229c33c8)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Realtek PCI-Express SD/MMC Card Interface driver
3  *
4  * Copyright(c) 2009-2013 Realtek Semiconductor Corp. All rights reserved.
5  *
6  * Author:
7  *   Wei WANG <wei_wang@realsil.com.cn>
8  */
9 
10 #include <linux/pci.h>
11 #include <linux/module.h>
12 #include <linux/slab.h>
13 #include <linux/highmem.h>
14 #include <linux/delay.h>
15 #include <linux/platform_device.h>
16 #include <linux/workqueue.h>
17 #include <linux/mmc/host.h>
18 #include <linux/mmc/mmc.h>
19 #include <linux/mmc/sd.h>
20 #include <linux/mmc/sdio.h>
21 #include <linux/mmc/card.h>
22 #include <linux/rtsx_pci.h>
23 #include <linux/unaligned.h>
24 #include <linux/pm_runtime.h>
25 
26 struct realtek_pci_sdmmc {
27 	struct platform_device	*pdev;
28 	struct rtsx_pcr		*pcr;
29 	struct mmc_host		*mmc;
30 	struct mmc_request	*mrq;
31 #define SDMMC_WORKQ_NAME	"rtsx_pci_sdmmc_workq"
32 
33 	struct work_struct	work;
34 	struct mutex		host_mutex;
35 
36 	u8			ssc_depth;
37 	unsigned int		clock;
38 	bool			vpclk;
39 	bool			double_clk;
40 	bool			eject;
41 	bool			initial_mode;
42 	int			prev_power_state;
43 	int			sg_count;
44 	s32			cookie;
45 	int			cookie_sg_count;
46 	bool			using_cookie;
47 };
48 
49 static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios);
50 
51 static inline struct device *sdmmc_dev(struct realtek_pci_sdmmc *host)
52 {
53 	return &(host->pdev->dev);
54 }
55 
56 static inline void sd_clear_error(struct realtek_pci_sdmmc *host)
57 {
58 	rtsx_pci_write_register(host->pcr, CARD_STOP,
59 			SD_STOP | SD_CLR_ERR, SD_STOP | SD_CLR_ERR);
60 }
61 
62 #ifdef DEBUG
63 static void dump_reg_range(struct realtek_pci_sdmmc *host, u16 start, u16 end)
64 {
65 	u16 len = end - start + 1;
66 	int i;
67 	u8 data[8];
68 
69 	for (i = 0; i < len; i += 8) {
70 		int j;
71 		int n = min(8, len - i);
72 
73 		memset(&data, 0, sizeof(data));
74 		for (j = 0; j < n; j++)
75 			rtsx_pci_read_register(host->pcr, start + i + j,
76 				data + j);
77 		dev_dbg(sdmmc_dev(host), "0x%04X(%d): %8ph\n",
78 			start + i, n, data);
79 	}
80 }
81 
82 static void sd_print_debug_regs(struct realtek_pci_sdmmc *host)
83 {
84 	dump_reg_range(host, 0xFDA0, 0xFDB3);
85 	dump_reg_range(host, 0xFD52, 0xFD69);
86 }
87 #else
88 #define sd_print_debug_regs(host)
89 #endif /* DEBUG */
90 
91 static inline int sd_get_cd_int(struct realtek_pci_sdmmc *host)
92 {
93 	return rtsx_pci_readl(host->pcr, RTSX_BIPR) & SD_EXIST;
94 }
95 
96 static void sd_cmd_set_sd_cmd(struct rtsx_pcr *pcr, struct mmc_command *cmd)
97 {
98 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CMD0, 0xFF,
99 		SD_CMD_START | cmd->opcode);
100 	rtsx_pci_write_be32(pcr, SD_CMD1, cmd->arg);
101 }
102 
103 static void sd_cmd_set_data_len(struct rtsx_pcr *pcr, u16 blocks, u16 blksz)
104 {
105 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_L, 0xFF, blocks);
106 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BLOCK_CNT_H, 0xFF, blocks >> 8);
107 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_L, 0xFF, blksz);
108 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_BYTE_CNT_H, 0xFF, blksz >> 8);
109 }
110 
111 static int sd_response_type(struct mmc_command *cmd)
112 {
113 	switch (mmc_resp_type(cmd)) {
114 	case MMC_RSP_NONE:
115 		return SD_RSP_TYPE_R0;
116 	case MMC_RSP_R1:
117 		return SD_RSP_TYPE_R1;
118 	case MMC_RSP_R1_NO_CRC:
119 		return SD_RSP_TYPE_R1 | SD_NO_CHECK_CRC7;
120 	case MMC_RSP_R1B:
121 		return SD_RSP_TYPE_R1b;
122 	case MMC_RSP_R2:
123 		return SD_RSP_TYPE_R2;
124 	case MMC_RSP_R3:
125 		return SD_RSP_TYPE_R3;
126 	default:
127 		return -EINVAL;
128 	}
129 }
130 
131 static int sd_status_index(int resp_type)
132 {
133 	if (resp_type == SD_RSP_TYPE_R0)
134 		return 0;
135 	else if (resp_type == SD_RSP_TYPE_R2)
136 		return 16;
137 
138 	return 5;
139 }
140 /*
141  * sd_pre_dma_transfer - do dma_map_sg() or using cookie
142  *
143  * @pre: if called in pre_req()
144  * return:
145  *	0 - do dma_map_sg()
146  *	1 - using cookie
147  */
148 static int sd_pre_dma_transfer(struct realtek_pci_sdmmc *host,
149 		struct mmc_data *data, bool pre)
150 {
151 	struct rtsx_pcr *pcr = host->pcr;
152 	int read = data->flags & MMC_DATA_READ;
153 	int count = 0;
154 	int using_cookie = 0;
155 
156 	if (!pre && data->host_cookie && data->host_cookie != host->cookie) {
157 		dev_err(sdmmc_dev(host),
158 			"error: data->host_cookie = %d, host->cookie = %d\n",
159 			data->host_cookie, host->cookie);
160 		data->host_cookie = 0;
161 	}
162 
163 	if (pre || data->host_cookie != host->cookie) {
164 		count = rtsx_pci_dma_map_sg(pcr, data->sg, data->sg_len, read);
165 	} else {
166 		count = host->cookie_sg_count;
167 		using_cookie = 1;
168 	}
169 
170 	if (pre) {
171 		host->cookie_sg_count = count;
172 		if (++host->cookie < 0)
173 			host->cookie = 1;
174 		data->host_cookie = host->cookie;
175 	} else {
176 		host->sg_count = count;
177 	}
178 
179 	return using_cookie;
180 }
181 
182 static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
183 {
184 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
185 	struct mmc_data *data = mrq->data;
186 
187 	if (data->host_cookie) {
188 		dev_err(sdmmc_dev(host),
189 			"error: reset data->host_cookie = %d\n",
190 			data->host_cookie);
191 		data->host_cookie = 0;
192 	}
193 
194 	sd_pre_dma_transfer(host, data, true);
195 	dev_dbg(sdmmc_dev(host), "pre dma sg: %d\n", host->cookie_sg_count);
196 }
197 
198 static void sdmmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
199 		int err)
200 {
201 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
202 	struct rtsx_pcr *pcr = host->pcr;
203 	struct mmc_data *data = mrq->data;
204 	int read = data->flags & MMC_DATA_READ;
205 
206 	rtsx_pci_dma_unmap_sg(pcr, data->sg, data->sg_len, read);
207 	data->host_cookie = 0;
208 }
209 
210 static void sd_send_cmd_get_rsp(struct realtek_pci_sdmmc *host,
211 		struct mmc_command *cmd)
212 {
213 	struct rtsx_pcr *pcr = host->pcr;
214 	u8 cmd_idx = (u8)cmd->opcode;
215 	u32 arg = cmd->arg;
216 	int err = 0;
217 	int timeout = 100;
218 	int i;
219 	u8 *ptr;
220 	int rsp_type;
221 	int stat_idx;
222 	bool clock_toggled = false;
223 
224 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
225 			__func__, cmd_idx, arg);
226 
227 	rsp_type = sd_response_type(cmd);
228 	if (rsp_type < 0)
229 		goto out;
230 
231 	stat_idx = sd_status_index(rsp_type);
232 
233 	if (rsp_type == SD_RSP_TYPE_R1b)
234 		timeout = cmd->busy_timeout ? cmd->busy_timeout : 3000;
235 
236 	if (cmd->opcode == SD_SWITCH_VOLTAGE) {
237 		err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
238 				0xFF, SD_CLK_TOGGLE_EN);
239 		if (err < 0)
240 			goto out;
241 
242 		clock_toggled = true;
243 	}
244 
245 	rtsx_pci_init_cmd(pcr);
246 	sd_cmd_set_sd_cmd(pcr, cmd);
247 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, rsp_type);
248 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
249 			0x01, PINGPONG_BUFFER);
250 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
251 			0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
252 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
253 		     SD_TRANSFER_END | SD_STAT_IDLE,
254 		     SD_TRANSFER_END | SD_STAT_IDLE);
255 
256 	if (rsp_type == SD_RSP_TYPE_R2) {
257 		/* Read data from ping-pong buffer */
258 		for (i = PPBUF_BASE2; i < PPBUF_BASE2 + 16; i++)
259 			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
260 	} else if (rsp_type != SD_RSP_TYPE_R0) {
261 		/* Read data from SD_CMDx registers */
262 		for (i = SD_CMD0; i <= SD_CMD4; i++)
263 			rtsx_pci_add_cmd(pcr, READ_REG_CMD, (u16)i, 0, 0);
264 	}
265 
266 	rtsx_pci_add_cmd(pcr, READ_REG_CMD, SD_STAT1, 0, 0);
267 
268 	err = rtsx_pci_send_cmd(pcr, timeout);
269 	if (err < 0) {
270 		sd_print_debug_regs(host);
271 		sd_clear_error(host);
272 		dev_dbg(sdmmc_dev(host),
273 			"rtsx_pci_send_cmd error (err = %d)\n", err);
274 		goto out;
275 	}
276 
277 	if (rsp_type == SD_RSP_TYPE_R0) {
278 		err = 0;
279 		goto out;
280 	}
281 
282 	/* Eliminate returned value of CHECK_REG_CMD */
283 	ptr = rtsx_pci_get_cmd_data(pcr) + 1;
284 
285 	/* Check (Start,Transmission) bit of Response */
286 	if ((ptr[0] & 0xC0) != 0) {
287 		err = -EILSEQ;
288 		dev_dbg(sdmmc_dev(host), "Invalid response bit\n");
289 		goto out;
290 	}
291 
292 	/* Check CRC7 */
293 	if (!(rsp_type & SD_NO_CHECK_CRC7)) {
294 		if (ptr[stat_idx] & SD_CRC7_ERR) {
295 			err = -EILSEQ;
296 			dev_dbg(sdmmc_dev(host), "CRC7 error\n");
297 			goto out;
298 		}
299 	}
300 
301 	if (rsp_type == SD_RSP_TYPE_R2) {
302 		/*
303 		 * The controller offloads the last byte {CRC-7, end bit 1'b1}
304 		 * of response type R2. Assign dummy CRC, 0, and end bit to the
305 		 * byte(ptr[16], goes into the LSB of resp[3] later).
306 		 */
307 		ptr[16] = 1;
308 
309 		for (i = 0; i < 4; i++) {
310 			cmd->resp[i] = get_unaligned_be32(ptr + 1 + i * 4);
311 			dev_dbg(sdmmc_dev(host), "cmd->resp[%d] = 0x%08x\n",
312 					i, cmd->resp[i]);
313 		}
314 	} else {
315 		cmd->resp[0] = get_unaligned_be32(ptr + 1);
316 		dev_dbg(sdmmc_dev(host), "cmd->resp[0] = 0x%08x\n",
317 				cmd->resp[0]);
318 	}
319 
320 out:
321 	cmd->error = err;
322 
323 	if (err && clock_toggled)
324 		rtsx_pci_write_register(pcr, SD_BUS_STAT,
325 				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
326 }
327 
328 static int sd_read_data(struct realtek_pci_sdmmc *host, struct mmc_command *cmd,
329 	u16 byte_cnt, u8 *buf, int buf_len, int timeout)
330 {
331 	struct rtsx_pcr *pcr = host->pcr;
332 	int err;
333 	u8 trans_mode;
334 
335 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
336 		__func__, cmd->opcode, cmd->arg);
337 
338 	if (!buf)
339 		buf_len = 0;
340 
341 	if (cmd->opcode == MMC_SEND_TUNING_BLOCK)
342 		trans_mode = SD_TM_AUTO_TUNING;
343 	else
344 		trans_mode = SD_TM_NORMAL_READ;
345 
346 	rtsx_pci_init_cmd(pcr);
347 	sd_cmd_set_sd_cmd(pcr, cmd);
348 	sd_cmd_set_data_len(pcr, 1, byte_cnt);
349 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
350 			SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
351 			SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_6);
352 	if (trans_mode != SD_TM_AUTO_TUNING)
353 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
354 				CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
355 
356 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER,
357 			0xFF, trans_mode | SD_TRANSFER_START);
358 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
359 			SD_TRANSFER_END, SD_TRANSFER_END);
360 
361 	err = rtsx_pci_send_cmd(pcr, timeout);
362 	if (err < 0) {
363 		sd_print_debug_regs(host);
364 		dev_dbg(sdmmc_dev(host),
365 			"rtsx_pci_send_cmd fail (err = %d)\n", err);
366 		return err;
367 	}
368 
369 	if (buf && buf_len) {
370 		err = rtsx_pci_read_ppbuf(pcr, buf, buf_len);
371 		if (err < 0) {
372 			dev_dbg(sdmmc_dev(host),
373 				"rtsx_pci_read_ppbuf fail (err = %d)\n", err);
374 			return err;
375 		}
376 	}
377 
378 	return 0;
379 }
380 
381 static int sd_write_data(struct realtek_pci_sdmmc *host,
382 	struct mmc_command *cmd, u16 byte_cnt, u8 *buf, int buf_len,
383 	int timeout)
384 {
385 	struct rtsx_pcr *pcr = host->pcr;
386 	int err;
387 
388 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
389 		__func__, cmd->opcode, cmd->arg);
390 
391 	if (!buf)
392 		buf_len = 0;
393 
394 	sd_send_cmd_get_rsp(host, cmd);
395 	if (cmd->error)
396 		return cmd->error;
397 
398 	if (buf && buf_len) {
399 		err = rtsx_pci_write_ppbuf(pcr, buf, buf_len);
400 		if (err < 0) {
401 			dev_dbg(sdmmc_dev(host),
402 				"rtsx_pci_write_ppbuf fail (err = %d)\n", err);
403 			return err;
404 		}
405 	}
406 
407 	rtsx_pci_init_cmd(pcr);
408 	sd_cmd_set_data_len(pcr, 1, byte_cnt);
409 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF,
410 		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
411 		SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 | SD_RSP_LEN_0);
412 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
413 			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
414 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
415 			SD_TRANSFER_END, SD_TRANSFER_END);
416 
417 	err = rtsx_pci_send_cmd(pcr, timeout);
418 	if (err < 0) {
419 		sd_print_debug_regs(host);
420 		dev_dbg(sdmmc_dev(host),
421 			"rtsx_pci_send_cmd fail (err = %d)\n", err);
422 		return err;
423 	}
424 
425 	return 0;
426 }
427 
428 static int sd_read_long_data(struct realtek_pci_sdmmc *host,
429 	struct mmc_request *mrq)
430 {
431 	struct rtsx_pcr *pcr = host->pcr;
432 	struct mmc_host *mmc = host->mmc;
433 	struct mmc_card *card = mmc->card;
434 	struct mmc_command *cmd = mrq->cmd;
435 	struct mmc_data *data = mrq->data;
436 	int uhs = mmc_card_uhs(card);
437 	u8 cfg2 = 0;
438 	int err;
439 	int resp_type;
440 	size_t data_len = data->blksz * data->blocks;
441 
442 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
443 		__func__, cmd->opcode, cmd->arg);
444 
445 	resp_type = sd_response_type(cmd);
446 	if (resp_type < 0)
447 		return resp_type;
448 
449 	if (!uhs)
450 		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
451 
452 	rtsx_pci_init_cmd(pcr);
453 	sd_cmd_set_sd_cmd(pcr, cmd);
454 	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
455 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
456 			DMA_DONE_INT, DMA_DONE_INT);
457 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
458 		0xFF, (u8)(data_len >> 24));
459 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
460 		0xFF, (u8)(data_len >> 16));
461 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
462 		0xFF, (u8)(data_len >> 8));
463 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
464 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
465 		0x03 | DMA_PACK_SIZE_MASK,
466 		DMA_DIR_FROM_CARD | DMA_EN | DMA_512);
467 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
468 			0x01, RING_BUFFER);
469 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2 | resp_type);
470 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
471 			SD_TRANSFER_START | SD_TM_AUTO_READ_2);
472 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
473 			SD_TRANSFER_END, SD_TRANSFER_END);
474 	rtsx_pci_send_cmd_no_wait(pcr);
475 
476 	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 1, 10000);
477 	if (err < 0) {
478 		sd_print_debug_regs(host);
479 		sd_clear_error(host);
480 		return err;
481 	}
482 
483 	return 0;
484 }
485 
486 static int sd_write_long_data(struct realtek_pci_sdmmc *host,
487 	struct mmc_request *mrq)
488 {
489 	struct rtsx_pcr *pcr = host->pcr;
490 	struct mmc_host *mmc = host->mmc;
491 	struct mmc_card *card = mmc->card;
492 	struct mmc_command *cmd = mrq->cmd;
493 	struct mmc_data *data = mrq->data;
494 	int uhs = mmc_card_uhs(card);
495 	u8 cfg2;
496 	int err;
497 	size_t data_len = data->blksz * data->blocks;
498 
499 	sd_send_cmd_get_rsp(host, cmd);
500 	if (cmd->error)
501 		return cmd->error;
502 
503 	dev_dbg(sdmmc_dev(host), "%s: SD/MMC CMD %d, arg = 0x%08x\n",
504 		__func__, cmd->opcode, cmd->arg);
505 
506 	cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16 |
507 		SD_NO_WAIT_BUSY_END | SD_NO_CHECK_CRC7 | SD_RSP_LEN_0;
508 
509 	if (!uhs)
510 		cfg2 |= SD_NO_CHECK_WAIT_CRC_TO;
511 
512 	rtsx_pci_init_cmd(pcr);
513 	sd_cmd_set_data_len(pcr, data->blocks, data->blksz);
514 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, IRQSTAT0,
515 			DMA_DONE_INT, DMA_DONE_INT);
516 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC3,
517 		0xFF, (u8)(data_len >> 24));
518 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC2,
519 		0xFF, (u8)(data_len >> 16));
520 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC1,
521 		0xFF, (u8)(data_len >> 8));
522 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMATC0, 0xFF, (u8)data_len);
523 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, DMACTL,
524 		0x03 | DMA_PACK_SIZE_MASK,
525 		DMA_DIR_TO_CARD | DMA_EN | DMA_512);
526 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_DATA_SOURCE,
527 			0x01, RING_BUFFER);
528 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG2, 0xFF, cfg2);
529 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_TRANSFER, 0xFF,
530 			SD_TRANSFER_START | SD_TM_AUTO_WRITE_3);
531 	rtsx_pci_add_cmd(pcr, CHECK_REG_CMD, SD_TRANSFER,
532 			SD_TRANSFER_END, SD_TRANSFER_END);
533 	rtsx_pci_send_cmd_no_wait(pcr);
534 	err = rtsx_pci_dma_transfer(pcr, data->sg, host->sg_count, 0, 10000);
535 	if (err < 0) {
536 		sd_clear_error(host);
537 		return err;
538 	}
539 
540 	return 0;
541 }
542 
543 static inline void sd_enable_initial_mode(struct realtek_pci_sdmmc *host)
544 {
545 	rtsx_pci_write_register(host->pcr, SD_CFG1,
546 			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_128);
547 }
548 
549 static inline void sd_disable_initial_mode(struct realtek_pci_sdmmc *host)
550 {
551 	rtsx_pci_write_register(host->pcr, SD_CFG1,
552 			SD_CLK_DIVIDE_MASK, SD_CLK_DIVIDE_0);
553 }
554 
555 static int sd_rw_multi(struct realtek_pci_sdmmc *host, struct mmc_request *mrq)
556 {
557 	struct mmc_data *data = mrq->data;
558 	int err;
559 
560 	if (host->sg_count < 0) {
561 		data->error = host->sg_count;
562 		dev_dbg(sdmmc_dev(host), "%s: sg_count = %d is invalid\n",
563 			__func__, host->sg_count);
564 		return data->error;
565 	}
566 
567 	if (data->flags & MMC_DATA_READ) {
568 		if (host->initial_mode)
569 			sd_disable_initial_mode(host);
570 
571 		err = sd_read_long_data(host, mrq);
572 
573 		if (host->initial_mode)
574 			sd_enable_initial_mode(host);
575 
576 		return err;
577 	}
578 
579 	return sd_write_long_data(host, mrq);
580 }
581 
582 static void sd_normal_rw(struct realtek_pci_sdmmc *host,
583 		struct mmc_request *mrq)
584 {
585 	struct mmc_command *cmd = mrq->cmd;
586 	struct mmc_data *data = mrq->data;
587 	u8 *buf;
588 
589 	buf = kzalloc(data->blksz, GFP_NOIO);
590 	if (!buf) {
591 		cmd->error = -ENOMEM;
592 		return;
593 	}
594 
595 	if (data->flags & MMC_DATA_READ) {
596 		if (host->initial_mode)
597 			sd_disable_initial_mode(host);
598 
599 		cmd->error = sd_read_data(host, cmd, (u16)data->blksz, buf,
600 				data->blksz, 200);
601 
602 		if (host->initial_mode)
603 			sd_enable_initial_mode(host);
604 
605 		sg_copy_from_buffer(data->sg, data->sg_len, buf, data->blksz);
606 	} else {
607 		sg_copy_to_buffer(data->sg, data->sg_len, buf, data->blksz);
608 
609 		cmd->error = sd_write_data(host, cmd, (u16)data->blksz, buf,
610 				data->blksz, 200);
611 	}
612 
613 	kfree(buf);
614 }
615 
616 static int sd_change_phase(struct realtek_pci_sdmmc *host,
617 		u8 sample_point, bool rx)
618 {
619 	struct rtsx_pcr *pcr = host->pcr;
620 	u16 SD_VP_CTL = 0;
621 	dev_dbg(sdmmc_dev(host), "%s(%s): sample_point = %d\n",
622 			__func__, rx ? "RX" : "TX", sample_point);
623 
624 	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, CHANGE_CLK);
625 	if (rx) {
626 		SD_VP_CTL = SD_VPRX_CTL;
627 		rtsx_pci_write_register(pcr, SD_VPRX_CTL,
628 			PHASE_SELECT_MASK, sample_point);
629 	} else {
630 		SD_VP_CTL = SD_VPTX_CTL;
631 		rtsx_pci_write_register(pcr, SD_VPTX_CTL,
632 			PHASE_SELECT_MASK, sample_point);
633 	}
634 	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET, 0);
635 	rtsx_pci_write_register(pcr, SD_VP_CTL, PHASE_NOT_RESET,
636 				PHASE_NOT_RESET);
637 	rtsx_pci_write_register(pcr, CLK_CTL, CHANGE_CLK, 0);
638 	rtsx_pci_write_register(pcr, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
639 
640 	return 0;
641 }
642 
643 static inline u32 test_phase_bit(u32 phase_map, unsigned int bit)
644 {
645 	bit %= RTSX_PHASE_MAX;
646 	return phase_map & (1 << bit);
647 }
648 
649 static int sd_get_phase_len(u32 phase_map, unsigned int start_bit)
650 {
651 	int i;
652 
653 	for (i = 0; i < RTSX_PHASE_MAX; i++) {
654 		if (test_phase_bit(phase_map, start_bit + i) == 0)
655 			return i;
656 	}
657 	return RTSX_PHASE_MAX;
658 }
659 
660 static u8 sd_search_final_phase(struct realtek_pci_sdmmc *host, u32 phase_map)
661 {
662 	int start = 0, len = 0;
663 	int start_final = 0, len_final = 0;
664 	u8 final_phase = 0xFF;
665 
666 	if (phase_map == 0) {
667 		dev_err(sdmmc_dev(host), "phase error: [map:%x]\n", phase_map);
668 		return final_phase;
669 	}
670 
671 	while (start < RTSX_PHASE_MAX) {
672 		len = sd_get_phase_len(phase_map, start);
673 		if (len_final < len) {
674 			start_final = start;
675 			len_final = len;
676 		}
677 		start += len ? len : 1;
678 	}
679 
680 	final_phase = (start_final + len_final / 2) % RTSX_PHASE_MAX;
681 	dev_dbg(sdmmc_dev(host), "phase: [map:%x] [maxlen:%d] [final:%d]\n",
682 		phase_map, len_final, final_phase);
683 
684 	return final_phase;
685 }
686 
687 static void sd_wait_data_idle(struct realtek_pci_sdmmc *host)
688 {
689 	int i;
690 	u8 val = 0;
691 
692 	for (i = 0; i < 100; i++) {
693 		rtsx_pci_read_register(host->pcr, SD_DATA_STATE, &val);
694 		if (val & SD_DATA_IDLE)
695 			return;
696 
697 		udelay(100);
698 	}
699 }
700 
701 static int sd_tuning_rx_cmd(struct realtek_pci_sdmmc *host,
702 		u8 opcode, u8 sample_point)
703 {
704 	int err;
705 	struct mmc_command cmd = {};
706 	struct rtsx_pcr *pcr = host->pcr;
707 
708 	sd_change_phase(host, sample_point, true);
709 
710 	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN,
711 		SD_RSP_80CLK_TIMEOUT_EN);
712 
713 	cmd.opcode = opcode;
714 	err = sd_read_data(host, &cmd, 0x40, NULL, 0, 100);
715 	if (err < 0) {
716 		/* Wait till SD DATA IDLE */
717 		sd_wait_data_idle(host);
718 		sd_clear_error(host);
719 		rtsx_pci_write_register(pcr, SD_CFG3,
720 			SD_RSP_80CLK_TIMEOUT_EN, 0);
721 		return err;
722 	}
723 
724 	rtsx_pci_write_register(pcr, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN, 0);
725 	return 0;
726 }
727 
728 static int sd_tuning_phase(struct realtek_pci_sdmmc *host,
729 		u8 opcode, u32 *phase_map)
730 {
731 	int err, i;
732 	u32 raw_phase_map = 0;
733 
734 	for (i = 0; i < RTSX_PHASE_MAX; i++) {
735 		err = sd_tuning_rx_cmd(host, opcode, (u8)i);
736 		if (err == 0)
737 			raw_phase_map |= 1 << i;
738 	}
739 
740 	if (phase_map)
741 		*phase_map = raw_phase_map;
742 
743 	return 0;
744 }
745 
746 static int sd_tuning_rx(struct realtek_pci_sdmmc *host, u8 opcode)
747 {
748 	int err, i;
749 	u32 raw_phase_map[RX_TUNING_CNT] = {0}, phase_map;
750 	u8 final_phase;
751 
752 	for (i = 0; i < RX_TUNING_CNT; i++) {
753 		err = sd_tuning_phase(host, opcode, &(raw_phase_map[i]));
754 		if (err < 0)
755 			return err;
756 
757 		if (raw_phase_map[i] == 0)
758 			break;
759 	}
760 
761 	phase_map = 0xFFFFFFFF;
762 	for (i = 0; i < RX_TUNING_CNT; i++) {
763 		dev_dbg(sdmmc_dev(host), "RX raw_phase_map[%d] = 0x%08x\n",
764 				i, raw_phase_map[i]);
765 		phase_map &= raw_phase_map[i];
766 	}
767 	dev_dbg(sdmmc_dev(host), "RX phase_map = 0x%08x\n", phase_map);
768 
769 	if (phase_map) {
770 		final_phase = sd_search_final_phase(host, phase_map);
771 		if (final_phase == 0xFF)
772 			return -EINVAL;
773 
774 		err = sd_change_phase(host, final_phase, true);
775 		if (err < 0)
776 			return err;
777 	} else {
778 		return -EINVAL;
779 	}
780 
781 	return 0;
782 }
783 
784 static inline int sdio_extblock_cmd(struct mmc_command *cmd,
785 	struct mmc_data *data)
786 {
787 	return (cmd->opcode == SD_IO_RW_EXTENDED) && (data->blksz == 512);
788 }
789 
790 static inline int sd_rw_cmd(struct mmc_command *cmd)
791 {
792 	return mmc_op_multi(cmd->opcode) ||
793 		(cmd->opcode == MMC_READ_SINGLE_BLOCK) ||
794 		(cmd->opcode == MMC_WRITE_BLOCK);
795 }
796 
797 static void sd_request(struct work_struct *work)
798 {
799 	struct realtek_pci_sdmmc *host = container_of(work,
800 			struct realtek_pci_sdmmc, work);
801 	struct rtsx_pcr *pcr = host->pcr;
802 
803 	struct mmc_host *mmc = host->mmc;
804 	struct mmc_request *mrq = host->mrq;
805 	struct mmc_command *cmd = mrq->cmd;
806 	struct mmc_data *data = mrq->data;
807 
808 	unsigned int data_size = 0;
809 	int err;
810 
811 	if (host->eject || !sd_get_cd_int(host)) {
812 		cmd->error = -ENOMEDIUM;
813 		goto finish;
814 	}
815 
816 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
817 	if (err) {
818 		cmd->error = err;
819 		goto finish;
820 	}
821 
822 	mutex_lock(&pcr->pcr_mutex);
823 
824 	rtsx_pci_start_run(pcr);
825 
826 	rtsx_pci_switch_clock(pcr, host->clock, host->ssc_depth,
827 			host->initial_mode, host->double_clk, host->vpclk);
828 	rtsx_pci_write_register(pcr, CARD_SELECT, 0x07, SD_MOD_SEL);
829 	rtsx_pci_write_register(pcr, CARD_SHARE_MODE,
830 			CARD_SHARE_MASK, CARD_SHARE_48_SD);
831 
832 	mutex_lock(&host->host_mutex);
833 	host->mrq = mrq;
834 	mutex_unlock(&host->host_mutex);
835 
836 	if (mrq->data)
837 		data_size = data->blocks * data->blksz;
838 
839 	if (!data_size) {
840 		sd_send_cmd_get_rsp(host, cmd);
841 	} else if (sd_rw_cmd(cmd) || sdio_extblock_cmd(cmd, data)) {
842 		cmd->error = sd_rw_multi(host, mrq);
843 		if (!host->using_cookie)
844 			sdmmc_post_req(host->mmc, host->mrq, 0);
845 
846 		if (mmc_op_multi(cmd->opcode) && mrq->stop)
847 			sd_send_cmd_get_rsp(host, mrq->stop);
848 	} else {
849 		sd_normal_rw(host, mrq);
850 	}
851 
852 	if (mrq->data) {
853 		if (cmd->error || data->error)
854 			data->bytes_xfered = 0;
855 		else
856 			data->bytes_xfered = data->blocks * data->blksz;
857 	}
858 
859 	mutex_unlock(&pcr->pcr_mutex);
860 
861 finish:
862 	if (cmd->error) {
863 		dev_dbg(sdmmc_dev(host), "CMD %d 0x%08x error(%d)\n",
864 			cmd->opcode, cmd->arg, cmd->error);
865 	}
866 
867 	mutex_lock(&host->host_mutex);
868 	host->mrq = NULL;
869 	mutex_unlock(&host->host_mutex);
870 
871 	mmc_request_done(mmc, mrq);
872 }
873 
874 static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
875 {
876 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
877 	struct mmc_data *data = mrq->data;
878 
879 	mutex_lock(&host->host_mutex);
880 	host->mrq = mrq;
881 	mutex_unlock(&host->host_mutex);
882 
883 	if (sd_rw_cmd(mrq->cmd) || sdio_extblock_cmd(mrq->cmd, data))
884 		host->using_cookie = sd_pre_dma_transfer(host, data, false);
885 
886 	schedule_work(&host->work);
887 }
888 
889 static int sd_set_bus_width(struct realtek_pci_sdmmc *host,
890 		unsigned char bus_width)
891 {
892 	int err = 0;
893 	u8 width[] = {
894 		[MMC_BUS_WIDTH_1] = SD_BUS_WIDTH_1BIT,
895 		[MMC_BUS_WIDTH_4] = SD_BUS_WIDTH_4BIT,
896 		[MMC_BUS_WIDTH_8] = SD_BUS_WIDTH_8BIT,
897 	};
898 
899 	if (bus_width <= MMC_BUS_WIDTH_8)
900 		err = rtsx_pci_write_register(host->pcr, SD_CFG1,
901 				0x03, width[bus_width]);
902 
903 	return err;
904 }
905 
906 static int sd_power_on(struct realtek_pci_sdmmc *host, unsigned char power_mode)
907 {
908 	struct rtsx_pcr *pcr = host->pcr;
909 	struct mmc_host *mmc = host->mmc;
910 	int err;
911 	u32 val;
912 	u8 test_mode;
913 
914 	if (host->prev_power_state == MMC_POWER_ON)
915 		return 0;
916 
917 	if (host->prev_power_state == MMC_POWER_UP) {
918 		rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, 0);
919 		goto finish;
920 	}
921 
922 	msleep(100);
923 
924 	rtsx_pci_init_cmd(pcr);
925 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SELECT, 0x07, SD_MOD_SEL);
926 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_SHARE_MODE,
927 			CARD_SHARE_MASK, CARD_SHARE_48_SD);
928 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN,
929 			SD_CLK_EN, SD_CLK_EN);
930 	err = rtsx_pci_send_cmd(pcr, 100);
931 	if (err < 0)
932 		return err;
933 
934 	err = rtsx_pci_card_pull_ctl_enable(pcr, RTSX_SD_CARD);
935 	if (err < 0)
936 		return err;
937 
938 	err = rtsx_pci_card_power_on(pcr, RTSX_SD_CARD);
939 	if (err < 0)
940 		return err;
941 
942 	mdelay(1);
943 
944 	err = rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, SD_OUTPUT_EN);
945 	if (err < 0)
946 		return err;
947 
948 	/* send at least 74 clocks */
949 	rtsx_pci_write_register(pcr, SD_BUS_STAT, SD_CLK_TOGGLE_EN, SD_CLK_TOGGLE_EN);
950 
951 	if ((PCI_PID(pcr) == PID_5261) || (PCI_PID(pcr) == PID_5264)) {
952 		/*
953 		 * If test mode is set switch to SD Express mandatorily,
954 		 * this is only for factory testing.
955 		 */
956 		rtsx_pci_read_register(pcr, RTS5261_FW_CFG_INFO0, &test_mode);
957 		if (test_mode & RTS5261_FW_EXPRESS_TEST_MASK) {
958 			sdmmc_init_sd_express(mmc, NULL);
959 			return 0;
960 		}
961 		if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
962 			mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
963 		/*
964 		 * HW read wp status when resuming from S3/S4,
965 		 * and then picks SD legacy interface if it's set
966 		 * in read-only mode.
967 		 */
968 		val = rtsx_pci_readl(pcr, RTSX_BIPR);
969 		if (val & SD_WRITE_PROTECT) {
970 			pcr->extra_caps &= ~EXTRA_CAPS_SD_EXPRESS;
971 			mmc->caps2 &= ~(MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V);
972 		}
973 	}
974 
975 finish:
976 	host->prev_power_state = power_mode;
977 	return 0;
978 }
979 
980 static int sd_power_off(struct realtek_pci_sdmmc *host)
981 {
982 	struct rtsx_pcr *pcr = host->pcr;
983 	int err;
984 
985 	host->prev_power_state = MMC_POWER_OFF;
986 
987 	rtsx_pci_init_cmd(pcr);
988 
989 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_EN, SD_CLK_EN, 0);
990 	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_OE, SD_OUTPUT_EN, 0);
991 
992 	err = rtsx_pci_send_cmd(pcr, 100);
993 	if (err < 0)
994 		return err;
995 
996 	err = rtsx_pci_card_power_off(pcr, RTSX_SD_CARD);
997 	if (err < 0)
998 		return err;
999 
1000 	return rtsx_pci_card_pull_ctl_disable(pcr, RTSX_SD_CARD);
1001 }
1002 
1003 static int sd_set_power_mode(struct realtek_pci_sdmmc *host,
1004 		unsigned char power_mode)
1005 {
1006 	int err;
1007 
1008 	if (power_mode == MMC_POWER_OFF)
1009 		err = sd_power_off(host);
1010 	else
1011 		err = sd_power_on(host, power_mode);
1012 
1013 	return err;
1014 }
1015 
1016 static int sd_set_timing(struct realtek_pci_sdmmc *host, unsigned char timing)
1017 {
1018 	struct rtsx_pcr *pcr = host->pcr;
1019 	int err = 0;
1020 
1021 	rtsx_pci_init_cmd(pcr);
1022 
1023 	switch (timing) {
1024 	case MMC_TIMING_UHS_SDR104:
1025 	case MMC_TIMING_UHS_SDR50:
1026 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1027 				0x0C | SD_ASYNC_FIFO_NOT_RST,
1028 				SD_30_MODE | SD_ASYNC_FIFO_NOT_RST);
1029 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1030 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1031 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1032 				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1033 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1034 		break;
1035 
1036 	case MMC_TIMING_MMC_DDR52:
1037 	case MMC_TIMING_UHS_DDR50:
1038 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1039 				0x0C | SD_ASYNC_FIFO_NOT_RST,
1040 				SD_DDR_MODE | SD_ASYNC_FIFO_NOT_RST);
1041 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1042 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1043 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1044 				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
1045 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1046 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1047 				DDR_VAR_TX_CMD_DAT, DDR_VAR_TX_CMD_DAT);
1048 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1049 				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD,
1050 				DDR_VAR_RX_DAT | DDR_VAR_RX_CMD);
1051 		break;
1052 
1053 	case MMC_TIMING_MMC_HS:
1054 	case MMC_TIMING_SD_HS:
1055 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_CFG1,
1056 				0x0C, SD_20_MODE);
1057 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1058 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1059 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1060 				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1061 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1062 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_PUSH_POINT_CTL,
1063 				SD20_TX_SEL_MASK, SD20_TX_14_AHEAD);
1064 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1065 				SD20_RX_SEL_MASK, SD20_RX_14_DELAY);
1066 		break;
1067 
1068 	default:
1069 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1070 				SD_CFG1, 0x0C, SD_20_MODE);
1071 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL,
1072 				CLK_LOW_FREQ, CLK_LOW_FREQ);
1073 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CARD_CLK_SOURCE, 0xFF,
1074 				CRC_FIX_CLK | SD30_VAR_CLK0 | SAMPLE_VAR_CLK1);
1075 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, CLK_CTL, CLK_LOW_FREQ, 0);
1076 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD,
1077 				SD_PUSH_POINT_CTL, 0xFF, 0);
1078 		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD_SAMPLE_POINT_CTL,
1079 				SD20_RX_SEL_MASK, SD20_RX_POS_EDGE);
1080 		break;
1081 	}
1082 
1083 	err = rtsx_pci_send_cmd(pcr, 100);
1084 
1085 	return err;
1086 }
1087 
1088 static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1089 {
1090 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1091 	struct rtsx_pcr *pcr = host->pcr;
1092 
1093 	if (host->eject)
1094 		return;
1095 
1096 	if (rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD))
1097 		return;
1098 
1099 	mutex_lock(&pcr->pcr_mutex);
1100 
1101 	rtsx_pci_start_run(pcr);
1102 
1103 	sd_set_bus_width(host, ios->bus_width);
1104 	sd_set_power_mode(host, ios->power_mode);
1105 	sd_set_timing(host, ios->timing);
1106 
1107 	host->vpclk = false;
1108 	host->double_clk = true;
1109 
1110 	switch (ios->timing) {
1111 	case MMC_TIMING_UHS_SDR104:
1112 	case MMC_TIMING_UHS_SDR50:
1113 		host->ssc_depth = RTSX_SSC_DEPTH_2M;
1114 		host->vpclk = true;
1115 		host->double_clk = false;
1116 		break;
1117 	case MMC_TIMING_MMC_DDR52:
1118 	case MMC_TIMING_UHS_DDR50:
1119 	case MMC_TIMING_UHS_SDR25:
1120 		host->ssc_depth = RTSX_SSC_DEPTH_1M;
1121 		break;
1122 	default:
1123 		host->ssc_depth = RTSX_SSC_DEPTH_500K;
1124 		break;
1125 	}
1126 
1127 	host->initial_mode = (ios->clock <= 1000000) ? true : false;
1128 
1129 	host->clock = ios->clock;
1130 	rtsx_pci_switch_clock(pcr, ios->clock, host->ssc_depth,
1131 			host->initial_mode, host->double_clk, host->vpclk);
1132 
1133 	mutex_unlock(&pcr->pcr_mutex);
1134 }
1135 
1136 static int sdmmc_get_ro(struct mmc_host *mmc)
1137 {
1138 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1139 	struct rtsx_pcr *pcr = host->pcr;
1140 	int ro = 0;
1141 	u32 val;
1142 
1143 	if (host->eject)
1144 		return -ENOMEDIUM;
1145 
1146 	mutex_lock(&pcr->pcr_mutex);
1147 
1148 	rtsx_pci_start_run(pcr);
1149 
1150 	/* Check SD mechanical write-protect switch */
1151 	val = rtsx_pci_readl(pcr, RTSX_BIPR);
1152 	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1153 	if (val & SD_WRITE_PROTECT)
1154 		ro = 1;
1155 
1156 	mutex_unlock(&pcr->pcr_mutex);
1157 
1158 	return ro;
1159 }
1160 
1161 static int sdmmc_get_cd(struct mmc_host *mmc)
1162 {
1163 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1164 	struct rtsx_pcr *pcr = host->pcr;
1165 	int cd = 0;
1166 	u32 val;
1167 
1168 	if (host->eject)
1169 		return cd;
1170 
1171 	mutex_lock(&pcr->pcr_mutex);
1172 
1173 	rtsx_pci_start_run(pcr);
1174 
1175 	/* Check SD card detect */
1176 	val = rtsx_pci_card_exist(pcr);
1177 	dev_dbg(sdmmc_dev(host), "%s: RTSX_BIPR = 0x%08x\n", __func__, val);
1178 	if (val & SD_EXIST)
1179 		cd = 1;
1180 
1181 	mutex_unlock(&pcr->pcr_mutex);
1182 
1183 	return cd;
1184 }
1185 
1186 static int sd_wait_voltage_stable_1(struct realtek_pci_sdmmc *host)
1187 {
1188 	struct rtsx_pcr *pcr = host->pcr;
1189 	int err;
1190 	u8 stat;
1191 
1192 	/* Reference to Signal Voltage Switch Sequence in SD spec.
1193 	 * Wait for a period of time so that the card can drive SD_CMD and
1194 	 * SD_DAT[3:0] to low after sending back CMD11 response.
1195 	 */
1196 	mdelay(1);
1197 
1198 	/* SD_CMD, SD_DAT[3:0] should be driven to low by card;
1199 	 * If either one of SD_CMD,SD_DAT[3:0] is not low,
1200 	 * abort the voltage switch sequence;
1201 	 */
1202 	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1203 	if (err < 0)
1204 		return err;
1205 
1206 	if (stat & (SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1207 				SD_DAT1_STATUS | SD_DAT0_STATUS))
1208 		return -EINVAL;
1209 
1210 	/* Stop toggle SD clock */
1211 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1212 			0xFF, SD_CLK_FORCE_STOP);
1213 	if (err < 0)
1214 		return err;
1215 
1216 	return 0;
1217 }
1218 
1219 static int sd_wait_voltage_stable_2(struct realtek_pci_sdmmc *host)
1220 {
1221 	struct rtsx_pcr *pcr = host->pcr;
1222 	int err;
1223 	u8 stat, mask, val;
1224 
1225 	/* Wait 1.8V output of voltage regulator in card stable */
1226 	msleep(50);
1227 
1228 	/* Toggle SD clock again */
1229 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT, 0xFF, SD_CLK_TOGGLE_EN);
1230 	if (err < 0)
1231 		return err;
1232 
1233 	/* Wait for a period of time so that the card can drive
1234 	 * SD_DAT[3:0] to high at 1.8V
1235 	 */
1236 	msleep(20);
1237 
1238 	/* SD_CMD, SD_DAT[3:0] should be pulled high by host */
1239 	err = rtsx_pci_read_register(pcr, SD_BUS_STAT, &stat);
1240 	if (err < 0)
1241 		return err;
1242 
1243 	mask = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1244 		SD_DAT1_STATUS | SD_DAT0_STATUS;
1245 	val = SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
1246 		SD_DAT1_STATUS | SD_DAT0_STATUS;
1247 	if ((stat & mask) != val) {
1248 		dev_dbg(sdmmc_dev(host),
1249 			"%s: SD_BUS_STAT = 0x%x\n", __func__, stat);
1250 		rtsx_pci_write_register(pcr, SD_BUS_STAT,
1251 				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1252 		rtsx_pci_write_register(pcr, CARD_CLK_EN, 0xFF, 0);
1253 		return -EINVAL;
1254 	}
1255 
1256 	return 0;
1257 }
1258 
1259 static int sdmmc_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
1260 {
1261 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1262 	struct rtsx_pcr *pcr = host->pcr;
1263 	int err = 0;
1264 	u8 voltage;
1265 
1266 	dev_dbg(sdmmc_dev(host), "%s: signal_voltage = %d\n",
1267 			__func__, ios->signal_voltage);
1268 
1269 	if (host->eject)
1270 		return -ENOMEDIUM;
1271 
1272 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1273 	if (err)
1274 		return err;
1275 
1276 	mutex_lock(&pcr->pcr_mutex);
1277 
1278 	rtsx_pci_start_run(pcr);
1279 
1280 	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
1281 		voltage = OUTPUT_3V3;
1282 	else
1283 		voltage = OUTPUT_1V8;
1284 
1285 	if (voltage == OUTPUT_1V8) {
1286 		err = sd_wait_voltage_stable_1(host);
1287 		if (err < 0)
1288 			goto out;
1289 	}
1290 
1291 	err = rtsx_pci_switch_output_voltage(pcr, voltage);
1292 	if (err < 0)
1293 		goto out;
1294 
1295 	if (voltage == OUTPUT_1V8) {
1296 		err = sd_wait_voltage_stable_2(host);
1297 		if (err < 0)
1298 			goto out;
1299 	}
1300 
1301 out:
1302 	/* Stop toggle SD clock in idle */
1303 	err = rtsx_pci_write_register(pcr, SD_BUS_STAT,
1304 			SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
1305 
1306 	mutex_unlock(&pcr->pcr_mutex);
1307 
1308 	return err;
1309 }
1310 
1311 static int sdmmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
1312 {
1313 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1314 	struct rtsx_pcr *pcr = host->pcr;
1315 	int err = 0;
1316 
1317 	if (host->eject)
1318 		return -ENOMEDIUM;
1319 
1320 	err = rtsx_pci_card_exclusive_check(host->pcr, RTSX_SD_CARD);
1321 	if (err)
1322 		return err;
1323 
1324 	mutex_lock(&pcr->pcr_mutex);
1325 
1326 	rtsx_pci_start_run(pcr);
1327 
1328 	/* Set initial TX phase */
1329 	switch (mmc->ios.timing) {
1330 	case MMC_TIMING_UHS_SDR104:
1331 		err = sd_change_phase(host, SDR104_TX_PHASE(pcr), false);
1332 		break;
1333 
1334 	case MMC_TIMING_UHS_SDR50:
1335 		err = sd_change_phase(host, SDR50_TX_PHASE(pcr), false);
1336 		break;
1337 
1338 	case MMC_TIMING_UHS_DDR50:
1339 		err = sd_change_phase(host, DDR50_TX_PHASE(pcr), false);
1340 		break;
1341 
1342 	default:
1343 		err = 0;
1344 	}
1345 
1346 	if (err)
1347 		goto out;
1348 
1349 	/* Tuning RX phase */
1350 	if ((mmc->ios.timing == MMC_TIMING_UHS_SDR104) ||
1351 			(mmc->ios.timing == MMC_TIMING_UHS_SDR50))
1352 		err = sd_tuning_rx(host, opcode);
1353 	else if (mmc->ios.timing == MMC_TIMING_UHS_DDR50)
1354 		err = sd_change_phase(host, DDR50_RX_PHASE(pcr), true);
1355 
1356 out:
1357 	mutex_unlock(&pcr->pcr_mutex);
1358 
1359 	return err;
1360 }
1361 
1362 static int sdmmc_init_sd_express(struct mmc_host *mmc, struct mmc_ios *ios)
1363 {
1364 	u32 relink_time;
1365 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
1366 	struct rtsx_pcr *pcr = host->pcr;
1367 
1368 	if (PCI_PID(pcr) == PID_5264) {
1369 		pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL2,
1370 				PCI_EXP_LNKCTL2_TLS, PCI_EXP_LNKCTL2_TLS_2_5GT);
1371 		pci_write_config_byte(pcr->pci, 0x80e, 0x02);
1372 		pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL2,
1373 				PCI_EXP_LNKCTL2_TLS, PCI_EXP_LNKCTL2_TLS_5_0GT);
1374 	}
1375 
1376 	/* Set relink_time for changing to PCIe card */
1377 	relink_time = 0x8FFF;
1378 
1379 	rtsx_pci_write_register(pcr, 0xFF01, 0xFF, relink_time);
1380 	rtsx_pci_write_register(pcr, 0xFF02, 0xFF, relink_time >> 8);
1381 	rtsx_pci_write_register(pcr, 0xFF03, 0x01, relink_time >> 16);
1382 
1383 	rtsx_pci_write_register(pcr, PETXCFG, 0x80, 0x80);
1384 	rtsx_pci_write_register(pcr, LDO_VCC_CFG0,
1385 		RTS5261_LDO1_OCP_THD_MASK,
1386 		pcr->option.sd_800mA_ocp_thd);
1387 
1388 	if (pcr->ops->disable_auto_blink)
1389 		pcr->ops->disable_auto_blink(pcr);
1390 
1391 	if (PCI_PID(pcr) == PID_5264) {
1392 		rtsx_pci_write_register(pcr, RTS5264_AUTOLOAD_CFG2,
1393 			RTS5264_CHIP_RST_N_SEL, RTS5264_CHIP_RST_N_SEL);
1394 		rtsx_pci_write_register(pcr, GPIO_CTL, 0x02, 0x00);
1395 	}
1396 
1397 	/* For PCIe/NVMe mode can't enter delink issue */
1398 	pcr->hw_param.interrupt_en &= ~(SD_INT_EN);
1399 	rtsx_pci_writel(pcr, RTSX_BIER, pcr->hw_param.interrupt_en);
1400 
1401 	rtsx_pci_write_register(pcr, RTS5260_AUTOLOAD_CFG4,
1402 		RTS5261_AUX_CLK_16M_EN, RTS5261_AUX_CLK_16M_EN);
1403 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG0,
1404 		RTS5261_FW_ENTER_EXPRESS, RTS5261_FW_ENTER_EXPRESS);
1405 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1406 		RTS5261_MCU_CLOCK_GATING, RTS5261_MCU_CLOCK_GATING);
1407 	rtsx_pci_write_register(pcr, RTS5261_FW_CFG1,
1408 		RTS5261_MCU_BUS_SEL_MASK | RTS5261_MCU_CLOCK_SEL_MASK
1409 		| RTS5261_DRIVER_ENABLE_FW,
1410 		RTS5261_MCU_CLOCK_SEL_16M | RTS5261_DRIVER_ENABLE_FW);
1411 	host->eject = true;
1412 	return 0;
1413 }
1414 
1415 static const struct mmc_host_ops realtek_pci_sdmmc_ops = {
1416 	.pre_req = sdmmc_pre_req,
1417 	.post_req = sdmmc_post_req,
1418 	.request = sdmmc_request,
1419 	.set_ios = sdmmc_set_ios,
1420 	.get_ro = sdmmc_get_ro,
1421 	.get_cd = sdmmc_get_cd,
1422 	.start_signal_voltage_switch = sdmmc_switch_voltage,
1423 	.execute_tuning = sdmmc_execute_tuning,
1424 	.init_sd_express = sdmmc_init_sd_express,
1425 };
1426 
1427 static void init_extra_caps(struct realtek_pci_sdmmc *host)
1428 {
1429 	struct mmc_host *mmc = host->mmc;
1430 	struct rtsx_pcr *pcr = host->pcr;
1431 
1432 	dev_dbg(sdmmc_dev(host), "pcr->extra_caps = 0x%x\n", pcr->extra_caps);
1433 
1434 	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR50)
1435 		mmc->caps |= MMC_CAP_UHS_SDR50;
1436 	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR104)
1437 		mmc->caps |= MMC_CAP_UHS_SDR104;
1438 	if (pcr->extra_caps & EXTRA_CAPS_SD_DDR50)
1439 		mmc->caps |= MMC_CAP_UHS_DDR50;
1440 	if (pcr->extra_caps & EXTRA_CAPS_MMC_HSDDR)
1441 		mmc->caps |= MMC_CAP_1_8V_DDR;
1442 	if (pcr->extra_caps & EXTRA_CAPS_MMC_8BIT)
1443 		mmc->caps |= MMC_CAP_8_BIT_DATA;
1444 	if (pcr->extra_caps & EXTRA_CAPS_NO_MMC)
1445 		mmc->caps2 |= MMC_CAP2_NO_MMC;
1446 	if (pcr->extra_caps & EXTRA_CAPS_SD_EXPRESS)
1447 		mmc->caps2 |= MMC_CAP2_SD_EXP | MMC_CAP2_SD_EXP_1_2V;
1448 }
1449 
1450 static void realtek_init_host(struct realtek_pci_sdmmc *host)
1451 {
1452 	struct mmc_host *mmc = host->mmc;
1453 	struct rtsx_pcr *pcr = host->pcr;
1454 
1455 	mmc->f_min = 250000;
1456 	mmc->f_max = 208000000;
1457 	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34 | MMC_VDD_165_195;
1458 	mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED |
1459 		MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST |
1460 		MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
1461 	if (pcr->rtd3_en)
1462 		mmc->caps = mmc->caps | MMC_CAP_AGGRESSIVE_PM;
1463 	mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE |
1464 		MMC_CAP2_NO_SDIO;
1465 	mmc->max_current_330 = 400;
1466 	mmc->max_current_180 = 800;
1467 	mmc->ops = &realtek_pci_sdmmc_ops;
1468 
1469 	init_extra_caps(host);
1470 
1471 	mmc->max_segs = 256;
1472 	mmc->max_seg_size = 65536;
1473 	mmc->max_blk_size = 512;
1474 	mmc->max_blk_count = 65535;
1475 	mmc->max_req_size = 524288;
1476 }
1477 
1478 static void rtsx_pci_sdmmc_card_event(struct platform_device *pdev)
1479 {
1480 	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1481 
1482 	host->cookie = -1;
1483 	mmc_detect_change(host->mmc, 0);
1484 }
1485 
1486 static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev)
1487 {
1488 	struct mmc_host *mmc;
1489 	struct realtek_pci_sdmmc *host;
1490 	struct rtsx_pcr *pcr;
1491 	struct pcr_handle *handle = pdev->dev.platform_data;
1492 	int ret;
1493 
1494 	if (!handle)
1495 		return -ENXIO;
1496 
1497 	pcr = handle->pcr;
1498 	if (!pcr)
1499 		return -ENXIO;
1500 
1501 	dev_dbg(&(pdev->dev), ": Realtek PCI-E SDMMC controller found\n");
1502 
1503 	mmc = mmc_alloc_host(sizeof(*host), &pdev->dev);
1504 	if (!mmc)
1505 		return -ENOMEM;
1506 
1507 	host = mmc_priv(mmc);
1508 	host->pcr = pcr;
1509 	mmc->ios.power_delay_ms = 5;
1510 	host->mmc = mmc;
1511 	host->pdev = pdev;
1512 	host->cookie = -1;
1513 	host->prev_power_state = MMC_POWER_OFF;
1514 	INIT_WORK(&host->work, sd_request);
1515 	platform_set_drvdata(pdev, host);
1516 	pcr->slots[RTSX_SD_CARD].p_dev = pdev;
1517 	pcr->slots[RTSX_SD_CARD].card_event = rtsx_pci_sdmmc_card_event;
1518 
1519 	mutex_init(&host->host_mutex);
1520 
1521 	realtek_init_host(host);
1522 
1523 	pm_runtime_no_callbacks(&pdev->dev);
1524 	pm_runtime_set_active(&pdev->dev);
1525 	pm_runtime_enable(&pdev->dev);
1526 	pm_runtime_set_autosuspend_delay(&pdev->dev, 200);
1527 	pm_runtime_mark_last_busy(&pdev->dev);
1528 	pm_runtime_use_autosuspend(&pdev->dev);
1529 
1530 	ret = mmc_add_host(mmc);
1531 	if (ret) {
1532 		pm_runtime_dont_use_autosuspend(&pdev->dev);
1533 		pm_runtime_disable(&pdev->dev);
1534 		mmc_free_host(mmc);
1535 		return ret;
1536 	}
1537 
1538 	return 0;
1539 }
1540 
1541 static void rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev)
1542 {
1543 	struct realtek_pci_sdmmc *host = platform_get_drvdata(pdev);
1544 	struct rtsx_pcr *pcr;
1545 	struct mmc_host *mmc;
1546 
1547 	pcr = host->pcr;
1548 	pcr->slots[RTSX_SD_CARD].p_dev = NULL;
1549 	pcr->slots[RTSX_SD_CARD].card_event = NULL;
1550 	mmc = host->mmc;
1551 
1552 	cancel_work_sync(&host->work);
1553 
1554 	mutex_lock(&host->host_mutex);
1555 	if (host->mrq) {
1556 		dev_dbg(&(pdev->dev),
1557 			"%s: Controller removed during transfer\n",
1558 			mmc_hostname(mmc));
1559 
1560 		rtsx_pci_complete_unfinished_transfer(pcr);
1561 
1562 		host->mrq->cmd->error = -ENOMEDIUM;
1563 		if (host->mrq->stop)
1564 			host->mrq->stop->error = -ENOMEDIUM;
1565 		mmc_request_done(mmc, host->mrq);
1566 	}
1567 	mutex_unlock(&host->host_mutex);
1568 
1569 	mmc_remove_host(mmc);
1570 	host->eject = true;
1571 
1572 	flush_work(&host->work);
1573 
1574 	pm_runtime_dont_use_autosuspend(&pdev->dev);
1575 	pm_runtime_disable(&pdev->dev);
1576 
1577 	mmc_free_host(mmc);
1578 
1579 	dev_dbg(&(pdev->dev),
1580 		": Realtek PCI-E SDMMC controller has been removed\n");
1581 }
1582 
1583 static const struct platform_device_id rtsx_pci_sdmmc_ids[] = {
1584 	{
1585 		.name = DRV_NAME_RTSX_PCI_SDMMC,
1586 	}, {
1587 		/* sentinel */
1588 	}
1589 };
1590 MODULE_DEVICE_TABLE(platform, rtsx_pci_sdmmc_ids);
1591 
1592 static struct platform_driver rtsx_pci_sdmmc_driver = {
1593 	.probe		= rtsx_pci_sdmmc_drv_probe,
1594 	.remove_new	= rtsx_pci_sdmmc_drv_remove,
1595 	.id_table       = rtsx_pci_sdmmc_ids,
1596 	.driver		= {
1597 		.name	= DRV_NAME_RTSX_PCI_SDMMC,
1598 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1599 	},
1600 };
1601 module_platform_driver(rtsx_pci_sdmmc_driver);
1602 
1603 MODULE_LICENSE("GPL");
1604 MODULE_AUTHOR("Wei WANG <wei_wang@realsil.com.cn>");
1605 MODULE_DESCRIPTION("Realtek PCI-E SD/MMC Card Host Driver");
1606