xref: /linux/drivers/mmc/host/sdhci.c (revision bab2c80e5a6c855657482eac9e97f5f3eedb509a)
1 /*
2  *  linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver
3  *
4  *  Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or (at
9  * your option) any later version.
10  *
11  * Thanks to the following companies for their support:
12  *
13  *     - JMicron (hardware and technical support)
14  */
15 
16 #include <linux/delay.h>
17 #include <linux/ktime.h>
18 #include <linux/highmem.h>
19 #include <linux/io.h>
20 #include <linux/module.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/slab.h>
23 #include <linux/scatterlist.h>
24 #include <linux/sizes.h>
25 #include <linux/swiotlb.h>
26 #include <linux/regulator/consumer.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/of.h>
29 
30 #include <linux/leds.h>
31 
32 #include <linux/mmc/mmc.h>
33 #include <linux/mmc/host.h>
34 #include <linux/mmc/card.h>
35 #include <linux/mmc/sdio.h>
36 #include <linux/mmc/slot-gpio.h>
37 
38 #include "sdhci.h"
39 
40 #define DRIVER_NAME "sdhci"
41 
42 #define DBG(f, x...) \
43 	pr_debug("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
44 
45 #define SDHCI_DUMP(f, x...) \
46 	pr_err("%s: " DRIVER_NAME ": " f, mmc_hostname(host->mmc), ## x)
47 
48 #define MAX_TUNING_LOOP 40
49 
50 static unsigned int debug_quirks = 0;
51 static unsigned int debug_quirks2;
52 
53 static void sdhci_finish_data(struct sdhci_host *);
54 
55 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable);
56 
57 void sdhci_dumpregs(struct sdhci_host *host)
58 {
59 	SDHCI_DUMP("============ SDHCI REGISTER DUMP ===========\n");
60 
61 	SDHCI_DUMP("Sys addr:  0x%08x | Version:  0x%08x\n",
62 		   sdhci_readl(host, SDHCI_DMA_ADDRESS),
63 		   sdhci_readw(host, SDHCI_HOST_VERSION));
64 	SDHCI_DUMP("Blk size:  0x%08x | Blk cnt:  0x%08x\n",
65 		   sdhci_readw(host, SDHCI_BLOCK_SIZE),
66 		   sdhci_readw(host, SDHCI_BLOCK_COUNT));
67 	SDHCI_DUMP("Argument:  0x%08x | Trn mode: 0x%08x\n",
68 		   sdhci_readl(host, SDHCI_ARGUMENT),
69 		   sdhci_readw(host, SDHCI_TRANSFER_MODE));
70 	SDHCI_DUMP("Present:   0x%08x | Host ctl: 0x%08x\n",
71 		   sdhci_readl(host, SDHCI_PRESENT_STATE),
72 		   sdhci_readb(host, SDHCI_HOST_CONTROL));
73 	SDHCI_DUMP("Power:     0x%08x | Blk gap:  0x%08x\n",
74 		   sdhci_readb(host, SDHCI_POWER_CONTROL),
75 		   sdhci_readb(host, SDHCI_BLOCK_GAP_CONTROL));
76 	SDHCI_DUMP("Wake-up:   0x%08x | Clock:    0x%08x\n",
77 		   sdhci_readb(host, SDHCI_WAKE_UP_CONTROL),
78 		   sdhci_readw(host, SDHCI_CLOCK_CONTROL));
79 	SDHCI_DUMP("Timeout:   0x%08x | Int stat: 0x%08x\n",
80 		   sdhci_readb(host, SDHCI_TIMEOUT_CONTROL),
81 		   sdhci_readl(host, SDHCI_INT_STATUS));
82 	SDHCI_DUMP("Int enab:  0x%08x | Sig enab: 0x%08x\n",
83 		   sdhci_readl(host, SDHCI_INT_ENABLE),
84 		   sdhci_readl(host, SDHCI_SIGNAL_ENABLE));
85 	SDHCI_DUMP("AC12 err:  0x%08x | Slot int: 0x%08x\n",
86 		   sdhci_readw(host, SDHCI_ACMD12_ERR),
87 		   sdhci_readw(host, SDHCI_SLOT_INT_STATUS));
88 	SDHCI_DUMP("Caps:      0x%08x | Caps_1:   0x%08x\n",
89 		   sdhci_readl(host, SDHCI_CAPABILITIES),
90 		   sdhci_readl(host, SDHCI_CAPABILITIES_1));
91 	SDHCI_DUMP("Cmd:       0x%08x | Max curr: 0x%08x\n",
92 		   sdhci_readw(host, SDHCI_COMMAND),
93 		   sdhci_readl(host, SDHCI_MAX_CURRENT));
94 	SDHCI_DUMP("Resp[0]:   0x%08x | Resp[1]:  0x%08x\n",
95 		   sdhci_readl(host, SDHCI_RESPONSE),
96 		   sdhci_readl(host, SDHCI_RESPONSE + 4));
97 	SDHCI_DUMP("Resp[2]:   0x%08x | Resp[3]:  0x%08x\n",
98 		   sdhci_readl(host, SDHCI_RESPONSE + 8),
99 		   sdhci_readl(host, SDHCI_RESPONSE + 12));
100 	SDHCI_DUMP("Host ctl2: 0x%08x\n",
101 		   sdhci_readw(host, SDHCI_HOST_CONTROL2));
102 
103 	if (host->flags & SDHCI_USE_ADMA) {
104 		if (host->flags & SDHCI_USE_64_BIT_DMA) {
105 			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x%08x\n",
106 				   sdhci_readl(host, SDHCI_ADMA_ERROR),
107 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS_HI),
108 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
109 		} else {
110 			SDHCI_DUMP("ADMA Err:  0x%08x | ADMA Ptr: 0x%08x\n",
111 				   sdhci_readl(host, SDHCI_ADMA_ERROR),
112 				   sdhci_readl(host, SDHCI_ADMA_ADDRESS));
113 		}
114 	}
115 
116 	SDHCI_DUMP("============================================\n");
117 }
118 EXPORT_SYMBOL_GPL(sdhci_dumpregs);
119 
120 /*****************************************************************************\
121  *                                                                           *
122  * Low level functions                                                       *
123  *                                                                           *
124 \*****************************************************************************/
125 
126 static inline bool sdhci_data_line_cmd(struct mmc_command *cmd)
127 {
128 	return cmd->data || cmd->flags & MMC_RSP_BUSY;
129 }
130 
131 static void sdhci_set_card_detection(struct sdhci_host *host, bool enable)
132 {
133 	u32 present;
134 
135 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) ||
136 	    !mmc_card_is_removable(host->mmc))
137 		return;
138 
139 	if (enable) {
140 		present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
141 				      SDHCI_CARD_PRESENT;
142 
143 		host->ier |= present ? SDHCI_INT_CARD_REMOVE :
144 				       SDHCI_INT_CARD_INSERT;
145 	} else {
146 		host->ier &= ~(SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT);
147 	}
148 
149 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
150 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
151 }
152 
153 static void sdhci_enable_card_detection(struct sdhci_host *host)
154 {
155 	sdhci_set_card_detection(host, true);
156 }
157 
158 static void sdhci_disable_card_detection(struct sdhci_host *host)
159 {
160 	sdhci_set_card_detection(host, false);
161 }
162 
163 static void sdhci_runtime_pm_bus_on(struct sdhci_host *host)
164 {
165 	if (host->bus_on)
166 		return;
167 	host->bus_on = true;
168 	pm_runtime_get_noresume(host->mmc->parent);
169 }
170 
171 static void sdhci_runtime_pm_bus_off(struct sdhci_host *host)
172 {
173 	if (!host->bus_on)
174 		return;
175 	host->bus_on = false;
176 	pm_runtime_put_noidle(host->mmc->parent);
177 }
178 
179 void sdhci_reset(struct sdhci_host *host, u8 mask)
180 {
181 	ktime_t timeout;
182 
183 	sdhci_writeb(host, mask, SDHCI_SOFTWARE_RESET);
184 
185 	if (mask & SDHCI_RESET_ALL) {
186 		host->clock = 0;
187 		/* Reset-all turns off SD Bus Power */
188 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
189 			sdhci_runtime_pm_bus_off(host);
190 	}
191 
192 	/* Wait max 100 ms */
193 	timeout = ktime_add_ms(ktime_get(), 100);
194 
195 	/* hw clears the bit when it's done */
196 	while (sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask) {
197 		if (ktime_after(ktime_get(), timeout)) {
198 			pr_err("%s: Reset 0x%x never completed.\n",
199 				mmc_hostname(host->mmc), (int)mask);
200 			sdhci_dumpregs(host);
201 			return;
202 		}
203 		udelay(10);
204 	}
205 }
206 EXPORT_SYMBOL_GPL(sdhci_reset);
207 
208 static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
209 {
210 	if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
211 		struct mmc_host *mmc = host->mmc;
212 
213 		if (!mmc->ops->get_cd(mmc))
214 			return;
215 	}
216 
217 	host->ops->reset(host, mask);
218 
219 	if (mask & SDHCI_RESET_ALL) {
220 		if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
221 			if (host->ops->enable_dma)
222 				host->ops->enable_dma(host);
223 		}
224 
225 		/* Resetting the controller clears many */
226 		host->preset_enabled = false;
227 	}
228 }
229 
230 static void sdhci_set_default_irqs(struct sdhci_host *host)
231 {
232 	host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
233 		    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT |
234 		    SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC |
235 		    SDHCI_INT_TIMEOUT | SDHCI_INT_DATA_END |
236 		    SDHCI_INT_RESPONSE;
237 
238 	if (host->tuning_mode == SDHCI_TUNING_MODE_2 ||
239 	    host->tuning_mode == SDHCI_TUNING_MODE_3)
240 		host->ier |= SDHCI_INT_RETUNE;
241 
242 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
243 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
244 }
245 
246 static void sdhci_init(struct sdhci_host *host, int soft)
247 {
248 	struct mmc_host *mmc = host->mmc;
249 
250 	if (soft)
251 		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
252 	else
253 		sdhci_do_reset(host, SDHCI_RESET_ALL);
254 
255 	sdhci_set_default_irqs(host);
256 
257 	host->cqe_on = false;
258 
259 	if (soft) {
260 		/* force clock reconfiguration */
261 		host->clock = 0;
262 		mmc->ops->set_ios(mmc, &mmc->ios);
263 	}
264 }
265 
266 static void sdhci_reinit(struct sdhci_host *host)
267 {
268 	sdhci_init(host, 0);
269 	sdhci_enable_card_detection(host);
270 }
271 
272 static void __sdhci_led_activate(struct sdhci_host *host)
273 {
274 	u8 ctrl;
275 
276 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
277 	ctrl |= SDHCI_CTRL_LED;
278 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
279 }
280 
281 static void __sdhci_led_deactivate(struct sdhci_host *host)
282 {
283 	u8 ctrl;
284 
285 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
286 	ctrl &= ~SDHCI_CTRL_LED;
287 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
288 }
289 
290 #if IS_REACHABLE(CONFIG_LEDS_CLASS)
291 static void sdhci_led_control(struct led_classdev *led,
292 			      enum led_brightness brightness)
293 {
294 	struct sdhci_host *host = container_of(led, struct sdhci_host, led);
295 	unsigned long flags;
296 
297 	spin_lock_irqsave(&host->lock, flags);
298 
299 	if (host->runtime_suspended)
300 		goto out;
301 
302 	if (brightness == LED_OFF)
303 		__sdhci_led_deactivate(host);
304 	else
305 		__sdhci_led_activate(host);
306 out:
307 	spin_unlock_irqrestore(&host->lock, flags);
308 }
309 
310 static int sdhci_led_register(struct sdhci_host *host)
311 {
312 	struct mmc_host *mmc = host->mmc;
313 
314 	snprintf(host->led_name, sizeof(host->led_name),
315 		 "%s::", mmc_hostname(mmc));
316 
317 	host->led.name = host->led_name;
318 	host->led.brightness = LED_OFF;
319 	host->led.default_trigger = mmc_hostname(mmc);
320 	host->led.brightness_set = sdhci_led_control;
321 
322 	return led_classdev_register(mmc_dev(mmc), &host->led);
323 }
324 
325 static void sdhci_led_unregister(struct sdhci_host *host)
326 {
327 	led_classdev_unregister(&host->led);
328 }
329 
330 static inline void sdhci_led_activate(struct sdhci_host *host)
331 {
332 }
333 
334 static inline void sdhci_led_deactivate(struct sdhci_host *host)
335 {
336 }
337 
338 #else
339 
340 static inline int sdhci_led_register(struct sdhci_host *host)
341 {
342 	return 0;
343 }
344 
345 static inline void sdhci_led_unregister(struct sdhci_host *host)
346 {
347 }
348 
349 static inline void sdhci_led_activate(struct sdhci_host *host)
350 {
351 	__sdhci_led_activate(host);
352 }
353 
354 static inline void sdhci_led_deactivate(struct sdhci_host *host)
355 {
356 	__sdhci_led_deactivate(host);
357 }
358 
359 #endif
360 
361 /*****************************************************************************\
362  *                                                                           *
363  * Core functions                                                            *
364  *                                                                           *
365 \*****************************************************************************/
366 
367 static void sdhci_read_block_pio(struct sdhci_host *host)
368 {
369 	unsigned long flags;
370 	size_t blksize, len, chunk;
371 	u32 uninitialized_var(scratch);
372 	u8 *buf;
373 
374 	DBG("PIO reading\n");
375 
376 	blksize = host->data->blksz;
377 	chunk = 0;
378 
379 	local_irq_save(flags);
380 
381 	while (blksize) {
382 		BUG_ON(!sg_miter_next(&host->sg_miter));
383 
384 		len = min(host->sg_miter.length, blksize);
385 
386 		blksize -= len;
387 		host->sg_miter.consumed = len;
388 
389 		buf = host->sg_miter.addr;
390 
391 		while (len) {
392 			if (chunk == 0) {
393 				scratch = sdhci_readl(host, SDHCI_BUFFER);
394 				chunk = 4;
395 			}
396 
397 			*buf = scratch & 0xFF;
398 
399 			buf++;
400 			scratch >>= 8;
401 			chunk--;
402 			len--;
403 		}
404 	}
405 
406 	sg_miter_stop(&host->sg_miter);
407 
408 	local_irq_restore(flags);
409 }
410 
411 static void sdhci_write_block_pio(struct sdhci_host *host)
412 {
413 	unsigned long flags;
414 	size_t blksize, len, chunk;
415 	u32 scratch;
416 	u8 *buf;
417 
418 	DBG("PIO writing\n");
419 
420 	blksize = host->data->blksz;
421 	chunk = 0;
422 	scratch = 0;
423 
424 	local_irq_save(flags);
425 
426 	while (blksize) {
427 		BUG_ON(!sg_miter_next(&host->sg_miter));
428 
429 		len = min(host->sg_miter.length, blksize);
430 
431 		blksize -= len;
432 		host->sg_miter.consumed = len;
433 
434 		buf = host->sg_miter.addr;
435 
436 		while (len) {
437 			scratch |= (u32)*buf << (chunk * 8);
438 
439 			buf++;
440 			chunk++;
441 			len--;
442 
443 			if ((chunk == 4) || ((len == 0) && (blksize == 0))) {
444 				sdhci_writel(host, scratch, SDHCI_BUFFER);
445 				chunk = 0;
446 				scratch = 0;
447 			}
448 		}
449 	}
450 
451 	sg_miter_stop(&host->sg_miter);
452 
453 	local_irq_restore(flags);
454 }
455 
456 static void sdhci_transfer_pio(struct sdhci_host *host)
457 {
458 	u32 mask;
459 
460 	if (host->blocks == 0)
461 		return;
462 
463 	if (host->data->flags & MMC_DATA_READ)
464 		mask = SDHCI_DATA_AVAILABLE;
465 	else
466 		mask = SDHCI_SPACE_AVAILABLE;
467 
468 	/*
469 	 * Some controllers (JMicron JMB38x) mess up the buffer bits
470 	 * for transfers < 4 bytes. As long as it is just one block,
471 	 * we can ignore the bits.
472 	 */
473 	if ((host->quirks & SDHCI_QUIRK_BROKEN_SMALL_PIO) &&
474 		(host->data->blocks == 1))
475 		mask = ~0;
476 
477 	while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
478 		if (host->quirks & SDHCI_QUIRK_PIO_NEEDS_DELAY)
479 			udelay(100);
480 
481 		if (host->data->flags & MMC_DATA_READ)
482 			sdhci_read_block_pio(host);
483 		else
484 			sdhci_write_block_pio(host);
485 
486 		host->blocks--;
487 		if (host->blocks == 0)
488 			break;
489 	}
490 
491 	DBG("PIO transfer complete.\n");
492 }
493 
494 static int sdhci_pre_dma_transfer(struct sdhci_host *host,
495 				  struct mmc_data *data, int cookie)
496 {
497 	int sg_count;
498 
499 	/*
500 	 * If the data buffers are already mapped, return the previous
501 	 * dma_map_sg() result.
502 	 */
503 	if (data->host_cookie == COOKIE_PRE_MAPPED)
504 		return data->sg_count;
505 
506 	/* Bounce write requests to the bounce buffer */
507 	if (host->bounce_buffer) {
508 		unsigned int length = data->blksz * data->blocks;
509 
510 		if (length > host->bounce_buffer_size) {
511 			pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
512 			       mmc_hostname(host->mmc), length,
513 			       host->bounce_buffer_size);
514 			return -EIO;
515 		}
516 		if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
517 			/* Copy the data to the bounce buffer */
518 			sg_copy_to_buffer(data->sg, data->sg_len,
519 					  host->bounce_buffer,
520 					  length);
521 		}
522 		/* Switch ownership to the DMA */
523 		dma_sync_single_for_device(host->mmc->parent,
524 					   host->bounce_addr,
525 					   host->bounce_buffer_size,
526 					   mmc_get_dma_dir(data));
527 		/* Just a dummy value */
528 		sg_count = 1;
529 	} else {
530 		/* Just access the data directly from memory */
531 		sg_count = dma_map_sg(mmc_dev(host->mmc),
532 				      data->sg, data->sg_len,
533 				      mmc_get_dma_dir(data));
534 	}
535 
536 	if (sg_count == 0)
537 		return -ENOSPC;
538 
539 	data->sg_count = sg_count;
540 	data->host_cookie = cookie;
541 
542 	return sg_count;
543 }
544 
545 static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags)
546 {
547 	local_irq_save(*flags);
548 	return kmap_atomic(sg_page(sg)) + sg->offset;
549 }
550 
551 static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags)
552 {
553 	kunmap_atomic(buffer);
554 	local_irq_restore(*flags);
555 }
556 
557 static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc,
558 				  dma_addr_t addr, int len, unsigned cmd)
559 {
560 	struct sdhci_adma2_64_desc *dma_desc = desc;
561 
562 	/* 32-bit and 64-bit descriptors have these members in same position */
563 	dma_desc->cmd = cpu_to_le16(cmd);
564 	dma_desc->len = cpu_to_le16(len);
565 	dma_desc->addr_lo = cpu_to_le32((u32)addr);
566 
567 	if (host->flags & SDHCI_USE_64_BIT_DMA)
568 		dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32);
569 }
570 
571 static void sdhci_adma_mark_end(void *desc)
572 {
573 	struct sdhci_adma2_64_desc *dma_desc = desc;
574 
575 	/* 32-bit and 64-bit descriptors have 'cmd' in same position */
576 	dma_desc->cmd |= cpu_to_le16(ADMA2_END);
577 }
578 
579 static void sdhci_adma_table_pre(struct sdhci_host *host,
580 	struct mmc_data *data, int sg_count)
581 {
582 	struct scatterlist *sg;
583 	unsigned long flags;
584 	dma_addr_t addr, align_addr;
585 	void *desc, *align;
586 	char *buffer;
587 	int len, offset, i;
588 
589 	/*
590 	 * The spec does not specify endianness of descriptor table.
591 	 * We currently guess that it is LE.
592 	 */
593 
594 	host->sg_count = sg_count;
595 
596 	desc = host->adma_table;
597 	align = host->align_buffer;
598 
599 	align_addr = host->align_addr;
600 
601 	for_each_sg(data->sg, sg, host->sg_count, i) {
602 		addr = sg_dma_address(sg);
603 		len = sg_dma_len(sg);
604 
605 		/*
606 		 * The SDHCI specification states that ADMA addresses must
607 		 * be 32-bit aligned. If they aren't, then we use a bounce
608 		 * buffer for the (up to three) bytes that screw up the
609 		 * alignment.
610 		 */
611 		offset = (SDHCI_ADMA2_ALIGN - (addr & SDHCI_ADMA2_MASK)) &
612 			 SDHCI_ADMA2_MASK;
613 		if (offset) {
614 			if (data->flags & MMC_DATA_WRITE) {
615 				buffer = sdhci_kmap_atomic(sg, &flags);
616 				memcpy(align, buffer, offset);
617 				sdhci_kunmap_atomic(buffer, &flags);
618 			}
619 
620 			/* tran, valid */
621 			sdhci_adma_write_desc(host, desc, align_addr, offset,
622 					      ADMA2_TRAN_VALID);
623 
624 			BUG_ON(offset > 65536);
625 
626 			align += SDHCI_ADMA2_ALIGN;
627 			align_addr += SDHCI_ADMA2_ALIGN;
628 
629 			desc += host->desc_sz;
630 
631 			addr += offset;
632 			len -= offset;
633 		}
634 
635 		BUG_ON(len > 65536);
636 
637 		if (len) {
638 			/* tran, valid */
639 			sdhci_adma_write_desc(host, desc, addr, len,
640 					      ADMA2_TRAN_VALID);
641 			desc += host->desc_sz;
642 		}
643 
644 		/*
645 		 * If this triggers then we have a calculation bug
646 		 * somewhere. :/
647 		 */
648 		WARN_ON((desc - host->adma_table) >= host->adma_table_sz);
649 	}
650 
651 	if (host->quirks & SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC) {
652 		/* Mark the last descriptor as the terminating descriptor */
653 		if (desc != host->adma_table) {
654 			desc -= host->desc_sz;
655 			sdhci_adma_mark_end(desc);
656 		}
657 	} else {
658 		/* Add a terminating entry - nop, end, valid */
659 		sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID);
660 	}
661 }
662 
663 static void sdhci_adma_table_post(struct sdhci_host *host,
664 	struct mmc_data *data)
665 {
666 	struct scatterlist *sg;
667 	int i, size;
668 	void *align;
669 	char *buffer;
670 	unsigned long flags;
671 
672 	if (data->flags & MMC_DATA_READ) {
673 		bool has_unaligned = false;
674 
675 		/* Do a quick scan of the SG list for any unaligned mappings */
676 		for_each_sg(data->sg, sg, host->sg_count, i)
677 			if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
678 				has_unaligned = true;
679 				break;
680 			}
681 
682 		if (has_unaligned) {
683 			dma_sync_sg_for_cpu(mmc_dev(host->mmc), data->sg,
684 					    data->sg_len, DMA_FROM_DEVICE);
685 
686 			align = host->align_buffer;
687 
688 			for_each_sg(data->sg, sg, host->sg_count, i) {
689 				if (sg_dma_address(sg) & SDHCI_ADMA2_MASK) {
690 					size = SDHCI_ADMA2_ALIGN -
691 					       (sg_dma_address(sg) & SDHCI_ADMA2_MASK);
692 
693 					buffer = sdhci_kmap_atomic(sg, &flags);
694 					memcpy(buffer, align, size);
695 					sdhci_kunmap_atomic(buffer, &flags);
696 
697 					align += SDHCI_ADMA2_ALIGN;
698 				}
699 			}
700 		}
701 	}
702 }
703 
704 static u32 sdhci_sdma_address(struct sdhci_host *host)
705 {
706 	if (host->bounce_buffer)
707 		return host->bounce_addr;
708 	else
709 		return sg_dma_address(host->data->sg);
710 }
711 
712 static unsigned int sdhci_target_timeout(struct sdhci_host *host,
713 					 struct mmc_command *cmd,
714 					 struct mmc_data *data)
715 {
716 	unsigned int target_timeout;
717 
718 	/* timeout in us */
719 	if (!data) {
720 		target_timeout = cmd->busy_timeout * 1000;
721 	} else {
722 		target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000);
723 		if (host->clock && data->timeout_clks) {
724 			unsigned long long val;
725 
726 			/*
727 			 * data->timeout_clks is in units of clock cycles.
728 			 * host->clock is in Hz.  target_timeout is in us.
729 			 * Hence, us = 1000000 * cycles / Hz.  Round up.
730 			 */
731 			val = 1000000ULL * data->timeout_clks;
732 			if (do_div(val, host->clock))
733 				target_timeout++;
734 			target_timeout += val;
735 		}
736 	}
737 
738 	return target_timeout;
739 }
740 
741 static void sdhci_calc_sw_timeout(struct sdhci_host *host,
742 				  struct mmc_command *cmd)
743 {
744 	struct mmc_data *data = cmd->data;
745 	struct mmc_host *mmc = host->mmc;
746 	struct mmc_ios *ios = &mmc->ios;
747 	unsigned char bus_width = 1 << ios->bus_width;
748 	unsigned int blksz;
749 	unsigned int freq;
750 	u64 target_timeout;
751 	u64 transfer_time;
752 
753 	target_timeout = sdhci_target_timeout(host, cmd, data);
754 	target_timeout *= NSEC_PER_USEC;
755 
756 	if (data) {
757 		blksz = data->blksz;
758 		freq = host->mmc->actual_clock ? : host->clock;
759 		transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width);
760 		do_div(transfer_time, freq);
761 		/* multiply by '2' to account for any unknowns */
762 		transfer_time = transfer_time * 2;
763 		/* calculate timeout for the entire data */
764 		host->data_timeout = data->blocks * target_timeout +
765 				     transfer_time;
766 	} else {
767 		host->data_timeout = target_timeout;
768 	}
769 
770 	if (host->data_timeout)
771 		host->data_timeout += MMC_CMD_TRANSFER_TIME;
772 }
773 
774 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd,
775 			     bool *too_big)
776 {
777 	u8 count;
778 	struct mmc_data *data = cmd->data;
779 	unsigned target_timeout, current_timeout;
780 
781 	*too_big = true;
782 
783 	/*
784 	 * If the host controller provides us with an incorrect timeout
785 	 * value, just skip the check and use 0xE.  The hardware may take
786 	 * longer to time out, but that's much better than having a too-short
787 	 * timeout value.
788 	 */
789 	if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL)
790 		return 0xE;
791 
792 	/* Unspecified timeout, assume max */
793 	if (!data && !cmd->busy_timeout)
794 		return 0xE;
795 
796 	/* timeout in us */
797 	target_timeout = sdhci_target_timeout(host, cmd, data);
798 
799 	/*
800 	 * Figure out needed cycles.
801 	 * We do this in steps in order to fit inside a 32 bit int.
802 	 * The first step is the minimum timeout, which will have a
803 	 * minimum resolution of 6 bits:
804 	 * (1) 2^13*1000 > 2^22,
805 	 * (2) host->timeout_clk < 2^16
806 	 *     =>
807 	 *     (1) / (2) > 2^6
808 	 */
809 	count = 0;
810 	current_timeout = (1 << 13) * 1000 / host->timeout_clk;
811 	while (current_timeout < target_timeout) {
812 		count++;
813 		current_timeout <<= 1;
814 		if (count >= 0xF)
815 			break;
816 	}
817 
818 	if (count >= 0xF) {
819 		if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT))
820 			DBG("Too large timeout 0x%x requested for CMD%d!\n",
821 			    count, cmd->opcode);
822 		count = 0xE;
823 	} else {
824 		*too_big = false;
825 	}
826 
827 	return count;
828 }
829 
830 static void sdhci_set_transfer_irqs(struct sdhci_host *host)
831 {
832 	u32 pio_irqs = SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL;
833 	u32 dma_irqs = SDHCI_INT_DMA_END | SDHCI_INT_ADMA_ERROR;
834 
835 	if (host->flags & SDHCI_REQ_USE_DMA)
836 		host->ier = (host->ier & ~pio_irqs) | dma_irqs;
837 	else
838 		host->ier = (host->ier & ~dma_irqs) | pio_irqs;
839 
840 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
841 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
842 }
843 
844 static void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable)
845 {
846 	if (enable)
847 		host->ier |= SDHCI_INT_DATA_TIMEOUT;
848 	else
849 		host->ier &= ~SDHCI_INT_DATA_TIMEOUT;
850 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
851 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
852 }
853 
854 static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd)
855 {
856 	u8 count;
857 
858 	if (host->ops->set_timeout) {
859 		host->ops->set_timeout(host, cmd);
860 	} else {
861 		bool too_big = false;
862 
863 		count = sdhci_calc_timeout(host, cmd, &too_big);
864 
865 		if (too_big &&
866 		    host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) {
867 			sdhci_calc_sw_timeout(host, cmd);
868 			sdhci_set_data_timeout_irq(host, false);
869 		} else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) {
870 			sdhci_set_data_timeout_irq(host, true);
871 		}
872 
873 		sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL);
874 	}
875 }
876 
877 static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
878 {
879 	u8 ctrl;
880 	struct mmc_data *data = cmd->data;
881 
882 	host->data_timeout = 0;
883 
884 	if (sdhci_data_line_cmd(cmd))
885 		sdhci_set_timeout(host, cmd);
886 
887 	if (!data)
888 		return;
889 
890 	WARN_ON(host->data);
891 
892 	/* Sanity checks */
893 	BUG_ON(data->blksz * data->blocks > 524288);
894 	BUG_ON(data->blksz > host->mmc->max_blk_size);
895 	BUG_ON(data->blocks > 65535);
896 
897 	host->data = data;
898 	host->data_early = 0;
899 	host->data->bytes_xfered = 0;
900 
901 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
902 		struct scatterlist *sg;
903 		unsigned int length_mask, offset_mask;
904 		int i;
905 
906 		host->flags |= SDHCI_REQ_USE_DMA;
907 
908 		/*
909 		 * FIXME: This doesn't account for merging when mapping the
910 		 * scatterlist.
911 		 *
912 		 * The assumption here being that alignment and lengths are
913 		 * the same after DMA mapping to device address space.
914 		 */
915 		length_mask = 0;
916 		offset_mask = 0;
917 		if (host->flags & SDHCI_USE_ADMA) {
918 			if (host->quirks & SDHCI_QUIRK_32BIT_ADMA_SIZE) {
919 				length_mask = 3;
920 				/*
921 				 * As we use up to 3 byte chunks to work
922 				 * around alignment problems, we need to
923 				 * check the offset as well.
924 				 */
925 				offset_mask = 3;
926 			}
927 		} else {
928 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE)
929 				length_mask = 3;
930 			if (host->quirks & SDHCI_QUIRK_32BIT_DMA_ADDR)
931 				offset_mask = 3;
932 		}
933 
934 		if (unlikely(length_mask | offset_mask)) {
935 			for_each_sg(data->sg, sg, data->sg_len, i) {
936 				if (sg->length & length_mask) {
937 					DBG("Reverting to PIO because of transfer size (%d)\n",
938 					    sg->length);
939 					host->flags &= ~SDHCI_REQ_USE_DMA;
940 					break;
941 				}
942 				if (sg->offset & offset_mask) {
943 					DBG("Reverting to PIO because of bad alignment\n");
944 					host->flags &= ~SDHCI_REQ_USE_DMA;
945 					break;
946 				}
947 			}
948 		}
949 	}
950 
951 	if (host->flags & SDHCI_REQ_USE_DMA) {
952 		int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED);
953 
954 		if (sg_cnt <= 0) {
955 			/*
956 			 * This only happens when someone fed
957 			 * us an invalid request.
958 			 */
959 			WARN_ON(1);
960 			host->flags &= ~SDHCI_REQ_USE_DMA;
961 		} else if (host->flags & SDHCI_USE_ADMA) {
962 			sdhci_adma_table_pre(host, data, sg_cnt);
963 
964 			sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS);
965 			if (host->flags & SDHCI_USE_64_BIT_DMA)
966 				sdhci_writel(host,
967 					     (u64)host->adma_addr >> 32,
968 					     SDHCI_ADMA_ADDRESS_HI);
969 		} else {
970 			WARN_ON(sg_cnt != 1);
971 			sdhci_writel(host, sdhci_sdma_address(host),
972 				     SDHCI_DMA_ADDRESS);
973 		}
974 	}
975 
976 	/*
977 	 * Always adjust the DMA selection as some controllers
978 	 * (e.g. JMicron) can't do PIO properly when the selection
979 	 * is ADMA.
980 	 */
981 	if (host->version >= SDHCI_SPEC_200) {
982 		ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
983 		ctrl &= ~SDHCI_CTRL_DMA_MASK;
984 		if ((host->flags & SDHCI_REQ_USE_DMA) &&
985 			(host->flags & SDHCI_USE_ADMA)) {
986 			if (host->flags & SDHCI_USE_64_BIT_DMA)
987 				ctrl |= SDHCI_CTRL_ADMA64;
988 			else
989 				ctrl |= SDHCI_CTRL_ADMA32;
990 		} else {
991 			ctrl |= SDHCI_CTRL_SDMA;
992 		}
993 		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
994 	}
995 
996 	if (!(host->flags & SDHCI_REQ_USE_DMA)) {
997 		int flags;
998 
999 		flags = SG_MITER_ATOMIC;
1000 		if (host->data->flags & MMC_DATA_READ)
1001 			flags |= SG_MITER_TO_SG;
1002 		else
1003 			flags |= SG_MITER_FROM_SG;
1004 		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
1005 		host->blocks = data->blocks;
1006 	}
1007 
1008 	sdhci_set_transfer_irqs(host);
1009 
1010 	/* Set the DMA boundary value and block size */
1011 	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
1012 		     SDHCI_BLOCK_SIZE);
1013 	sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
1014 }
1015 
1016 static inline bool sdhci_auto_cmd12(struct sdhci_host *host,
1017 				    struct mmc_request *mrq)
1018 {
1019 	return !mrq->sbc && (host->flags & SDHCI_AUTO_CMD12) &&
1020 	       !mrq->cap_cmd_during_tfr;
1021 }
1022 
1023 static void sdhci_set_transfer_mode(struct sdhci_host *host,
1024 	struct mmc_command *cmd)
1025 {
1026 	u16 mode = 0;
1027 	struct mmc_data *data = cmd->data;
1028 
1029 	if (data == NULL) {
1030 		if (host->quirks2 &
1031 			SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) {
1032 			sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
1033 		} else {
1034 		/* clear Auto CMD settings for no data CMDs */
1035 			mode = sdhci_readw(host, SDHCI_TRANSFER_MODE);
1036 			sdhci_writew(host, mode & ~(SDHCI_TRNS_AUTO_CMD12 |
1037 				SDHCI_TRNS_AUTO_CMD23), SDHCI_TRANSFER_MODE);
1038 		}
1039 		return;
1040 	}
1041 
1042 	WARN_ON(!host->data);
1043 
1044 	if (!(host->quirks2 & SDHCI_QUIRK2_SUPPORT_SINGLE))
1045 		mode = SDHCI_TRNS_BLK_CNT_EN;
1046 
1047 	if (mmc_op_multi(cmd->opcode) || data->blocks > 1) {
1048 		mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI;
1049 		/*
1050 		 * If we are sending CMD23, CMD12 never gets sent
1051 		 * on successful completion (so no Auto-CMD12).
1052 		 */
1053 		if (sdhci_auto_cmd12(host, cmd->mrq) &&
1054 		    (cmd->opcode != SD_IO_RW_EXTENDED))
1055 			mode |= SDHCI_TRNS_AUTO_CMD12;
1056 		else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) {
1057 			mode |= SDHCI_TRNS_AUTO_CMD23;
1058 			sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2);
1059 		}
1060 	}
1061 
1062 	if (data->flags & MMC_DATA_READ)
1063 		mode |= SDHCI_TRNS_READ;
1064 	if (host->flags & SDHCI_REQ_USE_DMA)
1065 		mode |= SDHCI_TRNS_DMA;
1066 
1067 	sdhci_writew(host, mode, SDHCI_TRANSFER_MODE);
1068 }
1069 
1070 static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq)
1071 {
1072 	return (!(host->flags & SDHCI_DEVICE_DEAD) &&
1073 		((mrq->cmd && mrq->cmd->error) ||
1074 		 (mrq->sbc && mrq->sbc->error) ||
1075 		 (mrq->data && ((mrq->data->error && !mrq->data->stop) ||
1076 				(mrq->data->stop && mrq->data->stop->error))) ||
1077 		 (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)));
1078 }
1079 
1080 static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1081 {
1082 	int i;
1083 
1084 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1085 		if (host->mrqs_done[i] == mrq) {
1086 			WARN_ON(1);
1087 			return;
1088 		}
1089 	}
1090 
1091 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
1092 		if (!host->mrqs_done[i]) {
1093 			host->mrqs_done[i] = mrq;
1094 			break;
1095 		}
1096 	}
1097 
1098 	WARN_ON(i >= SDHCI_MAX_MRQS);
1099 
1100 	tasklet_schedule(&host->finish_tasklet);
1101 }
1102 
1103 static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq)
1104 {
1105 	if (host->cmd && host->cmd->mrq == mrq)
1106 		host->cmd = NULL;
1107 
1108 	if (host->data_cmd && host->data_cmd->mrq == mrq)
1109 		host->data_cmd = NULL;
1110 
1111 	if (host->data && host->data->mrq == mrq)
1112 		host->data = NULL;
1113 
1114 	if (sdhci_needs_reset(host, mrq))
1115 		host->pending_reset = true;
1116 
1117 	__sdhci_finish_mrq(host, mrq);
1118 }
1119 
1120 static void sdhci_finish_data(struct sdhci_host *host)
1121 {
1122 	struct mmc_command *data_cmd = host->data_cmd;
1123 	struct mmc_data *data = host->data;
1124 
1125 	host->data = NULL;
1126 	host->data_cmd = NULL;
1127 
1128 	if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
1129 	    (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA))
1130 		sdhci_adma_table_post(host, data);
1131 
1132 	/*
1133 	 * The specification states that the block count register must
1134 	 * be updated, but it does not specify at what point in the
1135 	 * data flow. That makes the register entirely useless to read
1136 	 * back so we have to assume that nothing made it to the card
1137 	 * in the event of an error.
1138 	 */
1139 	if (data->error)
1140 		data->bytes_xfered = 0;
1141 	else
1142 		data->bytes_xfered = data->blksz * data->blocks;
1143 
1144 	/*
1145 	 * Need to send CMD12 if -
1146 	 * a) open-ended multiblock transfer (no CMD23)
1147 	 * b) error in multiblock transfer
1148 	 */
1149 	if (data->stop &&
1150 	    (data->error ||
1151 	     !data->mrq->sbc)) {
1152 
1153 		/*
1154 		 * The controller needs a reset of internal state machines
1155 		 * upon error conditions.
1156 		 */
1157 		if (data->error) {
1158 			if (!host->cmd || host->cmd == data_cmd)
1159 				sdhci_do_reset(host, SDHCI_RESET_CMD);
1160 			sdhci_do_reset(host, SDHCI_RESET_DATA);
1161 		}
1162 
1163 		/*
1164 		 * 'cap_cmd_during_tfr' request must not use the command line
1165 		 * after mmc_command_done() has been called. It is upper layer's
1166 		 * responsibility to send the stop command if required.
1167 		 */
1168 		if (data->mrq->cap_cmd_during_tfr) {
1169 			sdhci_finish_mrq(host, data->mrq);
1170 		} else {
1171 			/* Avoid triggering warning in sdhci_send_command() */
1172 			host->cmd = NULL;
1173 			sdhci_send_command(host, data->stop);
1174 		}
1175 	} else {
1176 		sdhci_finish_mrq(host, data->mrq);
1177 	}
1178 }
1179 
1180 static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq,
1181 			    unsigned long timeout)
1182 {
1183 	if (sdhci_data_line_cmd(mrq->cmd))
1184 		mod_timer(&host->data_timer, timeout);
1185 	else
1186 		mod_timer(&host->timer, timeout);
1187 }
1188 
1189 static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq)
1190 {
1191 	if (sdhci_data_line_cmd(mrq->cmd))
1192 		del_timer(&host->data_timer);
1193 	else
1194 		del_timer(&host->timer);
1195 }
1196 
1197 void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
1198 {
1199 	int flags;
1200 	u32 mask;
1201 	unsigned long timeout;
1202 
1203 	WARN_ON(host->cmd);
1204 
1205 	/* Initially, a command has no error */
1206 	cmd->error = 0;
1207 
1208 	if ((host->quirks2 & SDHCI_QUIRK2_STOP_WITH_TC) &&
1209 	    cmd->opcode == MMC_STOP_TRANSMISSION)
1210 		cmd->flags |= MMC_RSP_BUSY;
1211 
1212 	/* Wait max 10 ms */
1213 	timeout = 10;
1214 
1215 	mask = SDHCI_CMD_INHIBIT;
1216 	if (sdhci_data_line_cmd(cmd))
1217 		mask |= SDHCI_DATA_INHIBIT;
1218 
1219 	/* We shouldn't wait for data inihibit for stop commands, even
1220 	   though they might use busy signaling */
1221 	if (cmd->mrq->data && (cmd == cmd->mrq->data->stop))
1222 		mask &= ~SDHCI_DATA_INHIBIT;
1223 
1224 	while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) {
1225 		if (timeout == 0) {
1226 			pr_err("%s: Controller never released inhibit bit(s).\n",
1227 			       mmc_hostname(host->mmc));
1228 			sdhci_dumpregs(host);
1229 			cmd->error = -EIO;
1230 			sdhci_finish_mrq(host, cmd->mrq);
1231 			return;
1232 		}
1233 		timeout--;
1234 		mdelay(1);
1235 	}
1236 
1237 	host->cmd = cmd;
1238 	if (sdhci_data_line_cmd(cmd)) {
1239 		WARN_ON(host->data_cmd);
1240 		host->data_cmd = cmd;
1241 	}
1242 
1243 	sdhci_prepare_data(host, cmd);
1244 
1245 	sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT);
1246 
1247 	sdhci_set_transfer_mode(host, cmd);
1248 
1249 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1250 		pr_err("%s: Unsupported response type!\n",
1251 			mmc_hostname(host->mmc));
1252 		cmd->error = -EINVAL;
1253 		sdhci_finish_mrq(host, cmd->mrq);
1254 		return;
1255 	}
1256 
1257 	if (!(cmd->flags & MMC_RSP_PRESENT))
1258 		flags = SDHCI_CMD_RESP_NONE;
1259 	else if (cmd->flags & MMC_RSP_136)
1260 		flags = SDHCI_CMD_RESP_LONG;
1261 	else if (cmd->flags & MMC_RSP_BUSY)
1262 		flags = SDHCI_CMD_RESP_SHORT_BUSY;
1263 	else
1264 		flags = SDHCI_CMD_RESP_SHORT;
1265 
1266 	if (cmd->flags & MMC_RSP_CRC)
1267 		flags |= SDHCI_CMD_CRC;
1268 	if (cmd->flags & MMC_RSP_OPCODE)
1269 		flags |= SDHCI_CMD_INDEX;
1270 
1271 	/* CMD19 is special in that the Data Present Select should be set */
1272 	if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1273 	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200)
1274 		flags |= SDHCI_CMD_DATA;
1275 
1276 	timeout = jiffies;
1277 	if (host->data_timeout)
1278 		timeout += nsecs_to_jiffies(host->data_timeout);
1279 	else if (!cmd->data && cmd->busy_timeout > 9000)
1280 		timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
1281 	else
1282 		timeout += 10 * HZ;
1283 	sdhci_mod_timer(host, cmd->mrq, timeout);
1284 
1285 	sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND);
1286 }
1287 EXPORT_SYMBOL_GPL(sdhci_send_command);
1288 
1289 static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
1290 {
1291 	int i, reg;
1292 
1293 	for (i = 0; i < 4; i++) {
1294 		reg = SDHCI_RESPONSE + (3 - i) * 4;
1295 		cmd->resp[i] = sdhci_readl(host, reg);
1296 	}
1297 
1298 	if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
1299 		return;
1300 
1301 	/* CRC is stripped so we need to do some shifting */
1302 	for (i = 0; i < 4; i++) {
1303 		cmd->resp[i] <<= 8;
1304 		if (i != 3)
1305 			cmd->resp[i] |= cmd->resp[i + 1] >> 24;
1306 	}
1307 }
1308 
1309 static void sdhci_finish_command(struct sdhci_host *host)
1310 {
1311 	struct mmc_command *cmd = host->cmd;
1312 
1313 	host->cmd = NULL;
1314 
1315 	if (cmd->flags & MMC_RSP_PRESENT) {
1316 		if (cmd->flags & MMC_RSP_136) {
1317 			sdhci_read_rsp_136(host, cmd);
1318 		} else {
1319 			cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
1320 		}
1321 	}
1322 
1323 	if (cmd->mrq->cap_cmd_during_tfr && cmd == cmd->mrq->cmd)
1324 		mmc_command_done(host->mmc, cmd->mrq);
1325 
1326 	/*
1327 	 * The host can send and interrupt when the busy state has
1328 	 * ended, allowing us to wait without wasting CPU cycles.
1329 	 * The busy signal uses DAT0 so this is similar to waiting
1330 	 * for data to complete.
1331 	 *
1332 	 * Note: The 1.0 specification is a bit ambiguous about this
1333 	 *       feature so there might be some problems with older
1334 	 *       controllers.
1335 	 */
1336 	if (cmd->flags & MMC_RSP_BUSY) {
1337 		if (cmd->data) {
1338 			DBG("Cannot wait for busy signal when also doing a data transfer");
1339 		} else if (!(host->quirks & SDHCI_QUIRK_NO_BUSY_IRQ) &&
1340 			   cmd == host->data_cmd) {
1341 			/* Command complete before busy is ended */
1342 			return;
1343 		}
1344 	}
1345 
1346 	/* Finished CMD23, now send actual command. */
1347 	if (cmd == cmd->mrq->sbc) {
1348 		sdhci_send_command(host, cmd->mrq->cmd);
1349 	} else {
1350 
1351 		/* Processed actual command. */
1352 		if (host->data && host->data_early)
1353 			sdhci_finish_data(host);
1354 
1355 		if (!cmd->data)
1356 			sdhci_finish_mrq(host, cmd->mrq);
1357 	}
1358 }
1359 
1360 static u16 sdhci_get_preset_value(struct sdhci_host *host)
1361 {
1362 	u16 preset = 0;
1363 
1364 	switch (host->timing) {
1365 	case MMC_TIMING_UHS_SDR12:
1366 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1367 		break;
1368 	case MMC_TIMING_UHS_SDR25:
1369 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR25);
1370 		break;
1371 	case MMC_TIMING_UHS_SDR50:
1372 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR50);
1373 		break;
1374 	case MMC_TIMING_UHS_SDR104:
1375 	case MMC_TIMING_MMC_HS200:
1376 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR104);
1377 		break;
1378 	case MMC_TIMING_UHS_DDR50:
1379 	case MMC_TIMING_MMC_DDR52:
1380 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_DDR50);
1381 		break;
1382 	case MMC_TIMING_MMC_HS400:
1383 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400);
1384 		break;
1385 	default:
1386 		pr_warn("%s: Invalid UHS-I mode selected\n",
1387 			mmc_hostname(host->mmc));
1388 		preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12);
1389 		break;
1390 	}
1391 	return preset;
1392 }
1393 
1394 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
1395 		   unsigned int *actual_clock)
1396 {
1397 	int div = 0; /* Initialized for compiler warning */
1398 	int real_div = div, clk_mul = 1;
1399 	u16 clk = 0;
1400 	bool switch_base_clk = false;
1401 
1402 	if (host->version >= SDHCI_SPEC_300) {
1403 		if (host->preset_enabled) {
1404 			u16 pre_val;
1405 
1406 			clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1407 			pre_val = sdhci_get_preset_value(host);
1408 			div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK)
1409 				>> SDHCI_PRESET_SDCLK_FREQ_SHIFT;
1410 			if (host->clk_mul &&
1411 				(pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) {
1412 				clk = SDHCI_PROG_CLOCK_MODE;
1413 				real_div = div + 1;
1414 				clk_mul = host->clk_mul;
1415 			} else {
1416 				real_div = max_t(int, 1, div << 1);
1417 			}
1418 			goto clock_set;
1419 		}
1420 
1421 		/*
1422 		 * Check if the Host Controller supports Programmable Clock
1423 		 * Mode.
1424 		 */
1425 		if (host->clk_mul) {
1426 			for (div = 1; div <= 1024; div++) {
1427 				if ((host->max_clk * host->clk_mul / div)
1428 					<= clock)
1429 					break;
1430 			}
1431 			if ((host->max_clk * host->clk_mul / div) <= clock) {
1432 				/*
1433 				 * Set Programmable Clock Mode in the Clock
1434 				 * Control register.
1435 				 */
1436 				clk = SDHCI_PROG_CLOCK_MODE;
1437 				real_div = div;
1438 				clk_mul = host->clk_mul;
1439 				div--;
1440 			} else {
1441 				/*
1442 				 * Divisor can be too small to reach clock
1443 				 * speed requirement. Then use the base clock.
1444 				 */
1445 				switch_base_clk = true;
1446 			}
1447 		}
1448 
1449 		if (!host->clk_mul || switch_base_clk) {
1450 			/* Version 3.00 divisors must be a multiple of 2. */
1451 			if (host->max_clk <= clock)
1452 				div = 1;
1453 			else {
1454 				for (div = 2; div < SDHCI_MAX_DIV_SPEC_300;
1455 				     div += 2) {
1456 					if ((host->max_clk / div) <= clock)
1457 						break;
1458 				}
1459 			}
1460 			real_div = div;
1461 			div >>= 1;
1462 			if ((host->quirks2 & SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN)
1463 				&& !div && host->max_clk <= 25000000)
1464 				div = 1;
1465 		}
1466 	} else {
1467 		/* Version 2.00 divisors must be a power of 2. */
1468 		for (div = 1; div < SDHCI_MAX_DIV_SPEC_200; div *= 2) {
1469 			if ((host->max_clk / div) <= clock)
1470 				break;
1471 		}
1472 		real_div = div;
1473 		div >>= 1;
1474 	}
1475 
1476 clock_set:
1477 	if (real_div)
1478 		*actual_clock = (host->max_clk * clk_mul) / real_div;
1479 	clk |= (div & SDHCI_DIV_MASK) << SDHCI_DIVIDER_SHIFT;
1480 	clk |= ((div & SDHCI_DIV_HI_MASK) >> SDHCI_DIV_MASK_LEN)
1481 		<< SDHCI_DIVIDER_HI_SHIFT;
1482 
1483 	return clk;
1484 }
1485 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
1486 
1487 void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
1488 {
1489 	ktime_t timeout;
1490 
1491 	clk |= SDHCI_CLOCK_INT_EN;
1492 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1493 
1494 	/* Wait max 20 ms */
1495 	timeout = ktime_add_ms(ktime_get(), 20);
1496 	while (!((clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL))
1497 		& SDHCI_CLOCK_INT_STABLE)) {
1498 		if (ktime_after(ktime_get(), timeout)) {
1499 			pr_err("%s: Internal clock never stabilised.\n",
1500 			       mmc_hostname(host->mmc));
1501 			sdhci_dumpregs(host);
1502 			return;
1503 		}
1504 		udelay(10);
1505 	}
1506 
1507 	clk |= SDHCI_CLOCK_CARD_EN;
1508 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1509 }
1510 EXPORT_SYMBOL_GPL(sdhci_enable_clk);
1511 
1512 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
1513 {
1514 	u16 clk;
1515 
1516 	host->mmc->actual_clock = 0;
1517 
1518 	sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
1519 
1520 	if (clock == 0)
1521 		return;
1522 
1523 	clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
1524 	sdhci_enable_clk(host, clk);
1525 }
1526 EXPORT_SYMBOL_GPL(sdhci_set_clock);
1527 
1528 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
1529 				unsigned short vdd)
1530 {
1531 	struct mmc_host *mmc = host->mmc;
1532 
1533 	mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
1534 
1535 	if (mode != MMC_POWER_OFF)
1536 		sdhci_writeb(host, SDHCI_POWER_ON, SDHCI_POWER_CONTROL);
1537 	else
1538 		sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1539 }
1540 
1541 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
1542 			   unsigned short vdd)
1543 {
1544 	u8 pwr = 0;
1545 
1546 	if (mode != MMC_POWER_OFF) {
1547 		switch (1 << vdd) {
1548 		case MMC_VDD_165_195:
1549 		/*
1550 		 * Without a regulator, SDHCI does not support 2.0v
1551 		 * so we only get here if the driver deliberately
1552 		 * added the 2.0v range to ocr_avail. Map it to 1.8v
1553 		 * for the purpose of turning on the power.
1554 		 */
1555 		case MMC_VDD_20_21:
1556 			pwr = SDHCI_POWER_180;
1557 			break;
1558 		case MMC_VDD_29_30:
1559 		case MMC_VDD_30_31:
1560 			pwr = SDHCI_POWER_300;
1561 			break;
1562 		case MMC_VDD_32_33:
1563 		case MMC_VDD_33_34:
1564 			pwr = SDHCI_POWER_330;
1565 			break;
1566 		default:
1567 			WARN(1, "%s: Invalid vdd %#x\n",
1568 			     mmc_hostname(host->mmc), vdd);
1569 			break;
1570 		}
1571 	}
1572 
1573 	if (host->pwr == pwr)
1574 		return;
1575 
1576 	host->pwr = pwr;
1577 
1578 	if (pwr == 0) {
1579 		sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1580 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1581 			sdhci_runtime_pm_bus_off(host);
1582 	} else {
1583 		/*
1584 		 * Spec says that we should clear the power reg before setting
1585 		 * a new value. Some controllers don't seem to like this though.
1586 		 */
1587 		if (!(host->quirks & SDHCI_QUIRK_SINGLE_POWER_WRITE))
1588 			sdhci_writeb(host, 0, SDHCI_POWER_CONTROL);
1589 
1590 		/*
1591 		 * At least the Marvell CaFe chip gets confused if we set the
1592 		 * voltage and set turn on power at the same time, so set the
1593 		 * voltage first.
1594 		 */
1595 		if (host->quirks & SDHCI_QUIRK_NO_SIMULT_VDD_AND_POWER)
1596 			sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1597 
1598 		pwr |= SDHCI_POWER_ON;
1599 
1600 		sdhci_writeb(host, pwr, SDHCI_POWER_CONTROL);
1601 
1602 		if (host->quirks2 & SDHCI_QUIRK2_CARD_ON_NEEDS_BUS_ON)
1603 			sdhci_runtime_pm_bus_on(host);
1604 
1605 		/*
1606 		 * Some controllers need an extra 10ms delay of 10ms before
1607 		 * they can apply clock after applying power
1608 		 */
1609 		if (host->quirks & SDHCI_QUIRK_DELAY_AFTER_POWER)
1610 			mdelay(10);
1611 	}
1612 }
1613 EXPORT_SYMBOL_GPL(sdhci_set_power_noreg);
1614 
1615 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
1616 		     unsigned short vdd)
1617 {
1618 	if (IS_ERR(host->mmc->supply.vmmc))
1619 		sdhci_set_power_noreg(host, mode, vdd);
1620 	else
1621 		sdhci_set_power_reg(host, mode, vdd);
1622 }
1623 EXPORT_SYMBOL_GPL(sdhci_set_power);
1624 
1625 /*****************************************************************************\
1626  *                                                                           *
1627  * MMC callbacks                                                             *
1628  *                                                                           *
1629 \*****************************************************************************/
1630 
1631 static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1632 {
1633 	struct sdhci_host *host;
1634 	int present;
1635 	unsigned long flags;
1636 
1637 	host = mmc_priv(mmc);
1638 
1639 	/* Firstly check card presence */
1640 	present = mmc->ops->get_cd(mmc);
1641 
1642 	spin_lock_irqsave(&host->lock, flags);
1643 
1644 	sdhci_led_activate(host);
1645 
1646 	/*
1647 	 * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED
1648 	 * requests if Auto-CMD12 is enabled.
1649 	 */
1650 	if (sdhci_auto_cmd12(host, mrq)) {
1651 		if (mrq->stop) {
1652 			mrq->data->stop = NULL;
1653 			mrq->stop = NULL;
1654 		}
1655 	}
1656 
1657 	if (!present || host->flags & SDHCI_DEVICE_DEAD) {
1658 		mrq->cmd->error = -ENOMEDIUM;
1659 		sdhci_finish_mrq(host, mrq);
1660 	} else {
1661 		if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23))
1662 			sdhci_send_command(host, mrq->sbc);
1663 		else
1664 			sdhci_send_command(host, mrq->cmd);
1665 	}
1666 
1667 	mmiowb();
1668 	spin_unlock_irqrestore(&host->lock, flags);
1669 }
1670 
1671 void sdhci_set_bus_width(struct sdhci_host *host, int width)
1672 {
1673 	u8 ctrl;
1674 
1675 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1676 	if (width == MMC_BUS_WIDTH_8) {
1677 		ctrl &= ~SDHCI_CTRL_4BITBUS;
1678 		ctrl |= SDHCI_CTRL_8BITBUS;
1679 	} else {
1680 		if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
1681 			ctrl &= ~SDHCI_CTRL_8BITBUS;
1682 		if (width == MMC_BUS_WIDTH_4)
1683 			ctrl |= SDHCI_CTRL_4BITBUS;
1684 		else
1685 			ctrl &= ~SDHCI_CTRL_4BITBUS;
1686 	}
1687 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1688 }
1689 EXPORT_SYMBOL_GPL(sdhci_set_bus_width);
1690 
1691 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
1692 {
1693 	u16 ctrl_2;
1694 
1695 	ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1696 	/* Select Bus Speed Mode for host */
1697 	ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
1698 	if ((timing == MMC_TIMING_MMC_HS200) ||
1699 	    (timing == MMC_TIMING_UHS_SDR104))
1700 		ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
1701 	else if (timing == MMC_TIMING_UHS_SDR12)
1702 		ctrl_2 |= SDHCI_CTRL_UHS_SDR12;
1703 	else if (timing == MMC_TIMING_UHS_SDR25)
1704 		ctrl_2 |= SDHCI_CTRL_UHS_SDR25;
1705 	else if (timing == MMC_TIMING_UHS_SDR50)
1706 		ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
1707 	else if ((timing == MMC_TIMING_UHS_DDR50) ||
1708 		 (timing == MMC_TIMING_MMC_DDR52))
1709 		ctrl_2 |= SDHCI_CTRL_UHS_DDR50;
1710 	else if (timing == MMC_TIMING_MMC_HS400)
1711 		ctrl_2 |= SDHCI_CTRL_HS400; /* Non-standard */
1712 	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1713 }
1714 EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling);
1715 
1716 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1717 {
1718 	struct sdhci_host *host = mmc_priv(mmc);
1719 	u8 ctrl;
1720 
1721 	if (ios->power_mode == MMC_POWER_UNDEFINED)
1722 		return;
1723 
1724 	if (host->flags & SDHCI_DEVICE_DEAD) {
1725 		if (!IS_ERR(mmc->supply.vmmc) &&
1726 		    ios->power_mode == MMC_POWER_OFF)
1727 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
1728 		return;
1729 	}
1730 
1731 	/*
1732 	 * Reset the chip on each power off.
1733 	 * Should clear out any weird states.
1734 	 */
1735 	if (ios->power_mode == MMC_POWER_OFF) {
1736 		sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
1737 		sdhci_reinit(host);
1738 	}
1739 
1740 	if (host->version >= SDHCI_SPEC_300 &&
1741 		(ios->power_mode == MMC_POWER_UP) &&
1742 		!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN))
1743 		sdhci_enable_preset_value(host, false);
1744 
1745 	if (!ios->clock || ios->clock != host->clock) {
1746 		host->ops->set_clock(host, ios->clock);
1747 		host->clock = ios->clock;
1748 
1749 		if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK &&
1750 		    host->clock) {
1751 			host->timeout_clk = host->mmc->actual_clock ?
1752 						host->mmc->actual_clock / 1000 :
1753 						host->clock / 1000;
1754 			host->mmc->max_busy_timeout =
1755 				host->ops->get_max_timeout_count ?
1756 				host->ops->get_max_timeout_count(host) :
1757 				1 << 27;
1758 			host->mmc->max_busy_timeout /= host->timeout_clk;
1759 		}
1760 	}
1761 
1762 	if (host->ops->set_power)
1763 		host->ops->set_power(host, ios->power_mode, ios->vdd);
1764 	else
1765 		sdhci_set_power(host, ios->power_mode, ios->vdd);
1766 
1767 	if (host->ops->platform_send_init_74_clocks)
1768 		host->ops->platform_send_init_74_clocks(host, ios->power_mode);
1769 
1770 	host->ops->set_bus_width(host, ios->bus_width);
1771 
1772 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
1773 
1774 	if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
1775 		if (ios->timing == MMC_TIMING_SD_HS ||
1776 		     ios->timing == MMC_TIMING_MMC_HS ||
1777 		     ios->timing == MMC_TIMING_MMC_HS400 ||
1778 		     ios->timing == MMC_TIMING_MMC_HS200 ||
1779 		     ios->timing == MMC_TIMING_MMC_DDR52 ||
1780 		     ios->timing == MMC_TIMING_UHS_SDR50 ||
1781 		     ios->timing == MMC_TIMING_UHS_SDR104 ||
1782 		     ios->timing == MMC_TIMING_UHS_DDR50 ||
1783 		     ios->timing == MMC_TIMING_UHS_SDR25)
1784 			ctrl |= SDHCI_CTRL_HISPD;
1785 		else
1786 			ctrl &= ~SDHCI_CTRL_HISPD;
1787 	}
1788 
1789 	if (host->version >= SDHCI_SPEC_300) {
1790 		u16 clk, ctrl_2;
1791 
1792 		if (!host->preset_enabled) {
1793 			sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1794 			/*
1795 			 * We only need to set Driver Strength if the
1796 			 * preset value enable is not set.
1797 			 */
1798 			ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
1799 			ctrl_2 &= ~SDHCI_CTRL_DRV_TYPE_MASK;
1800 			if (ios->drv_type == MMC_SET_DRIVER_TYPE_A)
1801 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_A;
1802 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_B)
1803 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1804 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_C)
1805 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_C;
1806 			else if (ios->drv_type == MMC_SET_DRIVER_TYPE_D)
1807 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_D;
1808 			else {
1809 				pr_warn("%s: invalid driver type, default to driver type B\n",
1810 					mmc_hostname(mmc));
1811 				ctrl_2 |= SDHCI_CTRL_DRV_TYPE_B;
1812 			}
1813 
1814 			sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
1815 		} else {
1816 			/*
1817 			 * According to SDHC Spec v3.00, if the Preset Value
1818 			 * Enable in the Host Control 2 register is set, we
1819 			 * need to reset SD Clock Enable before changing High
1820 			 * Speed Enable to avoid generating clock gliches.
1821 			 */
1822 
1823 			/* Reset SD Clock Enable */
1824 			clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1825 			clk &= ~SDHCI_CLOCK_CARD_EN;
1826 			sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1827 
1828 			sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1829 
1830 			/* Re-enable SD Clock */
1831 			host->ops->set_clock(host, host->clock);
1832 		}
1833 
1834 		/* Reset SD Clock Enable */
1835 		clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
1836 		clk &= ~SDHCI_CLOCK_CARD_EN;
1837 		sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
1838 
1839 		host->ops->set_uhs_signaling(host, ios->timing);
1840 		host->timing = ios->timing;
1841 
1842 		if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) &&
1843 				((ios->timing == MMC_TIMING_UHS_SDR12) ||
1844 				 (ios->timing == MMC_TIMING_UHS_SDR25) ||
1845 				 (ios->timing == MMC_TIMING_UHS_SDR50) ||
1846 				 (ios->timing == MMC_TIMING_UHS_SDR104) ||
1847 				 (ios->timing == MMC_TIMING_UHS_DDR50) ||
1848 				 (ios->timing == MMC_TIMING_MMC_DDR52))) {
1849 			u16 preset;
1850 
1851 			sdhci_enable_preset_value(host, true);
1852 			preset = sdhci_get_preset_value(host);
1853 			ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK)
1854 				>> SDHCI_PRESET_DRV_SHIFT;
1855 		}
1856 
1857 		/* Re-enable SD Clock */
1858 		host->ops->set_clock(host, host->clock);
1859 	} else
1860 		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
1861 
1862 	/*
1863 	 * Some (ENE) controllers go apeshit on some ios operation,
1864 	 * signalling timeout and CRC errors even on CMD0. Resetting
1865 	 * it on each ios seems to solve the problem.
1866 	 */
1867 	if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
1868 		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1869 
1870 	mmiowb();
1871 }
1872 EXPORT_SYMBOL_GPL(sdhci_set_ios);
1873 
1874 static int sdhci_get_cd(struct mmc_host *mmc)
1875 {
1876 	struct sdhci_host *host = mmc_priv(mmc);
1877 	int gpio_cd = mmc_gpio_get_cd(mmc);
1878 
1879 	if (host->flags & SDHCI_DEVICE_DEAD)
1880 		return 0;
1881 
1882 	/* If nonremovable, assume that the card is always present. */
1883 	if (!mmc_card_is_removable(host->mmc))
1884 		return 1;
1885 
1886 	/*
1887 	 * Try slot gpio detect, if defined it take precedence
1888 	 * over build in controller functionality
1889 	 */
1890 	if (gpio_cd >= 0)
1891 		return !!gpio_cd;
1892 
1893 	/* If polling, assume that the card is always present. */
1894 	if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)
1895 		return 1;
1896 
1897 	/* Host native card detect */
1898 	return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1899 }
1900 
1901 static int sdhci_check_ro(struct sdhci_host *host)
1902 {
1903 	unsigned long flags;
1904 	int is_readonly;
1905 
1906 	spin_lock_irqsave(&host->lock, flags);
1907 
1908 	if (host->flags & SDHCI_DEVICE_DEAD)
1909 		is_readonly = 0;
1910 	else if (host->ops->get_ro)
1911 		is_readonly = host->ops->get_ro(host);
1912 	else
1913 		is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE)
1914 				& SDHCI_WRITE_PROTECT);
1915 
1916 	spin_unlock_irqrestore(&host->lock, flags);
1917 
1918 	/* This quirk needs to be replaced by a callback-function later */
1919 	return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ?
1920 		!is_readonly : is_readonly;
1921 }
1922 
1923 #define SAMPLE_COUNT	5
1924 
1925 static int sdhci_get_ro(struct mmc_host *mmc)
1926 {
1927 	struct sdhci_host *host = mmc_priv(mmc);
1928 	int i, ro_count;
1929 
1930 	if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT))
1931 		return sdhci_check_ro(host);
1932 
1933 	ro_count = 0;
1934 	for (i = 0; i < SAMPLE_COUNT; i++) {
1935 		if (sdhci_check_ro(host)) {
1936 			if (++ro_count > SAMPLE_COUNT / 2)
1937 				return 1;
1938 		}
1939 		msleep(30);
1940 	}
1941 	return 0;
1942 }
1943 
1944 static void sdhci_hw_reset(struct mmc_host *mmc)
1945 {
1946 	struct sdhci_host *host = mmc_priv(mmc);
1947 
1948 	if (host->ops && host->ops->hw_reset)
1949 		host->ops->hw_reset(host);
1950 }
1951 
1952 static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable)
1953 {
1954 	if (!(host->flags & SDHCI_DEVICE_DEAD)) {
1955 		if (enable)
1956 			host->ier |= SDHCI_INT_CARD_INT;
1957 		else
1958 			host->ier &= ~SDHCI_INT_CARD_INT;
1959 
1960 		sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
1961 		sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
1962 		mmiowb();
1963 	}
1964 }
1965 
1966 void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1967 {
1968 	struct sdhci_host *host = mmc_priv(mmc);
1969 	unsigned long flags;
1970 
1971 	if (enable)
1972 		pm_runtime_get_noresume(host->mmc->parent);
1973 
1974 	spin_lock_irqsave(&host->lock, flags);
1975 	if (enable)
1976 		host->flags |= SDHCI_SDIO_IRQ_ENABLED;
1977 	else
1978 		host->flags &= ~SDHCI_SDIO_IRQ_ENABLED;
1979 
1980 	sdhci_enable_sdio_irq_nolock(host, enable);
1981 	spin_unlock_irqrestore(&host->lock, flags);
1982 
1983 	if (!enable)
1984 		pm_runtime_put_noidle(host->mmc->parent);
1985 }
1986 EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq);
1987 
1988 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
1989 				      struct mmc_ios *ios)
1990 {
1991 	struct sdhci_host *host = mmc_priv(mmc);
1992 	u16 ctrl;
1993 	int ret;
1994 
1995 	/*
1996 	 * Signal Voltage Switching is only applicable for Host Controllers
1997 	 * v3.00 and above.
1998 	 */
1999 	if (host->version < SDHCI_SPEC_300)
2000 		return 0;
2001 
2002 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2003 
2004 	switch (ios->signal_voltage) {
2005 	case MMC_SIGNAL_VOLTAGE_330:
2006 		if (!(host->flags & SDHCI_SIGNALING_330))
2007 			return -EINVAL;
2008 		/* Set 1.8V Signal Enable in the Host Control2 register to 0 */
2009 		ctrl &= ~SDHCI_CTRL_VDD_180;
2010 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2011 
2012 		if (!IS_ERR(mmc->supply.vqmmc)) {
2013 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2014 			if (ret) {
2015 				pr_warn("%s: Switching to 3.3V signalling voltage failed\n",
2016 					mmc_hostname(mmc));
2017 				return -EIO;
2018 			}
2019 		}
2020 		/* Wait for 5ms */
2021 		usleep_range(5000, 5500);
2022 
2023 		/* 3.3V regulator output should be stable within 5 ms */
2024 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2025 		if (!(ctrl & SDHCI_CTRL_VDD_180))
2026 			return 0;
2027 
2028 		pr_warn("%s: 3.3V regulator output did not became stable\n",
2029 			mmc_hostname(mmc));
2030 
2031 		return -EAGAIN;
2032 	case MMC_SIGNAL_VOLTAGE_180:
2033 		if (!(host->flags & SDHCI_SIGNALING_180))
2034 			return -EINVAL;
2035 		if (!IS_ERR(mmc->supply.vqmmc)) {
2036 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2037 			if (ret) {
2038 				pr_warn("%s: Switching to 1.8V signalling voltage failed\n",
2039 					mmc_hostname(mmc));
2040 				return -EIO;
2041 			}
2042 		}
2043 
2044 		/*
2045 		 * Enable 1.8V Signal Enable in the Host Control2
2046 		 * register
2047 		 */
2048 		ctrl |= SDHCI_CTRL_VDD_180;
2049 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2050 
2051 		/* Some controller need to do more when switching */
2052 		if (host->ops->voltage_switch)
2053 			host->ops->voltage_switch(host);
2054 
2055 		/* 1.8V regulator output should be stable within 5 ms */
2056 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2057 		if (ctrl & SDHCI_CTRL_VDD_180)
2058 			return 0;
2059 
2060 		pr_warn("%s: 1.8V regulator output did not became stable\n",
2061 			mmc_hostname(mmc));
2062 
2063 		return -EAGAIN;
2064 	case MMC_SIGNAL_VOLTAGE_120:
2065 		if (!(host->flags & SDHCI_SIGNALING_120))
2066 			return -EINVAL;
2067 		if (!IS_ERR(mmc->supply.vqmmc)) {
2068 			ret = mmc_regulator_set_vqmmc(mmc, ios);
2069 			if (ret) {
2070 				pr_warn("%s: Switching to 1.2V signalling voltage failed\n",
2071 					mmc_hostname(mmc));
2072 				return -EIO;
2073 			}
2074 		}
2075 		return 0;
2076 	default:
2077 		/* No signal voltage switch required */
2078 		return 0;
2079 	}
2080 }
2081 EXPORT_SYMBOL_GPL(sdhci_start_signal_voltage_switch);
2082 
2083 static int sdhci_card_busy(struct mmc_host *mmc)
2084 {
2085 	struct sdhci_host *host = mmc_priv(mmc);
2086 	u32 present_state;
2087 
2088 	/* Check whether DAT[0] is 0 */
2089 	present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
2090 
2091 	return !(present_state & SDHCI_DATA_0_LVL_MASK);
2092 }
2093 
2094 static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
2095 {
2096 	struct sdhci_host *host = mmc_priv(mmc);
2097 	unsigned long flags;
2098 
2099 	spin_lock_irqsave(&host->lock, flags);
2100 	host->flags |= SDHCI_HS400_TUNING;
2101 	spin_unlock_irqrestore(&host->lock, flags);
2102 
2103 	return 0;
2104 }
2105 
2106 static void sdhci_start_tuning(struct sdhci_host *host)
2107 {
2108 	u16 ctrl;
2109 
2110 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2111 	ctrl |= SDHCI_CTRL_EXEC_TUNING;
2112 	if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
2113 		ctrl |= SDHCI_CTRL_TUNED_CLK;
2114 	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2115 
2116 	/*
2117 	 * As per the Host Controller spec v3.00, tuning command
2118 	 * generates Buffer Read Ready interrupt, so enable that.
2119 	 *
2120 	 * Note: The spec clearly says that when tuning sequence
2121 	 * is being performed, the controller does not generate
2122 	 * interrupts other than Buffer Read Ready interrupt. But
2123 	 * to make sure we don't hit a controller bug, we _only_
2124 	 * enable Buffer Read Ready interrupt here.
2125 	 */
2126 	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
2127 	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
2128 }
2129 
2130 static void sdhci_end_tuning(struct sdhci_host *host)
2131 {
2132 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2133 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2134 }
2135 
2136 static void sdhci_reset_tuning(struct sdhci_host *host)
2137 {
2138 	u16 ctrl;
2139 
2140 	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2141 	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
2142 	ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
2143 	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2144 }
2145 
2146 static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
2147 {
2148 	sdhci_reset_tuning(host);
2149 
2150 	sdhci_do_reset(host, SDHCI_RESET_CMD);
2151 	sdhci_do_reset(host, SDHCI_RESET_DATA);
2152 
2153 	sdhci_end_tuning(host);
2154 
2155 	mmc_abort_tuning(host->mmc, opcode);
2156 }
2157 
2158 /*
2159  * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
2160  * tuning command does not have a data payload (or rather the hardware does it
2161  * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
2162  * interrupt setup is different to other commands and there is no timeout
2163  * interrupt so special handling is needed.
2164  */
2165 static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
2166 {
2167 	struct mmc_host *mmc = host->mmc;
2168 	struct mmc_command cmd = {};
2169 	struct mmc_request mrq = {};
2170 	unsigned long flags;
2171 	u32 b = host->sdma_boundary;
2172 
2173 	spin_lock_irqsave(&host->lock, flags);
2174 
2175 	cmd.opcode = opcode;
2176 	cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
2177 	cmd.mrq = &mrq;
2178 
2179 	mrq.cmd = &cmd;
2180 	/*
2181 	 * In response to CMD19, the card sends 64 bytes of tuning
2182 	 * block to the Host Controller. So we set the block size
2183 	 * to 64 here.
2184 	 */
2185 	if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
2186 	    mmc->ios.bus_width == MMC_BUS_WIDTH_8)
2187 		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
2188 	else
2189 		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
2190 
2191 	/*
2192 	 * The tuning block is sent by the card to the host controller.
2193 	 * So we set the TRNS_READ bit in the Transfer Mode register.
2194 	 * This also takes care of setting DMA Enable and Multi Block
2195 	 * Select in the same register to 0.
2196 	 */
2197 	sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
2198 
2199 	sdhci_send_command(host, &cmd);
2200 
2201 	host->cmd = NULL;
2202 
2203 	sdhci_del_timer(host, &mrq);
2204 
2205 	host->tuning_done = 0;
2206 
2207 	mmiowb();
2208 	spin_unlock_irqrestore(&host->lock, flags);
2209 
2210 	/* Wait for Buffer Read Ready interrupt */
2211 	wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
2212 			   msecs_to_jiffies(50));
2213 
2214 }
2215 
2216 static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
2217 {
2218 	int i;
2219 
2220 	/*
2221 	 * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
2222 	 * of loops reaches 40 times.
2223 	 */
2224 	for (i = 0; i < MAX_TUNING_LOOP; i++) {
2225 		u16 ctrl;
2226 
2227 		sdhci_send_tuning(host, opcode);
2228 
2229 		if (!host->tuning_done) {
2230 			pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
2231 				mmc_hostname(host->mmc));
2232 			sdhci_abort_tuning(host, opcode);
2233 			return;
2234 		}
2235 
2236 		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2237 		if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
2238 			if (ctrl & SDHCI_CTRL_TUNED_CLK)
2239 				return; /* Success! */
2240 			break;
2241 		}
2242 
2243 		/* Spec does not require a delay between tuning cycles */
2244 		if (host->tuning_delay > 0)
2245 			mdelay(host->tuning_delay);
2246 	}
2247 
2248 	pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
2249 		mmc_hostname(host->mmc));
2250 	sdhci_reset_tuning(host);
2251 }
2252 
2253 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
2254 {
2255 	struct sdhci_host *host = mmc_priv(mmc);
2256 	int err = 0;
2257 	unsigned int tuning_count = 0;
2258 	bool hs400_tuning;
2259 
2260 	hs400_tuning = host->flags & SDHCI_HS400_TUNING;
2261 
2262 	if (host->tuning_mode == SDHCI_TUNING_MODE_1)
2263 		tuning_count = host->tuning_count;
2264 
2265 	/*
2266 	 * The Host Controller needs tuning in case of SDR104 and DDR50
2267 	 * mode, and for SDR50 mode when Use Tuning for SDR50 is set in
2268 	 * the Capabilities register.
2269 	 * If the Host Controller supports the HS200 mode then the
2270 	 * tuning function has to be executed.
2271 	 */
2272 	switch (host->timing) {
2273 	/* HS400 tuning is done in HS200 mode */
2274 	case MMC_TIMING_MMC_HS400:
2275 		err = -EINVAL;
2276 		goto out;
2277 
2278 	case MMC_TIMING_MMC_HS200:
2279 		/*
2280 		 * Periodic re-tuning for HS400 is not expected to be needed, so
2281 		 * disable it here.
2282 		 */
2283 		if (hs400_tuning)
2284 			tuning_count = 0;
2285 		break;
2286 
2287 	case MMC_TIMING_UHS_SDR104:
2288 	case MMC_TIMING_UHS_DDR50:
2289 		break;
2290 
2291 	case MMC_TIMING_UHS_SDR50:
2292 		if (host->flags & SDHCI_SDR50_NEEDS_TUNING)
2293 			break;
2294 		/* FALLTHROUGH */
2295 
2296 	default:
2297 		goto out;
2298 	}
2299 
2300 	if (host->ops->platform_execute_tuning) {
2301 		err = host->ops->platform_execute_tuning(host, opcode);
2302 		goto out;
2303 	}
2304 
2305 	host->mmc->retune_period = tuning_count;
2306 
2307 	if (host->tuning_delay < 0)
2308 		host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK;
2309 
2310 	sdhci_start_tuning(host);
2311 
2312 	__sdhci_execute_tuning(host, opcode);
2313 
2314 	sdhci_end_tuning(host);
2315 out:
2316 	host->flags &= ~SDHCI_HS400_TUNING;
2317 
2318 	return err;
2319 }
2320 EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
2321 
2322 static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable)
2323 {
2324 	/* Host Controller v3.00 defines preset value registers */
2325 	if (host->version < SDHCI_SPEC_300)
2326 		return;
2327 
2328 	/*
2329 	 * We only enable or disable Preset Value if they are not already
2330 	 * enabled or disabled respectively. Otherwise, we bail out.
2331 	 */
2332 	if (host->preset_enabled != enable) {
2333 		u16 ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
2334 
2335 		if (enable)
2336 			ctrl |= SDHCI_CTRL_PRESET_VAL_ENABLE;
2337 		else
2338 			ctrl &= ~SDHCI_CTRL_PRESET_VAL_ENABLE;
2339 
2340 		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
2341 
2342 		if (enable)
2343 			host->flags |= SDHCI_PV_ENABLED;
2344 		else
2345 			host->flags &= ~SDHCI_PV_ENABLED;
2346 
2347 		host->preset_enabled = enable;
2348 	}
2349 }
2350 
2351 static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
2352 				int err)
2353 {
2354 	struct sdhci_host *host = mmc_priv(mmc);
2355 	struct mmc_data *data = mrq->data;
2356 
2357 	if (data->host_cookie != COOKIE_UNMAPPED)
2358 		dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
2359 			     mmc_get_dma_dir(data));
2360 
2361 	data->host_cookie = COOKIE_UNMAPPED;
2362 }
2363 
2364 static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
2365 {
2366 	struct sdhci_host *host = mmc_priv(mmc);
2367 
2368 	mrq->data->host_cookie = COOKIE_UNMAPPED;
2369 
2370 	/*
2371 	 * No pre-mapping in the pre hook if we're using the bounce buffer,
2372 	 * for that we would need two bounce buffers since one buffer is
2373 	 * in flight when this is getting called.
2374 	 */
2375 	if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
2376 		sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
2377 }
2378 
2379 static inline bool sdhci_has_requests(struct sdhci_host *host)
2380 {
2381 	return host->cmd || host->data_cmd;
2382 }
2383 
2384 static void sdhci_error_out_mrqs(struct sdhci_host *host, int err)
2385 {
2386 	if (host->data_cmd) {
2387 		host->data_cmd->error = err;
2388 		sdhci_finish_mrq(host, host->data_cmd->mrq);
2389 	}
2390 
2391 	if (host->cmd) {
2392 		host->cmd->error = err;
2393 		sdhci_finish_mrq(host, host->cmd->mrq);
2394 	}
2395 }
2396 
2397 static void sdhci_card_event(struct mmc_host *mmc)
2398 {
2399 	struct sdhci_host *host = mmc_priv(mmc);
2400 	unsigned long flags;
2401 	int present;
2402 
2403 	/* First check if client has provided their own card event */
2404 	if (host->ops->card_event)
2405 		host->ops->card_event(host);
2406 
2407 	present = mmc->ops->get_cd(mmc);
2408 
2409 	spin_lock_irqsave(&host->lock, flags);
2410 
2411 	/* Check sdhci_has_requests() first in case we are runtime suspended */
2412 	if (sdhci_has_requests(host) && !present) {
2413 		pr_err("%s: Card removed during transfer!\n",
2414 			mmc_hostname(host->mmc));
2415 		pr_err("%s: Resetting controller.\n",
2416 			mmc_hostname(host->mmc));
2417 
2418 		sdhci_do_reset(host, SDHCI_RESET_CMD);
2419 		sdhci_do_reset(host, SDHCI_RESET_DATA);
2420 
2421 		sdhci_error_out_mrqs(host, -ENOMEDIUM);
2422 	}
2423 
2424 	spin_unlock_irqrestore(&host->lock, flags);
2425 }
2426 
2427 static const struct mmc_host_ops sdhci_ops = {
2428 	.request	= sdhci_request,
2429 	.post_req	= sdhci_post_req,
2430 	.pre_req	= sdhci_pre_req,
2431 	.set_ios	= sdhci_set_ios,
2432 	.get_cd		= sdhci_get_cd,
2433 	.get_ro		= sdhci_get_ro,
2434 	.hw_reset	= sdhci_hw_reset,
2435 	.enable_sdio_irq = sdhci_enable_sdio_irq,
2436 	.start_signal_voltage_switch	= sdhci_start_signal_voltage_switch,
2437 	.prepare_hs400_tuning		= sdhci_prepare_hs400_tuning,
2438 	.execute_tuning			= sdhci_execute_tuning,
2439 	.card_event			= sdhci_card_event,
2440 	.card_busy	= sdhci_card_busy,
2441 };
2442 
2443 /*****************************************************************************\
2444  *                                                                           *
2445  * Tasklets                                                                  *
2446  *                                                                           *
2447 \*****************************************************************************/
2448 
2449 static bool sdhci_request_done(struct sdhci_host *host)
2450 {
2451 	unsigned long flags;
2452 	struct mmc_request *mrq;
2453 	int i;
2454 
2455 	spin_lock_irqsave(&host->lock, flags);
2456 
2457 	for (i = 0; i < SDHCI_MAX_MRQS; i++) {
2458 		mrq = host->mrqs_done[i];
2459 		if (mrq)
2460 			break;
2461 	}
2462 
2463 	if (!mrq) {
2464 		spin_unlock_irqrestore(&host->lock, flags);
2465 		return true;
2466 	}
2467 
2468 	sdhci_del_timer(host, mrq);
2469 
2470 	/*
2471 	 * Always unmap the data buffers if they were mapped by
2472 	 * sdhci_prepare_data() whenever we finish with a request.
2473 	 * This avoids leaking DMA mappings on error.
2474 	 */
2475 	if (host->flags & SDHCI_REQ_USE_DMA) {
2476 		struct mmc_data *data = mrq->data;
2477 
2478 		if (data && data->host_cookie == COOKIE_MAPPED) {
2479 			if (host->bounce_buffer) {
2480 				/*
2481 				 * On reads, copy the bounced data into the
2482 				 * sglist
2483 				 */
2484 				if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
2485 					unsigned int length = data->bytes_xfered;
2486 
2487 					if (length > host->bounce_buffer_size) {
2488 						pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
2489 						       mmc_hostname(host->mmc),
2490 						       host->bounce_buffer_size,
2491 						       data->bytes_xfered);
2492 						/* Cap it down and continue */
2493 						length = host->bounce_buffer_size;
2494 					}
2495 					dma_sync_single_for_cpu(
2496 						host->mmc->parent,
2497 						host->bounce_addr,
2498 						host->bounce_buffer_size,
2499 						DMA_FROM_DEVICE);
2500 					sg_copy_from_buffer(data->sg,
2501 						data->sg_len,
2502 						host->bounce_buffer,
2503 						length);
2504 				} else {
2505 					/* No copying, just switch ownership */
2506 					dma_sync_single_for_cpu(
2507 						host->mmc->parent,
2508 						host->bounce_addr,
2509 						host->bounce_buffer_size,
2510 						mmc_get_dma_dir(data));
2511 				}
2512 			} else {
2513 				/* Unmap the raw data */
2514 				dma_unmap_sg(mmc_dev(host->mmc), data->sg,
2515 					     data->sg_len,
2516 					     mmc_get_dma_dir(data));
2517 			}
2518 			data->host_cookie = COOKIE_UNMAPPED;
2519 		}
2520 	}
2521 
2522 	/*
2523 	 * The controller needs a reset of internal state machines
2524 	 * upon error conditions.
2525 	 */
2526 	if (sdhci_needs_reset(host, mrq)) {
2527 		/*
2528 		 * Do not finish until command and data lines are available for
2529 		 * reset. Note there can only be one other mrq, so it cannot
2530 		 * also be in mrqs_done, otherwise host->cmd and host->data_cmd
2531 		 * would both be null.
2532 		 */
2533 		if (host->cmd || host->data_cmd) {
2534 			spin_unlock_irqrestore(&host->lock, flags);
2535 			return true;
2536 		}
2537 
2538 		/* Some controllers need this kick or reset won't work here */
2539 		if (host->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)
2540 			/* This is to force an update */
2541 			host->ops->set_clock(host, host->clock);
2542 
2543 		/* Spec says we should do both at the same time, but Ricoh
2544 		   controllers do not like that. */
2545 		sdhci_do_reset(host, SDHCI_RESET_CMD);
2546 		sdhci_do_reset(host, SDHCI_RESET_DATA);
2547 
2548 		host->pending_reset = false;
2549 	}
2550 
2551 	if (!sdhci_has_requests(host))
2552 		sdhci_led_deactivate(host);
2553 
2554 	host->mrqs_done[i] = NULL;
2555 
2556 	mmiowb();
2557 	spin_unlock_irqrestore(&host->lock, flags);
2558 
2559 	mmc_request_done(host->mmc, mrq);
2560 
2561 	return false;
2562 }
2563 
2564 static void sdhci_tasklet_finish(unsigned long param)
2565 {
2566 	struct sdhci_host *host = (struct sdhci_host *)param;
2567 
2568 	while (!sdhci_request_done(host))
2569 		;
2570 }
2571 
2572 static void sdhci_timeout_timer(struct timer_list *t)
2573 {
2574 	struct sdhci_host *host;
2575 	unsigned long flags;
2576 
2577 	host = from_timer(host, t, timer);
2578 
2579 	spin_lock_irqsave(&host->lock, flags);
2580 
2581 	if (host->cmd && !sdhci_data_line_cmd(host->cmd)) {
2582 		pr_err("%s: Timeout waiting for hardware cmd interrupt.\n",
2583 		       mmc_hostname(host->mmc));
2584 		sdhci_dumpregs(host);
2585 
2586 		host->cmd->error = -ETIMEDOUT;
2587 		sdhci_finish_mrq(host, host->cmd->mrq);
2588 	}
2589 
2590 	mmiowb();
2591 	spin_unlock_irqrestore(&host->lock, flags);
2592 }
2593 
2594 static void sdhci_timeout_data_timer(struct timer_list *t)
2595 {
2596 	struct sdhci_host *host;
2597 	unsigned long flags;
2598 
2599 	host = from_timer(host, t, data_timer);
2600 
2601 	spin_lock_irqsave(&host->lock, flags);
2602 
2603 	if (host->data || host->data_cmd ||
2604 	    (host->cmd && sdhci_data_line_cmd(host->cmd))) {
2605 		pr_err("%s: Timeout waiting for hardware interrupt.\n",
2606 		       mmc_hostname(host->mmc));
2607 		sdhci_dumpregs(host);
2608 
2609 		if (host->data) {
2610 			host->data->error = -ETIMEDOUT;
2611 			sdhci_finish_data(host);
2612 		} else if (host->data_cmd) {
2613 			host->data_cmd->error = -ETIMEDOUT;
2614 			sdhci_finish_mrq(host, host->data_cmd->mrq);
2615 		} else {
2616 			host->cmd->error = -ETIMEDOUT;
2617 			sdhci_finish_mrq(host, host->cmd->mrq);
2618 		}
2619 	}
2620 
2621 	mmiowb();
2622 	spin_unlock_irqrestore(&host->lock, flags);
2623 }
2624 
2625 /*****************************************************************************\
2626  *                                                                           *
2627  * Interrupt handling                                                        *
2628  *                                                                           *
2629 \*****************************************************************************/
2630 
2631 static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask)
2632 {
2633 	if (!host->cmd) {
2634 		/*
2635 		 * SDHCI recovers from errors by resetting the cmd and data
2636 		 * circuits.  Until that is done, there very well might be more
2637 		 * interrupts, so ignore them in that case.
2638 		 */
2639 		if (host->pending_reset)
2640 			return;
2641 		pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n",
2642 		       mmc_hostname(host->mmc), (unsigned)intmask);
2643 		sdhci_dumpregs(host);
2644 		return;
2645 	}
2646 
2647 	if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC |
2648 		       SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) {
2649 		if (intmask & SDHCI_INT_TIMEOUT)
2650 			host->cmd->error = -ETIMEDOUT;
2651 		else
2652 			host->cmd->error = -EILSEQ;
2653 
2654 		/*
2655 		 * If this command initiates a data phase and a response
2656 		 * CRC error is signalled, the card can start transferring
2657 		 * data - the card may have received the command without
2658 		 * error.  We must not terminate the mmc_request early.
2659 		 *
2660 		 * If the card did not receive the command or returned an
2661 		 * error which prevented it sending data, the data phase
2662 		 * will time out.
2663 		 */
2664 		if (host->cmd->data &&
2665 		    (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) ==
2666 		     SDHCI_INT_CRC) {
2667 			host->cmd = NULL;
2668 			return;
2669 		}
2670 
2671 		sdhci_finish_mrq(host, host->cmd->mrq);
2672 		return;
2673 	}
2674 
2675 	if (intmask & SDHCI_INT_RESPONSE)
2676 		sdhci_finish_command(host);
2677 }
2678 
2679 static void sdhci_adma_show_error(struct sdhci_host *host)
2680 {
2681 	void *desc = host->adma_table;
2682 
2683 	sdhci_dumpregs(host);
2684 
2685 	while (true) {
2686 		struct sdhci_adma2_64_desc *dma_desc = desc;
2687 
2688 		if (host->flags & SDHCI_USE_64_BIT_DMA)
2689 			DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n",
2690 			    desc, le32_to_cpu(dma_desc->addr_hi),
2691 			    le32_to_cpu(dma_desc->addr_lo),
2692 			    le16_to_cpu(dma_desc->len),
2693 			    le16_to_cpu(dma_desc->cmd));
2694 		else
2695 			DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n",
2696 			    desc, le32_to_cpu(dma_desc->addr_lo),
2697 			    le16_to_cpu(dma_desc->len),
2698 			    le16_to_cpu(dma_desc->cmd));
2699 
2700 		desc += host->desc_sz;
2701 
2702 		if (dma_desc->cmd & cpu_to_le16(ADMA2_END))
2703 			break;
2704 	}
2705 }
2706 
2707 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
2708 {
2709 	u32 command;
2710 
2711 	/* CMD19 generates _only_ Buffer Read Ready interrupt */
2712 	if (intmask & SDHCI_INT_DATA_AVAIL) {
2713 		command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND));
2714 		if (command == MMC_SEND_TUNING_BLOCK ||
2715 		    command == MMC_SEND_TUNING_BLOCK_HS200) {
2716 			host->tuning_done = 1;
2717 			wake_up(&host->buf_ready_int);
2718 			return;
2719 		}
2720 	}
2721 
2722 	if (!host->data) {
2723 		struct mmc_command *data_cmd = host->data_cmd;
2724 
2725 		/*
2726 		 * The "data complete" interrupt is also used to
2727 		 * indicate that a busy state has ended. See comment
2728 		 * above in sdhci_cmd_irq().
2729 		 */
2730 		if (data_cmd && (data_cmd->flags & MMC_RSP_BUSY)) {
2731 			if (intmask & SDHCI_INT_DATA_TIMEOUT) {
2732 				host->data_cmd = NULL;
2733 				data_cmd->error = -ETIMEDOUT;
2734 				sdhci_finish_mrq(host, data_cmd->mrq);
2735 				return;
2736 			}
2737 			if (intmask & SDHCI_INT_DATA_END) {
2738 				host->data_cmd = NULL;
2739 				/*
2740 				 * Some cards handle busy-end interrupt
2741 				 * before the command completed, so make
2742 				 * sure we do things in the proper order.
2743 				 */
2744 				if (host->cmd == data_cmd)
2745 					return;
2746 
2747 				sdhci_finish_mrq(host, data_cmd->mrq);
2748 				return;
2749 			}
2750 		}
2751 
2752 		/*
2753 		 * SDHCI recovers from errors by resetting the cmd and data
2754 		 * circuits. Until that is done, there very well might be more
2755 		 * interrupts, so ignore them in that case.
2756 		 */
2757 		if (host->pending_reset)
2758 			return;
2759 
2760 		pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n",
2761 		       mmc_hostname(host->mmc), (unsigned)intmask);
2762 		sdhci_dumpregs(host);
2763 
2764 		return;
2765 	}
2766 
2767 	if (intmask & SDHCI_INT_DATA_TIMEOUT)
2768 		host->data->error = -ETIMEDOUT;
2769 	else if (intmask & SDHCI_INT_DATA_END_BIT)
2770 		host->data->error = -EILSEQ;
2771 	else if ((intmask & SDHCI_INT_DATA_CRC) &&
2772 		SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))
2773 			!= MMC_BUS_TEST_R)
2774 		host->data->error = -EILSEQ;
2775 	else if (intmask & SDHCI_INT_ADMA_ERROR) {
2776 		pr_err("%s: ADMA error\n", mmc_hostname(host->mmc));
2777 		sdhci_adma_show_error(host);
2778 		host->data->error = -EIO;
2779 		if (host->ops->adma_workaround)
2780 			host->ops->adma_workaround(host, intmask);
2781 	}
2782 
2783 	if (host->data->error)
2784 		sdhci_finish_data(host);
2785 	else {
2786 		if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
2787 			sdhci_transfer_pio(host);
2788 
2789 		/*
2790 		 * We currently don't do anything fancy with DMA
2791 		 * boundaries, but as we can't disable the feature
2792 		 * we need to at least restart the transfer.
2793 		 *
2794 		 * According to the spec sdhci_readl(host, SDHCI_DMA_ADDRESS)
2795 		 * should return a valid address to continue from, but as
2796 		 * some controllers are faulty, don't trust them.
2797 		 */
2798 		if (intmask & SDHCI_INT_DMA_END) {
2799 			u32 dmastart, dmanow;
2800 
2801 			dmastart = sdhci_sdma_address(host);
2802 			dmanow = dmastart + host->data->bytes_xfered;
2803 			/*
2804 			 * Force update to the next DMA block boundary.
2805 			 */
2806 			dmanow = (dmanow &
2807 				~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) +
2808 				SDHCI_DEFAULT_BOUNDARY_SIZE;
2809 			host->data->bytes_xfered = dmanow - dmastart;
2810 			DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n",
2811 			    dmastart, host->data->bytes_xfered, dmanow);
2812 			sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS);
2813 		}
2814 
2815 		if (intmask & SDHCI_INT_DATA_END) {
2816 			if (host->cmd == host->data_cmd) {
2817 				/*
2818 				 * Data managed to finish before the
2819 				 * command completed. Make sure we do
2820 				 * things in the proper order.
2821 				 */
2822 				host->data_early = 1;
2823 			} else {
2824 				sdhci_finish_data(host);
2825 			}
2826 		}
2827 	}
2828 }
2829 
2830 static irqreturn_t sdhci_irq(int irq, void *dev_id)
2831 {
2832 	irqreturn_t result = IRQ_NONE;
2833 	struct sdhci_host *host = dev_id;
2834 	u32 intmask, mask, unexpected = 0;
2835 	int max_loops = 16;
2836 
2837 	spin_lock(&host->lock);
2838 
2839 	if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) {
2840 		spin_unlock(&host->lock);
2841 		return IRQ_NONE;
2842 	}
2843 
2844 	intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2845 	if (!intmask || intmask == 0xffffffff) {
2846 		result = IRQ_NONE;
2847 		goto out;
2848 	}
2849 
2850 	do {
2851 		DBG("IRQ status 0x%08x\n", intmask);
2852 
2853 		if (host->ops->irq) {
2854 			intmask = host->ops->irq(host, intmask);
2855 			if (!intmask)
2856 				goto cont;
2857 		}
2858 
2859 		/* Clear selected interrupts. */
2860 		mask = intmask & (SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2861 				  SDHCI_INT_BUS_POWER);
2862 		sdhci_writel(host, mask, SDHCI_INT_STATUS);
2863 
2864 		if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2865 			u32 present = sdhci_readl(host, SDHCI_PRESENT_STATE) &
2866 				      SDHCI_CARD_PRESENT;
2867 
2868 			/*
2869 			 * There is a observation on i.mx esdhc.  INSERT
2870 			 * bit will be immediately set again when it gets
2871 			 * cleared, if a card is inserted.  We have to mask
2872 			 * the irq to prevent interrupt storm which will
2873 			 * freeze the system.  And the REMOVE gets the
2874 			 * same situation.
2875 			 *
2876 			 * More testing are needed here to ensure it works
2877 			 * for other platforms though.
2878 			 */
2879 			host->ier &= ~(SDHCI_INT_CARD_INSERT |
2880 				       SDHCI_INT_CARD_REMOVE);
2881 			host->ier |= present ? SDHCI_INT_CARD_REMOVE :
2882 					       SDHCI_INT_CARD_INSERT;
2883 			sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
2884 			sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
2885 
2886 			sdhci_writel(host, intmask & (SDHCI_INT_CARD_INSERT |
2887 				     SDHCI_INT_CARD_REMOVE), SDHCI_INT_STATUS);
2888 
2889 			host->thread_isr |= intmask & (SDHCI_INT_CARD_INSERT |
2890 						       SDHCI_INT_CARD_REMOVE);
2891 			result = IRQ_WAKE_THREAD;
2892 		}
2893 
2894 		if (intmask & SDHCI_INT_CMD_MASK)
2895 			sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK);
2896 
2897 		if (intmask & SDHCI_INT_DATA_MASK)
2898 			sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK);
2899 
2900 		if (intmask & SDHCI_INT_BUS_POWER)
2901 			pr_err("%s: Card is consuming too much power!\n",
2902 				mmc_hostname(host->mmc));
2903 
2904 		if (intmask & SDHCI_INT_RETUNE)
2905 			mmc_retune_needed(host->mmc);
2906 
2907 		if ((intmask & SDHCI_INT_CARD_INT) &&
2908 		    (host->ier & SDHCI_INT_CARD_INT)) {
2909 			sdhci_enable_sdio_irq_nolock(host, false);
2910 			host->thread_isr |= SDHCI_INT_CARD_INT;
2911 			result = IRQ_WAKE_THREAD;
2912 		}
2913 
2914 		intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
2915 			     SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK |
2916 			     SDHCI_INT_ERROR | SDHCI_INT_BUS_POWER |
2917 			     SDHCI_INT_RETUNE | SDHCI_INT_CARD_INT);
2918 
2919 		if (intmask) {
2920 			unexpected |= intmask;
2921 			sdhci_writel(host, intmask, SDHCI_INT_STATUS);
2922 		}
2923 cont:
2924 		if (result == IRQ_NONE)
2925 			result = IRQ_HANDLED;
2926 
2927 		intmask = sdhci_readl(host, SDHCI_INT_STATUS);
2928 	} while (intmask && --max_loops);
2929 out:
2930 	spin_unlock(&host->lock);
2931 
2932 	if (unexpected) {
2933 		pr_err("%s: Unexpected interrupt 0x%08x.\n",
2934 			   mmc_hostname(host->mmc), unexpected);
2935 		sdhci_dumpregs(host);
2936 	}
2937 
2938 	return result;
2939 }
2940 
2941 static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
2942 {
2943 	struct sdhci_host *host = dev_id;
2944 	unsigned long flags;
2945 	u32 isr;
2946 
2947 	spin_lock_irqsave(&host->lock, flags);
2948 	isr = host->thread_isr;
2949 	host->thread_isr = 0;
2950 	spin_unlock_irqrestore(&host->lock, flags);
2951 
2952 	if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2953 		struct mmc_host *mmc = host->mmc;
2954 
2955 		mmc->ops->card_event(mmc);
2956 		mmc_detect_change(mmc, msecs_to_jiffies(200));
2957 	}
2958 
2959 	if (isr & SDHCI_INT_CARD_INT) {
2960 		sdio_run_irqs(host->mmc);
2961 
2962 		spin_lock_irqsave(&host->lock, flags);
2963 		if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
2964 			sdhci_enable_sdio_irq_nolock(host, true);
2965 		spin_unlock_irqrestore(&host->lock, flags);
2966 	}
2967 
2968 	return isr ? IRQ_HANDLED : IRQ_NONE;
2969 }
2970 
2971 /*****************************************************************************\
2972  *                                                                           *
2973  * Suspend/resume                                                            *
2974  *                                                                           *
2975 \*****************************************************************************/
2976 
2977 #ifdef CONFIG_PM
2978 
2979 static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host)
2980 {
2981 	return mmc_card_is_removable(host->mmc) &&
2982 	       !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
2983 	       !mmc_can_gpio_cd(host->mmc);
2984 }
2985 
2986 /*
2987  * To enable wakeup events, the corresponding events have to be enabled in
2988  * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal
2989  * Table' in the SD Host Controller Standard Specification.
2990  * It is useless to restore SDHCI_INT_ENABLE state in
2991  * sdhci_disable_irq_wakeups() since it will be set by
2992  * sdhci_enable_card_detection() or sdhci_init().
2993  */
2994 static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
2995 {
2996 	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
2997 		  SDHCI_WAKE_ON_INT;
2998 	u32 irq_val = 0;
2999 	u8 wake_val = 0;
3000 	u8 val;
3001 
3002 	if (sdhci_cd_irq_can_wakeup(host)) {
3003 		wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
3004 		irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
3005 	}
3006 
3007 	if (mmc_card_wake_sdio_irq(host->mmc)) {
3008 		wake_val |= SDHCI_WAKE_ON_INT;
3009 		irq_val |= SDHCI_INT_CARD_INT;
3010 	}
3011 
3012 	if (!irq_val)
3013 		return false;
3014 
3015 	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3016 	val &= ~mask;
3017 	val |= wake_val;
3018 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3019 
3020 	sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
3021 
3022 	host->irq_wake_enabled = !enable_irq_wake(host->irq);
3023 
3024 	return host->irq_wake_enabled;
3025 }
3026 
3027 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
3028 {
3029 	u8 val;
3030 	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
3031 			| SDHCI_WAKE_ON_INT;
3032 
3033 	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
3034 	val &= ~mask;
3035 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
3036 
3037 	disable_irq_wake(host->irq);
3038 
3039 	host->irq_wake_enabled = false;
3040 }
3041 
3042 int sdhci_suspend_host(struct sdhci_host *host)
3043 {
3044 	sdhci_disable_card_detection(host);
3045 
3046 	mmc_retune_timer_stop(host->mmc);
3047 
3048 	if (!device_may_wakeup(mmc_dev(host->mmc)) ||
3049 	    !sdhci_enable_irq_wakeups(host)) {
3050 		host->ier = 0;
3051 		sdhci_writel(host, 0, SDHCI_INT_ENABLE);
3052 		sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
3053 		free_irq(host->irq, host);
3054 	}
3055 
3056 	return 0;
3057 }
3058 
3059 EXPORT_SYMBOL_GPL(sdhci_suspend_host);
3060 
3061 int sdhci_resume_host(struct sdhci_host *host)
3062 {
3063 	struct mmc_host *mmc = host->mmc;
3064 	int ret = 0;
3065 
3066 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3067 		if (host->ops->enable_dma)
3068 			host->ops->enable_dma(host);
3069 	}
3070 
3071 	if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) &&
3072 	    (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) {
3073 		/* Card keeps power but host controller does not */
3074 		sdhci_init(host, 0);
3075 		host->pwr = 0;
3076 		host->clock = 0;
3077 		mmc->ops->set_ios(mmc, &mmc->ios);
3078 	} else {
3079 		sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER));
3080 		mmiowb();
3081 	}
3082 
3083 	if (host->irq_wake_enabled) {
3084 		sdhci_disable_irq_wakeups(host);
3085 	} else {
3086 		ret = request_threaded_irq(host->irq, sdhci_irq,
3087 					   sdhci_thread_irq, IRQF_SHARED,
3088 					   mmc_hostname(host->mmc), host);
3089 		if (ret)
3090 			return ret;
3091 	}
3092 
3093 	sdhci_enable_card_detection(host);
3094 
3095 	return ret;
3096 }
3097 
3098 EXPORT_SYMBOL_GPL(sdhci_resume_host);
3099 
3100 int sdhci_runtime_suspend_host(struct sdhci_host *host)
3101 {
3102 	unsigned long flags;
3103 
3104 	mmc_retune_timer_stop(host->mmc);
3105 
3106 	spin_lock_irqsave(&host->lock, flags);
3107 	host->ier &= SDHCI_INT_CARD_INT;
3108 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3109 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3110 	spin_unlock_irqrestore(&host->lock, flags);
3111 
3112 	synchronize_hardirq(host->irq);
3113 
3114 	spin_lock_irqsave(&host->lock, flags);
3115 	host->runtime_suspended = true;
3116 	spin_unlock_irqrestore(&host->lock, flags);
3117 
3118 	return 0;
3119 }
3120 EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host);
3121 
3122 int sdhci_runtime_resume_host(struct sdhci_host *host)
3123 {
3124 	struct mmc_host *mmc = host->mmc;
3125 	unsigned long flags;
3126 	int host_flags = host->flags;
3127 
3128 	if (host_flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3129 		if (host->ops->enable_dma)
3130 			host->ops->enable_dma(host);
3131 	}
3132 
3133 	sdhci_init(host, 0);
3134 
3135 	if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
3136 	    mmc->ios.power_mode != MMC_POWER_OFF) {
3137 		/* Force clock and power re-program */
3138 		host->pwr = 0;
3139 		host->clock = 0;
3140 		mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios);
3141 		mmc->ops->set_ios(mmc, &mmc->ios);
3142 
3143 		if ((host_flags & SDHCI_PV_ENABLED) &&
3144 		    !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) {
3145 			spin_lock_irqsave(&host->lock, flags);
3146 			sdhci_enable_preset_value(host, true);
3147 			spin_unlock_irqrestore(&host->lock, flags);
3148 		}
3149 
3150 		if ((mmc->caps2 & MMC_CAP2_HS400_ES) &&
3151 		    mmc->ops->hs400_enhanced_strobe)
3152 			mmc->ops->hs400_enhanced_strobe(mmc, &mmc->ios);
3153 	}
3154 
3155 	spin_lock_irqsave(&host->lock, flags);
3156 
3157 	host->runtime_suspended = false;
3158 
3159 	/* Enable SDIO IRQ */
3160 	if (host->flags & SDHCI_SDIO_IRQ_ENABLED)
3161 		sdhci_enable_sdio_irq_nolock(host, true);
3162 
3163 	/* Enable Card Detection */
3164 	sdhci_enable_card_detection(host);
3165 
3166 	spin_unlock_irqrestore(&host->lock, flags);
3167 
3168 	return 0;
3169 }
3170 EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host);
3171 
3172 #endif /* CONFIG_PM */
3173 
3174 /*****************************************************************************\
3175  *                                                                           *
3176  * Command Queue Engine (CQE) helpers                                        *
3177  *                                                                           *
3178 \*****************************************************************************/
3179 
3180 void sdhci_cqe_enable(struct mmc_host *mmc)
3181 {
3182 	struct sdhci_host *host = mmc_priv(mmc);
3183 	unsigned long flags;
3184 	u8 ctrl;
3185 
3186 	spin_lock_irqsave(&host->lock, flags);
3187 
3188 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
3189 	ctrl &= ~SDHCI_CTRL_DMA_MASK;
3190 	if (host->flags & SDHCI_USE_64_BIT_DMA)
3191 		ctrl |= SDHCI_CTRL_ADMA64;
3192 	else
3193 		ctrl |= SDHCI_CTRL_ADMA32;
3194 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
3195 
3196 	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
3197 		     SDHCI_BLOCK_SIZE);
3198 
3199 	/* Set maximum timeout */
3200 	sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL);
3201 
3202 	host->ier = host->cqe_ier;
3203 
3204 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
3205 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
3206 
3207 	host->cqe_on = true;
3208 
3209 	pr_debug("%s: sdhci: CQE on, IRQ mask %#x, IRQ status %#x\n",
3210 		 mmc_hostname(mmc), host->ier,
3211 		 sdhci_readl(host, SDHCI_INT_STATUS));
3212 
3213 	mmiowb();
3214 	spin_unlock_irqrestore(&host->lock, flags);
3215 }
3216 EXPORT_SYMBOL_GPL(sdhci_cqe_enable);
3217 
3218 void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
3219 {
3220 	struct sdhci_host *host = mmc_priv(mmc);
3221 	unsigned long flags;
3222 
3223 	spin_lock_irqsave(&host->lock, flags);
3224 
3225 	sdhci_set_default_irqs(host);
3226 
3227 	host->cqe_on = false;
3228 
3229 	if (recovery) {
3230 		sdhci_do_reset(host, SDHCI_RESET_CMD);
3231 		sdhci_do_reset(host, SDHCI_RESET_DATA);
3232 	}
3233 
3234 	pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
3235 		 mmc_hostname(mmc), host->ier,
3236 		 sdhci_readl(host, SDHCI_INT_STATUS));
3237 
3238 	mmiowb();
3239 	spin_unlock_irqrestore(&host->lock, flags);
3240 }
3241 EXPORT_SYMBOL_GPL(sdhci_cqe_disable);
3242 
3243 bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
3244 		   int *data_error)
3245 {
3246 	u32 mask;
3247 
3248 	if (!host->cqe_on)
3249 		return false;
3250 
3251 	if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC))
3252 		*cmd_error = -EILSEQ;
3253 	else if (intmask & SDHCI_INT_TIMEOUT)
3254 		*cmd_error = -ETIMEDOUT;
3255 	else
3256 		*cmd_error = 0;
3257 
3258 	if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC))
3259 		*data_error = -EILSEQ;
3260 	else if (intmask & SDHCI_INT_DATA_TIMEOUT)
3261 		*data_error = -ETIMEDOUT;
3262 	else if (intmask & SDHCI_INT_ADMA_ERROR)
3263 		*data_error = -EIO;
3264 	else
3265 		*data_error = 0;
3266 
3267 	/* Clear selected interrupts. */
3268 	mask = intmask & host->cqe_ier;
3269 	sdhci_writel(host, mask, SDHCI_INT_STATUS);
3270 
3271 	if (intmask & SDHCI_INT_BUS_POWER)
3272 		pr_err("%s: Card is consuming too much power!\n",
3273 		       mmc_hostname(host->mmc));
3274 
3275 	intmask &= ~(host->cqe_ier | SDHCI_INT_ERROR);
3276 	if (intmask) {
3277 		sdhci_writel(host, intmask, SDHCI_INT_STATUS);
3278 		pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n",
3279 		       mmc_hostname(host->mmc), intmask);
3280 		sdhci_dumpregs(host);
3281 	}
3282 
3283 	return true;
3284 }
3285 EXPORT_SYMBOL_GPL(sdhci_cqe_irq);
3286 
3287 /*****************************************************************************\
3288  *                                                                           *
3289  * Device allocation/registration                                            *
3290  *                                                                           *
3291 \*****************************************************************************/
3292 
3293 struct sdhci_host *sdhci_alloc_host(struct device *dev,
3294 	size_t priv_size)
3295 {
3296 	struct mmc_host *mmc;
3297 	struct sdhci_host *host;
3298 
3299 	WARN_ON(dev == NULL);
3300 
3301 	mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev);
3302 	if (!mmc)
3303 		return ERR_PTR(-ENOMEM);
3304 
3305 	host = mmc_priv(mmc);
3306 	host->mmc = mmc;
3307 	host->mmc_host_ops = sdhci_ops;
3308 	mmc->ops = &host->mmc_host_ops;
3309 
3310 	host->flags = SDHCI_SIGNALING_330;
3311 
3312 	host->cqe_ier     = SDHCI_CQE_INT_MASK;
3313 	host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK;
3314 
3315 	host->tuning_delay = -1;
3316 
3317 	host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
3318 
3319 	return host;
3320 }
3321 
3322 EXPORT_SYMBOL_GPL(sdhci_alloc_host);
3323 
3324 static int sdhci_set_dma_mask(struct sdhci_host *host)
3325 {
3326 	struct mmc_host *mmc = host->mmc;
3327 	struct device *dev = mmc_dev(mmc);
3328 	int ret = -EINVAL;
3329 
3330 	if (host->quirks2 & SDHCI_QUIRK2_BROKEN_64_BIT_DMA)
3331 		host->flags &= ~SDHCI_USE_64_BIT_DMA;
3332 
3333 	/* Try 64-bit mask if hardware is capable  of it */
3334 	if (host->flags & SDHCI_USE_64_BIT_DMA) {
3335 		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
3336 		if (ret) {
3337 			pr_warn("%s: Failed to set 64-bit DMA mask.\n",
3338 				mmc_hostname(mmc));
3339 			host->flags &= ~SDHCI_USE_64_BIT_DMA;
3340 		}
3341 	}
3342 
3343 	/* 32-bit mask as default & fallback */
3344 	if (ret) {
3345 		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
3346 		if (ret)
3347 			pr_warn("%s: Failed to set 32-bit DMA mask.\n",
3348 				mmc_hostname(mmc));
3349 	}
3350 
3351 	return ret;
3352 }
3353 
3354 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
3355 {
3356 	u16 v;
3357 	u64 dt_caps_mask = 0;
3358 	u64 dt_caps = 0;
3359 
3360 	if (host->read_caps)
3361 		return;
3362 
3363 	host->read_caps = true;
3364 
3365 	if (debug_quirks)
3366 		host->quirks = debug_quirks;
3367 
3368 	if (debug_quirks2)
3369 		host->quirks2 = debug_quirks2;
3370 
3371 	sdhci_do_reset(host, SDHCI_RESET_ALL);
3372 
3373 	of_property_read_u64(mmc_dev(host->mmc)->of_node,
3374 			     "sdhci-caps-mask", &dt_caps_mask);
3375 	of_property_read_u64(mmc_dev(host->mmc)->of_node,
3376 			     "sdhci-caps", &dt_caps);
3377 
3378 	v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
3379 	host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
3380 
3381 	if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
3382 		return;
3383 
3384 	if (caps) {
3385 		host->caps = *caps;
3386 	} else {
3387 		host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
3388 		host->caps &= ~lower_32_bits(dt_caps_mask);
3389 		host->caps |= lower_32_bits(dt_caps);
3390 	}
3391 
3392 	if (host->version < SDHCI_SPEC_300)
3393 		return;
3394 
3395 	if (caps1) {
3396 		host->caps1 = *caps1;
3397 	} else {
3398 		host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
3399 		host->caps1 &= ~upper_32_bits(dt_caps_mask);
3400 		host->caps1 |= upper_32_bits(dt_caps);
3401 	}
3402 }
3403 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
3404 
3405 static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
3406 {
3407 	struct mmc_host *mmc = host->mmc;
3408 	unsigned int max_blocks;
3409 	unsigned int bounce_size;
3410 	int ret;
3411 
3412 	/*
3413 	 * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
3414 	 * has diminishing returns, this is probably because SD/MMC
3415 	 * cards are usually optimized to handle this size of requests.
3416 	 */
3417 	bounce_size = SZ_64K;
3418 	/*
3419 	 * Adjust downwards to maximum request size if this is less
3420 	 * than our segment size, else hammer down the maximum
3421 	 * request size to the maximum buffer size.
3422 	 */
3423 	if (mmc->max_req_size < bounce_size)
3424 		bounce_size = mmc->max_req_size;
3425 	max_blocks = bounce_size / 512;
3426 
3427 	/*
3428 	 * When we just support one segment, we can get significant
3429 	 * speedups by the help of a bounce buffer to group scattered
3430 	 * reads/writes together.
3431 	 */
3432 	host->bounce_buffer = devm_kmalloc(mmc->parent,
3433 					   bounce_size,
3434 					   GFP_KERNEL);
3435 	if (!host->bounce_buffer) {
3436 		pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
3437 		       mmc_hostname(mmc),
3438 		       bounce_size);
3439 		/*
3440 		 * Exiting with zero here makes sure we proceed with
3441 		 * mmc->max_segs == 1.
3442 		 */
3443 		return 0;
3444 	}
3445 
3446 	host->bounce_addr = dma_map_single(mmc->parent,
3447 					   host->bounce_buffer,
3448 					   bounce_size,
3449 					   DMA_BIDIRECTIONAL);
3450 	ret = dma_mapping_error(mmc->parent, host->bounce_addr);
3451 	if (ret)
3452 		/* Again fall back to max_segs == 1 */
3453 		return 0;
3454 	host->bounce_buffer_size = bounce_size;
3455 
3456 	/* Lie about this since we're bouncing */
3457 	mmc->max_segs = max_blocks;
3458 	mmc->max_seg_size = bounce_size;
3459 	mmc->max_req_size = bounce_size;
3460 
3461 	pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
3462 		mmc_hostname(mmc), max_blocks, bounce_size);
3463 
3464 	return 0;
3465 }
3466 
3467 int sdhci_setup_host(struct sdhci_host *host)
3468 {
3469 	struct mmc_host *mmc;
3470 	u32 max_current_caps;
3471 	unsigned int ocr_avail;
3472 	unsigned int override_timeout_clk;
3473 	u32 max_clk;
3474 	int ret;
3475 
3476 	WARN_ON(host == NULL);
3477 	if (host == NULL)
3478 		return -EINVAL;
3479 
3480 	mmc = host->mmc;
3481 
3482 	/*
3483 	 * If there are external regulators, get them. Note this must be done
3484 	 * early before resetting the host and reading the capabilities so that
3485 	 * the host can take the appropriate action if regulators are not
3486 	 * available.
3487 	 */
3488 	ret = mmc_regulator_get_supply(mmc);
3489 	if (ret)
3490 		return ret;
3491 
3492 	DBG("Version:   0x%08x | Present:  0x%08x\n",
3493 	    sdhci_readw(host, SDHCI_HOST_VERSION),
3494 	    sdhci_readl(host, SDHCI_PRESENT_STATE));
3495 	DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
3496 	    sdhci_readl(host, SDHCI_CAPABILITIES),
3497 	    sdhci_readl(host, SDHCI_CAPABILITIES_1));
3498 
3499 	sdhci_read_caps(host);
3500 
3501 	override_timeout_clk = host->timeout_clk;
3502 
3503 	if (host->version > SDHCI_SPEC_300) {
3504 		pr_err("%s: Unknown controller version (%d). You may experience problems.\n",
3505 		       mmc_hostname(mmc), host->version);
3506 	}
3507 
3508 	if (host->quirks & SDHCI_QUIRK_FORCE_DMA)
3509 		host->flags |= SDHCI_USE_SDMA;
3510 	else if (!(host->caps & SDHCI_CAN_DO_SDMA))
3511 		DBG("Controller doesn't have SDMA capability\n");
3512 	else
3513 		host->flags |= SDHCI_USE_SDMA;
3514 
3515 	if ((host->quirks & SDHCI_QUIRK_BROKEN_DMA) &&
3516 		(host->flags & SDHCI_USE_SDMA)) {
3517 		DBG("Disabling DMA as it is marked broken\n");
3518 		host->flags &= ~SDHCI_USE_SDMA;
3519 	}
3520 
3521 	if ((host->version >= SDHCI_SPEC_200) &&
3522 		(host->caps & SDHCI_CAN_DO_ADMA2))
3523 		host->flags |= SDHCI_USE_ADMA;
3524 
3525 	if ((host->quirks & SDHCI_QUIRK_BROKEN_ADMA) &&
3526 		(host->flags & SDHCI_USE_ADMA)) {
3527 		DBG("Disabling ADMA as it is marked broken\n");
3528 		host->flags &= ~SDHCI_USE_ADMA;
3529 	}
3530 
3531 	/*
3532 	 * It is assumed that a 64-bit capable device has set a 64-bit DMA mask
3533 	 * and *must* do 64-bit DMA.  A driver has the opportunity to change
3534 	 * that during the first call to ->enable_dma().  Similarly
3535 	 * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to
3536 	 * implement.
3537 	 */
3538 	if (host->caps & SDHCI_CAN_64BIT)
3539 		host->flags |= SDHCI_USE_64_BIT_DMA;
3540 
3541 	if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
3542 		ret = sdhci_set_dma_mask(host);
3543 
3544 		if (!ret && host->ops->enable_dma)
3545 			ret = host->ops->enable_dma(host);
3546 
3547 		if (ret) {
3548 			pr_warn("%s: No suitable DMA available - falling back to PIO\n",
3549 				mmc_hostname(mmc));
3550 			host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA);
3551 
3552 			ret = 0;
3553 		}
3554 	}
3555 
3556 	/* SDMA does not support 64-bit DMA */
3557 	if (host->flags & SDHCI_USE_64_BIT_DMA)
3558 		host->flags &= ~SDHCI_USE_SDMA;
3559 
3560 	if (host->flags & SDHCI_USE_ADMA) {
3561 		dma_addr_t dma;
3562 		void *buf;
3563 
3564 		/*
3565 		 * The DMA descriptor table size is calculated as the maximum
3566 		 * number of segments times 2, to allow for an alignment
3567 		 * descriptor for each segment, plus 1 for a nop end descriptor,
3568 		 * all multipled by the descriptor size.
3569 		 */
3570 		if (host->flags & SDHCI_USE_64_BIT_DMA) {
3571 			host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3572 					      SDHCI_ADMA2_64_DESC_SZ;
3573 			host->desc_sz = SDHCI_ADMA2_64_DESC_SZ;
3574 		} else {
3575 			host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) *
3576 					      SDHCI_ADMA2_32_DESC_SZ;
3577 			host->desc_sz = SDHCI_ADMA2_32_DESC_SZ;
3578 		}
3579 
3580 		host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN;
3581 		buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz +
3582 					 host->adma_table_sz, &dma, GFP_KERNEL);
3583 		if (!buf) {
3584 			pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n",
3585 				mmc_hostname(mmc));
3586 			host->flags &= ~SDHCI_USE_ADMA;
3587 		} else if ((dma + host->align_buffer_sz) &
3588 			   (SDHCI_ADMA2_DESC_ALIGN - 1)) {
3589 			pr_warn("%s: unable to allocate aligned ADMA descriptor\n",
3590 				mmc_hostname(mmc));
3591 			host->flags &= ~SDHCI_USE_ADMA;
3592 			dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3593 					  host->adma_table_sz, buf, dma);
3594 		} else {
3595 			host->align_buffer = buf;
3596 			host->align_addr = dma;
3597 
3598 			host->adma_table = buf + host->align_buffer_sz;
3599 			host->adma_addr = dma + host->align_buffer_sz;
3600 		}
3601 	}
3602 
3603 	/*
3604 	 * If we use DMA, then it's up to the caller to set the DMA
3605 	 * mask, but PIO does not need the hw shim so we set a new
3606 	 * mask here in that case.
3607 	 */
3608 	if (!(host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA))) {
3609 		host->dma_mask = DMA_BIT_MASK(64);
3610 		mmc_dev(mmc)->dma_mask = &host->dma_mask;
3611 	}
3612 
3613 	if (host->version >= SDHCI_SPEC_300)
3614 		host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK)
3615 			>> SDHCI_CLOCK_BASE_SHIFT;
3616 	else
3617 		host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK)
3618 			>> SDHCI_CLOCK_BASE_SHIFT;
3619 
3620 	host->max_clk *= 1000000;
3621 	if (host->max_clk == 0 || host->quirks &
3622 			SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN) {
3623 		if (!host->ops->get_max_clock) {
3624 			pr_err("%s: Hardware doesn't specify base clock frequency.\n",
3625 			       mmc_hostname(mmc));
3626 			ret = -ENODEV;
3627 			goto undma;
3628 		}
3629 		host->max_clk = host->ops->get_max_clock(host);
3630 	}
3631 
3632 	/*
3633 	 * In case of Host Controller v3.00, find out whether clock
3634 	 * multiplier is supported.
3635 	 */
3636 	host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >>
3637 			SDHCI_CLOCK_MUL_SHIFT;
3638 
3639 	/*
3640 	 * In case the value in Clock Multiplier is 0, then programmable
3641 	 * clock mode is not supported, otherwise the actual clock
3642 	 * multiplier is one more than the value of Clock Multiplier
3643 	 * in the Capabilities Register.
3644 	 */
3645 	if (host->clk_mul)
3646 		host->clk_mul += 1;
3647 
3648 	/*
3649 	 * Set host parameters.
3650 	 */
3651 	max_clk = host->max_clk;
3652 
3653 	if (host->ops->get_min_clock)
3654 		mmc->f_min = host->ops->get_min_clock(host);
3655 	else if (host->version >= SDHCI_SPEC_300) {
3656 		if (host->clk_mul) {
3657 			mmc->f_min = (host->max_clk * host->clk_mul) / 1024;
3658 			max_clk = host->max_clk * host->clk_mul;
3659 		} else
3660 			mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300;
3661 	} else
3662 		mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200;
3663 
3664 	if (!mmc->f_max || mmc->f_max > max_clk)
3665 		mmc->f_max = max_clk;
3666 
3667 	if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) {
3668 		host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >>
3669 					SDHCI_TIMEOUT_CLK_SHIFT;
3670 
3671 		if (host->caps & SDHCI_TIMEOUT_CLK_UNIT)
3672 			host->timeout_clk *= 1000;
3673 
3674 		if (host->timeout_clk == 0) {
3675 			if (!host->ops->get_timeout_clock) {
3676 				pr_err("%s: Hardware doesn't specify timeout clock frequency.\n",
3677 					mmc_hostname(mmc));
3678 				ret = -ENODEV;
3679 				goto undma;
3680 			}
3681 
3682 			host->timeout_clk =
3683 				DIV_ROUND_UP(host->ops->get_timeout_clock(host),
3684 					     1000);
3685 		}
3686 
3687 		if (override_timeout_clk)
3688 			host->timeout_clk = override_timeout_clk;
3689 
3690 		mmc->max_busy_timeout = host->ops->get_max_timeout_count ?
3691 			host->ops->get_max_timeout_count(host) : 1 << 27;
3692 		mmc->max_busy_timeout /= host->timeout_clk;
3693 	}
3694 
3695 	if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT &&
3696 	    !host->ops->get_max_timeout_count)
3697 		mmc->max_busy_timeout = 0;
3698 
3699 	mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23;
3700 	mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
3701 
3702 	if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12)
3703 		host->flags |= SDHCI_AUTO_CMD12;
3704 
3705 	/* Auto-CMD23 stuff only works in ADMA or PIO. */
3706 	if ((host->version >= SDHCI_SPEC_300) &&
3707 	    ((host->flags & SDHCI_USE_ADMA) ||
3708 	     !(host->flags & SDHCI_USE_SDMA)) &&
3709 	     !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) {
3710 		host->flags |= SDHCI_AUTO_CMD23;
3711 		DBG("Auto-CMD23 available\n");
3712 	} else {
3713 		DBG("Auto-CMD23 unavailable\n");
3714 	}
3715 
3716 	/*
3717 	 * A controller may support 8-bit width, but the board itself
3718 	 * might not have the pins brought out.  Boards that support
3719 	 * 8-bit width must set "mmc->caps |= MMC_CAP_8_BIT_DATA;" in
3720 	 * their platform code before calling sdhci_add_host(), and we
3721 	 * won't assume 8-bit width for hosts without that CAP.
3722 	 */
3723 	if (!(host->quirks & SDHCI_QUIRK_FORCE_1_BIT_DATA))
3724 		mmc->caps |= MMC_CAP_4_BIT_DATA;
3725 
3726 	if (host->quirks2 & SDHCI_QUIRK2_HOST_NO_CMD23)
3727 		mmc->caps &= ~MMC_CAP_CMD23;
3728 
3729 	if (host->caps & SDHCI_CAN_DO_HISPD)
3730 		mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED;
3731 
3732 	if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) &&
3733 	    mmc_card_is_removable(mmc) &&
3734 	    mmc_gpio_get_cd(host->mmc) < 0)
3735 		mmc->caps |= MMC_CAP_NEEDS_POLL;
3736 
3737 	/* If vqmmc regulator and no 1.8V signalling, then there's no UHS */
3738 	if (!IS_ERR(mmc->supply.vqmmc)) {
3739 		ret = regulator_enable(mmc->supply.vqmmc);
3740 		if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000,
3741 						    1950000))
3742 			host->caps1 &= ~(SDHCI_SUPPORT_SDR104 |
3743 					 SDHCI_SUPPORT_SDR50 |
3744 					 SDHCI_SUPPORT_DDR50);
3745 		if (ret) {
3746 			pr_warn("%s: Failed to enable vqmmc regulator: %d\n",
3747 				mmc_hostname(mmc), ret);
3748 			mmc->supply.vqmmc = ERR_PTR(-EINVAL);
3749 		}
3750 	}
3751 
3752 	if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) {
3753 		host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3754 				 SDHCI_SUPPORT_DDR50);
3755 		/*
3756 		 * The SDHCI controller in a SoC might support HS200/HS400
3757 		 * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property),
3758 		 * but if the board is modeled such that the IO lines are not
3759 		 * connected to 1.8v then HS200/HS400 cannot be supported.
3760 		 * Disable HS200/HS400 if the board does not have 1.8v connected
3761 		 * to the IO lines. (Applicable for other modes in 1.8v)
3762 		 */
3763 		mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES);
3764 		mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS);
3765 	}
3766 
3767 	/* Any UHS-I mode in caps implies SDR12 and SDR25 support. */
3768 	if (host->caps1 & (SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 |
3769 			   SDHCI_SUPPORT_DDR50))
3770 		mmc->caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
3771 
3772 	/* SDR104 supports also implies SDR50 support */
3773 	if (host->caps1 & SDHCI_SUPPORT_SDR104) {
3774 		mmc->caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
3775 		/* SD3.0: SDR104 is supported so (for eMMC) the caps2
3776 		 * field can be promoted to support HS200.
3777 		 */
3778 		if (!(host->quirks2 & SDHCI_QUIRK2_BROKEN_HS200))
3779 			mmc->caps2 |= MMC_CAP2_HS200;
3780 	} else if (host->caps1 & SDHCI_SUPPORT_SDR50) {
3781 		mmc->caps |= MMC_CAP_UHS_SDR50;
3782 	}
3783 
3784 	if (host->quirks2 & SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 &&
3785 	    (host->caps1 & SDHCI_SUPPORT_HS400))
3786 		mmc->caps2 |= MMC_CAP2_HS400;
3787 
3788 	if ((mmc->caps2 & MMC_CAP2_HSX00_1_2V) &&
3789 	    (IS_ERR(mmc->supply.vqmmc) ||
3790 	     !regulator_is_supported_voltage(mmc->supply.vqmmc, 1100000,
3791 					     1300000)))
3792 		mmc->caps2 &= ~MMC_CAP2_HSX00_1_2V;
3793 
3794 	if ((host->caps1 & SDHCI_SUPPORT_DDR50) &&
3795 	    !(host->quirks2 & SDHCI_QUIRK2_BROKEN_DDR50))
3796 		mmc->caps |= MMC_CAP_UHS_DDR50;
3797 
3798 	/* Does the host need tuning for SDR50? */
3799 	if (host->caps1 & SDHCI_USE_SDR50_TUNING)
3800 		host->flags |= SDHCI_SDR50_NEEDS_TUNING;
3801 
3802 	/* Driver Type(s) (A, C, D) supported by the host */
3803 	if (host->caps1 & SDHCI_DRIVER_TYPE_A)
3804 		mmc->caps |= MMC_CAP_DRIVER_TYPE_A;
3805 	if (host->caps1 & SDHCI_DRIVER_TYPE_C)
3806 		mmc->caps |= MMC_CAP_DRIVER_TYPE_C;
3807 	if (host->caps1 & SDHCI_DRIVER_TYPE_D)
3808 		mmc->caps |= MMC_CAP_DRIVER_TYPE_D;
3809 
3810 	/* Initial value for re-tuning timer count */
3811 	host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >>
3812 			     SDHCI_RETUNING_TIMER_COUNT_SHIFT;
3813 
3814 	/*
3815 	 * In case Re-tuning Timer is not disabled, the actual value of
3816 	 * re-tuning timer will be 2 ^ (n - 1).
3817 	 */
3818 	if (host->tuning_count)
3819 		host->tuning_count = 1 << (host->tuning_count - 1);
3820 
3821 	/* Re-tuning mode supported by the Host Controller */
3822 	host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >>
3823 			     SDHCI_RETUNING_MODE_SHIFT;
3824 
3825 	ocr_avail = 0;
3826 
3827 	/*
3828 	 * According to SD Host Controller spec v3.00, if the Host System
3829 	 * can afford more than 150mA, Host Driver should set XPC to 1. Also
3830 	 * the value is meaningful only if Voltage Support in the Capabilities
3831 	 * register is set. The actual current value is 4 times the register
3832 	 * value.
3833 	 */
3834 	max_current_caps = sdhci_readl(host, SDHCI_MAX_CURRENT);
3835 	if (!max_current_caps && !IS_ERR(mmc->supply.vmmc)) {
3836 		int curr = regulator_get_current_limit(mmc->supply.vmmc);
3837 		if (curr > 0) {
3838 
3839 			/* convert to SDHCI_MAX_CURRENT format */
3840 			curr = curr/1000;  /* convert to mA */
3841 			curr = curr/SDHCI_MAX_CURRENT_MULTIPLIER;
3842 
3843 			curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT);
3844 			max_current_caps =
3845 				(curr << SDHCI_MAX_CURRENT_330_SHIFT) |
3846 				(curr << SDHCI_MAX_CURRENT_300_SHIFT) |
3847 				(curr << SDHCI_MAX_CURRENT_180_SHIFT);
3848 		}
3849 	}
3850 
3851 	if (host->caps & SDHCI_CAN_VDD_330) {
3852 		ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34;
3853 
3854 		mmc->max_current_330 = ((max_current_caps &
3855 				   SDHCI_MAX_CURRENT_330_MASK) >>
3856 				   SDHCI_MAX_CURRENT_330_SHIFT) *
3857 				   SDHCI_MAX_CURRENT_MULTIPLIER;
3858 	}
3859 	if (host->caps & SDHCI_CAN_VDD_300) {
3860 		ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31;
3861 
3862 		mmc->max_current_300 = ((max_current_caps &
3863 				   SDHCI_MAX_CURRENT_300_MASK) >>
3864 				   SDHCI_MAX_CURRENT_300_SHIFT) *
3865 				   SDHCI_MAX_CURRENT_MULTIPLIER;
3866 	}
3867 	if (host->caps & SDHCI_CAN_VDD_180) {
3868 		ocr_avail |= MMC_VDD_165_195;
3869 
3870 		mmc->max_current_180 = ((max_current_caps &
3871 				   SDHCI_MAX_CURRENT_180_MASK) >>
3872 				   SDHCI_MAX_CURRENT_180_SHIFT) *
3873 				   SDHCI_MAX_CURRENT_MULTIPLIER;
3874 	}
3875 
3876 	/* If OCR set by host, use it instead. */
3877 	if (host->ocr_mask)
3878 		ocr_avail = host->ocr_mask;
3879 
3880 	/* If OCR set by external regulators, give it highest prio. */
3881 	if (mmc->ocr_avail)
3882 		ocr_avail = mmc->ocr_avail;
3883 
3884 	mmc->ocr_avail = ocr_avail;
3885 	mmc->ocr_avail_sdio = ocr_avail;
3886 	if (host->ocr_avail_sdio)
3887 		mmc->ocr_avail_sdio &= host->ocr_avail_sdio;
3888 	mmc->ocr_avail_sd = ocr_avail;
3889 	if (host->ocr_avail_sd)
3890 		mmc->ocr_avail_sd &= host->ocr_avail_sd;
3891 	else /* normal SD controllers don't support 1.8V */
3892 		mmc->ocr_avail_sd &= ~MMC_VDD_165_195;
3893 	mmc->ocr_avail_mmc = ocr_avail;
3894 	if (host->ocr_avail_mmc)
3895 		mmc->ocr_avail_mmc &= host->ocr_avail_mmc;
3896 
3897 	if (mmc->ocr_avail == 0) {
3898 		pr_err("%s: Hardware doesn't report any support voltages.\n",
3899 		       mmc_hostname(mmc));
3900 		ret = -ENODEV;
3901 		goto unreg;
3902 	}
3903 
3904 	if ((mmc->caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
3905 			  MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104 |
3906 			  MMC_CAP_UHS_DDR50 | MMC_CAP_1_8V_DDR)) ||
3907 	    (mmc->caps2 & (MMC_CAP2_HS200_1_8V_SDR | MMC_CAP2_HS400_1_8V)))
3908 		host->flags |= SDHCI_SIGNALING_180;
3909 
3910 	if (mmc->caps2 & MMC_CAP2_HSX00_1_2V)
3911 		host->flags |= SDHCI_SIGNALING_120;
3912 
3913 	spin_lock_init(&host->lock);
3914 
3915 	/*
3916 	 * Maximum number of sectors in one transfer. Limited by SDMA boundary
3917 	 * size (512KiB). Note some tuning modes impose a 4MiB limit, but this
3918 	 * is less anyway.
3919 	 */
3920 	mmc->max_req_size = 524288;
3921 
3922 	/*
3923 	 * Maximum number of segments. Depends on if the hardware
3924 	 * can do scatter/gather or not.
3925 	 */
3926 	if (host->flags & SDHCI_USE_ADMA) {
3927 		mmc->max_segs = SDHCI_MAX_SEGS;
3928 	} else if (host->flags & SDHCI_USE_SDMA) {
3929 		mmc->max_segs = 1;
3930 		if (swiotlb_max_segment()) {
3931 			unsigned int max_req_size = (1 << IO_TLB_SHIFT) *
3932 						IO_TLB_SEGSIZE;
3933 			mmc->max_req_size = min(mmc->max_req_size,
3934 						max_req_size);
3935 		}
3936 	} else { /* PIO */
3937 		mmc->max_segs = SDHCI_MAX_SEGS;
3938 	}
3939 
3940 	/*
3941 	 * Maximum segment size. Could be one segment with the maximum number
3942 	 * of bytes. When doing hardware scatter/gather, each entry cannot
3943 	 * be larger than 64 KiB though.
3944 	 */
3945 	if (host->flags & SDHCI_USE_ADMA) {
3946 		if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC)
3947 			mmc->max_seg_size = 65535;
3948 		else
3949 			mmc->max_seg_size = 65536;
3950 	} else {
3951 		mmc->max_seg_size = mmc->max_req_size;
3952 	}
3953 
3954 	/*
3955 	 * Maximum block size. This varies from controller to controller and
3956 	 * is specified in the capabilities register.
3957 	 */
3958 	if (host->quirks & SDHCI_QUIRK_FORCE_BLK_SZ_2048) {
3959 		mmc->max_blk_size = 2;
3960 	} else {
3961 		mmc->max_blk_size = (host->caps & SDHCI_MAX_BLOCK_MASK) >>
3962 				SDHCI_MAX_BLOCK_SHIFT;
3963 		if (mmc->max_blk_size >= 3) {
3964 			pr_warn("%s: Invalid maximum block size, assuming 512 bytes\n",
3965 				mmc_hostname(mmc));
3966 			mmc->max_blk_size = 0;
3967 		}
3968 	}
3969 
3970 	mmc->max_blk_size = 512 << mmc->max_blk_size;
3971 
3972 	/*
3973 	 * Maximum block count.
3974 	 */
3975 	mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
3976 
3977 	if (mmc->max_segs == 1) {
3978 		/* This may alter mmc->*_blk_* parameters */
3979 		ret = sdhci_allocate_bounce_buffer(host);
3980 		if (ret)
3981 			return ret;
3982 	}
3983 
3984 	return 0;
3985 
3986 unreg:
3987 	if (!IS_ERR(mmc->supply.vqmmc))
3988 		regulator_disable(mmc->supply.vqmmc);
3989 undma:
3990 	if (host->align_buffer)
3991 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
3992 				  host->adma_table_sz, host->align_buffer,
3993 				  host->align_addr);
3994 	host->adma_table = NULL;
3995 	host->align_buffer = NULL;
3996 
3997 	return ret;
3998 }
3999 EXPORT_SYMBOL_GPL(sdhci_setup_host);
4000 
4001 void sdhci_cleanup_host(struct sdhci_host *host)
4002 {
4003 	struct mmc_host *mmc = host->mmc;
4004 
4005 	if (!IS_ERR(mmc->supply.vqmmc))
4006 		regulator_disable(mmc->supply.vqmmc);
4007 
4008 	if (host->align_buffer)
4009 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4010 				  host->adma_table_sz, host->align_buffer,
4011 				  host->align_addr);
4012 	host->adma_table = NULL;
4013 	host->align_buffer = NULL;
4014 }
4015 EXPORT_SYMBOL_GPL(sdhci_cleanup_host);
4016 
4017 int __sdhci_add_host(struct sdhci_host *host)
4018 {
4019 	struct mmc_host *mmc = host->mmc;
4020 	int ret;
4021 
4022 	/*
4023 	 * Init tasklets.
4024 	 */
4025 	tasklet_init(&host->finish_tasklet,
4026 		sdhci_tasklet_finish, (unsigned long)host);
4027 
4028 	timer_setup(&host->timer, sdhci_timeout_timer, 0);
4029 	timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0);
4030 
4031 	init_waitqueue_head(&host->buf_ready_int);
4032 
4033 	sdhci_init(host, 0);
4034 
4035 	ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq,
4036 				   IRQF_SHARED,	mmc_hostname(mmc), host);
4037 	if (ret) {
4038 		pr_err("%s: Failed to request IRQ %d: %d\n",
4039 		       mmc_hostname(mmc), host->irq, ret);
4040 		goto untasklet;
4041 	}
4042 
4043 	ret = sdhci_led_register(host);
4044 	if (ret) {
4045 		pr_err("%s: Failed to register LED device: %d\n",
4046 		       mmc_hostname(mmc), ret);
4047 		goto unirq;
4048 	}
4049 
4050 	mmiowb();
4051 
4052 	ret = mmc_add_host(mmc);
4053 	if (ret)
4054 		goto unled;
4055 
4056 	pr_info("%s: SDHCI controller on %s [%s] using %s\n",
4057 		mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)),
4058 		(host->flags & SDHCI_USE_ADMA) ?
4059 		(host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" :
4060 		(host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO");
4061 
4062 	sdhci_enable_card_detection(host);
4063 
4064 	return 0;
4065 
4066 unled:
4067 	sdhci_led_unregister(host);
4068 unirq:
4069 	sdhci_do_reset(host, SDHCI_RESET_ALL);
4070 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4071 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4072 	free_irq(host->irq, host);
4073 untasklet:
4074 	tasklet_kill(&host->finish_tasklet);
4075 
4076 	return ret;
4077 }
4078 EXPORT_SYMBOL_GPL(__sdhci_add_host);
4079 
4080 int sdhci_add_host(struct sdhci_host *host)
4081 {
4082 	int ret;
4083 
4084 	ret = sdhci_setup_host(host);
4085 	if (ret)
4086 		return ret;
4087 
4088 	ret = __sdhci_add_host(host);
4089 	if (ret)
4090 		goto cleanup;
4091 
4092 	return 0;
4093 
4094 cleanup:
4095 	sdhci_cleanup_host(host);
4096 
4097 	return ret;
4098 }
4099 EXPORT_SYMBOL_GPL(sdhci_add_host);
4100 
4101 void sdhci_remove_host(struct sdhci_host *host, int dead)
4102 {
4103 	struct mmc_host *mmc = host->mmc;
4104 	unsigned long flags;
4105 
4106 	if (dead) {
4107 		spin_lock_irqsave(&host->lock, flags);
4108 
4109 		host->flags |= SDHCI_DEVICE_DEAD;
4110 
4111 		if (sdhci_has_requests(host)) {
4112 			pr_err("%s: Controller removed during "
4113 				" transfer!\n", mmc_hostname(mmc));
4114 			sdhci_error_out_mrqs(host, -ENOMEDIUM);
4115 		}
4116 
4117 		spin_unlock_irqrestore(&host->lock, flags);
4118 	}
4119 
4120 	sdhci_disable_card_detection(host);
4121 
4122 	mmc_remove_host(mmc);
4123 
4124 	sdhci_led_unregister(host);
4125 
4126 	if (!dead)
4127 		sdhci_do_reset(host, SDHCI_RESET_ALL);
4128 
4129 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
4130 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
4131 	free_irq(host->irq, host);
4132 
4133 	del_timer_sync(&host->timer);
4134 	del_timer_sync(&host->data_timer);
4135 
4136 	tasklet_kill(&host->finish_tasklet);
4137 
4138 	if (!IS_ERR(mmc->supply.vqmmc))
4139 		regulator_disable(mmc->supply.vqmmc);
4140 
4141 	if (host->align_buffer)
4142 		dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz +
4143 				  host->adma_table_sz, host->align_buffer,
4144 				  host->align_addr);
4145 
4146 	host->adma_table = NULL;
4147 	host->align_buffer = NULL;
4148 }
4149 
4150 EXPORT_SYMBOL_GPL(sdhci_remove_host);
4151 
4152 void sdhci_free_host(struct sdhci_host *host)
4153 {
4154 	mmc_free_host(host->mmc);
4155 }
4156 
4157 EXPORT_SYMBOL_GPL(sdhci_free_host);
4158 
4159 /*****************************************************************************\
4160  *                                                                           *
4161  * Driver init/exit                                                          *
4162  *                                                                           *
4163 \*****************************************************************************/
4164 
4165 static int __init sdhci_drv_init(void)
4166 {
4167 	pr_info(DRIVER_NAME
4168 		": Secure Digital Host Controller Interface driver\n");
4169 	pr_info(DRIVER_NAME ": Copyright(c) Pierre Ossman\n");
4170 
4171 	return 0;
4172 }
4173 
4174 static void __exit sdhci_drv_exit(void)
4175 {
4176 }
4177 
4178 module_init(sdhci_drv_init);
4179 module_exit(sdhci_drv_exit);
4180 
4181 module_param(debug_quirks, uint, 0444);
4182 module_param(debug_quirks2, uint, 0444);
4183 
4184 MODULE_AUTHOR("Pierre Ossman <pierre@ossman.eu>");
4185 MODULE_DESCRIPTION("Secure Digital Host Controller Interface core driver");
4186 MODULE_LICENSE("GPL");
4187 
4188 MODULE_PARM_DESC(debug_quirks, "Force certain quirks.");
4189 MODULE_PARM_DESC(debug_quirks2, "Force certain other quirks.");
4190