xref: /freebsd/sys/dev/sdhci/sdhci.c (revision 681ce946f33e75c590e97c53076e86dff1fe8f4a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2008 Alexander Motin <mav@FreeBSD.org>
5  * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/callout.h>
36 #include <sys/conf.h>
37 #include <sys/kernel.h>
38 #include <sys/kobj.h>
39 #include <sys/libkern.h>
40 #include <sys/lock.h>
41 #include <sys/malloc.h>
42 #include <sys/module.h>
43 #include <sys/mutex.h>
44 #include <sys/resource.h>
45 #include <sys/rman.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/sbuf.h>
49 
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52 #include <machine/stdarg.h>
53 
54 #include <dev/mmc/bridge.h>
55 #include <dev/mmc/mmcreg.h>
56 #include <dev/mmc/mmcbrvar.h>
57 
58 #include <dev/sdhci/sdhci.h>
59 
60 #include <cam/cam.h>
61 #include <cam/cam_ccb.h>
62 #include <cam/cam_debug.h>
63 #include <cam/cam_sim.h>
64 #include <cam/cam_xpt_sim.h>
65 
66 #include "mmcbr_if.h"
67 #include "sdhci_if.h"
68 
69 #include "opt_mmccam.h"
70 
71 SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
72     "sdhci driver");
73 
74 static int sdhci_debug = 0;
75 SYSCTL_INT(_hw_sdhci, OID_AUTO, debug, CTLFLAG_RWTUN, &sdhci_debug, 0,
76     "Debug level");
77 u_int sdhci_quirk_clear = 0;
78 SYSCTL_INT(_hw_sdhci, OID_AUTO, quirk_clear, CTLFLAG_RWTUN, &sdhci_quirk_clear,
79     0, "Mask of quirks to clear");
80 u_int sdhci_quirk_set = 0;
81 SYSCTL_INT(_hw_sdhci, OID_AUTO, quirk_set, CTLFLAG_RWTUN, &sdhci_quirk_set, 0,
82     "Mask of quirks to set");
83 
84 #define	RD1(slot, off)	SDHCI_READ_1((slot)->bus, (slot), (off))
85 #define	RD2(slot, off)	SDHCI_READ_2((slot)->bus, (slot), (off))
86 #define	RD4(slot, off)	SDHCI_READ_4((slot)->bus, (slot), (off))
87 #define	RD_MULTI_4(slot, off, ptr, count)	\
88     SDHCI_READ_MULTI_4((slot)->bus, (slot), (off), (ptr), (count))
89 
90 #define	WR1(slot, off, val)	SDHCI_WRITE_1((slot)->bus, (slot), (off), (val))
91 #define	WR2(slot, off, val)	SDHCI_WRITE_2((slot)->bus, (slot), (off), (val))
92 #define	WR4(slot, off, val)	SDHCI_WRITE_4((slot)->bus, (slot), (off), (val))
93 #define	WR_MULTI_4(slot, off, ptr, count)	\
94     SDHCI_WRITE_MULTI_4((slot)->bus, (slot), (off), (ptr), (count))
95 
96 static void sdhci_acmd_irq(struct sdhci_slot *slot, uint16_t acmd_err);
97 static void sdhci_card_poll(void *arg);
98 static void sdhci_card_task(void *arg, int pending);
99 static void sdhci_cmd_irq(struct sdhci_slot *slot, uint32_t intmask);
100 static void sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask);
101 static int sdhci_exec_tuning(struct sdhci_slot *slot, bool reset);
102 static void sdhci_handle_card_present_locked(struct sdhci_slot *slot,
103     bool is_present);
104 static void sdhci_finish_command(struct sdhci_slot *slot);
105 static void sdhci_init(struct sdhci_slot *slot);
106 static void sdhci_read_block_pio(struct sdhci_slot *slot);
107 static void sdhci_req_done(struct sdhci_slot *slot);
108 static void sdhci_req_wakeup(struct mmc_request *req);
109 static void sdhci_retune(void *arg);
110 static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock);
111 static void sdhci_set_power(struct sdhci_slot *slot, u_char power);
112 static void sdhci_set_transfer_mode(struct sdhci_slot *slot,
113    const struct mmc_data *data);
114 static void sdhci_start(struct sdhci_slot *slot);
115 static void sdhci_timeout(void *arg);
116 static void sdhci_start_command(struct sdhci_slot *slot,
117    struct mmc_command *cmd);
118 static void sdhci_start_data(struct sdhci_slot *slot,
119    const struct mmc_data *data);
120 static void sdhci_write_block_pio(struct sdhci_slot *slot);
121 static void sdhci_transfer_pio(struct sdhci_slot *slot);
122 
123 #ifdef MMCCAM
124 /* CAM-related */
125 static void sdhci_cam_action(struct cam_sim *sim, union ccb *ccb);
126 static int sdhci_cam_get_possible_host_clock(const struct sdhci_slot *slot,
127     int proposed_clock);
128 static void sdhci_cam_poll(struct cam_sim *sim);
129 static int sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb);
130 static int sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb);
131 static int sdhci_cam_update_ios(struct sdhci_slot *slot);
132 #endif
133 
134 /* helper routines */
135 static int sdhci_dma_alloc(struct sdhci_slot *slot, uint32_t caps);
136 static void sdhci_dma_free(struct sdhci_slot *slot);
137 static void sdhci_dumpcaps(struct sdhci_slot *slot);
138 static void sdhci_dumpcaps_buf(struct sdhci_slot *slot, struct sbuf *s);
139 static void sdhci_dumpregs(struct sdhci_slot *slot);
140 static void sdhci_dumpregs_buf(struct sdhci_slot *slot, struct sbuf *s);
141 static int sdhci_syctl_dumpcaps(SYSCTL_HANDLER_ARGS);
142 static int sdhci_syctl_dumpregs(SYSCTL_HANDLER_ARGS);
143 static void sdhci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs,
144     int error);
145 static int slot_printf(const struct sdhci_slot *slot, const char * fmt, ...)
146     __printflike(2, 3);
147 static int slot_sprintf(const struct sdhci_slot *slot, struct sbuf *s,
148     const char * fmt, ...) __printflike(3, 4);
149 static uint32_t sdhci_tuning_intmask(const struct sdhci_slot *slot);
150 
151 #define	SDHCI_LOCK(_slot)		mtx_lock(&(_slot)->mtx)
152 #define	SDHCI_UNLOCK(_slot)		mtx_unlock(&(_slot)->mtx)
153 #define	SDHCI_LOCK_INIT(_slot) \
154 	mtx_init(&_slot->mtx, "SD slot mtx", "sdhci", MTX_DEF)
155 #define	SDHCI_LOCK_DESTROY(_slot)	mtx_destroy(&_slot->mtx);
156 #define	SDHCI_ASSERT_LOCKED(_slot)	mtx_assert(&_slot->mtx, MA_OWNED);
157 #define	SDHCI_ASSERT_UNLOCKED(_slot)	mtx_assert(&_slot->mtx, MA_NOTOWNED);
158 
159 #define	SDHCI_DEFAULT_MAX_FREQ	50
160 
161 #define	SDHCI_200_MAX_DIVIDER	256
162 #define	SDHCI_300_MAX_DIVIDER	2046
163 
164 #define	SDHCI_CARD_PRESENT_TICKS	(hz / 5)
165 #define	SDHCI_INSERT_DELAY_TICKS	(hz / 2)
166 
167 /*
168  * Broadcom BCM577xx Controller Constants
169  */
170 /* Maximum divider supported by the default clock source. */
171 #define	BCM577XX_DEFAULT_MAX_DIVIDER	256
172 /* Alternative clock's base frequency. */
173 #define	BCM577XX_ALT_CLOCK_BASE		63000000
174 
175 #define	BCM577XX_HOST_CONTROL		0x198
176 #define	BCM577XX_CTRL_CLKSEL_MASK	0xFFFFCFFF
177 #define	BCM577XX_CTRL_CLKSEL_SHIFT	12
178 #define	BCM577XX_CTRL_CLKSEL_DEFAULT	0x0
179 #define	BCM577XX_CTRL_CLKSEL_64MHZ	0x3
180 
181 static void
182 sdhci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
183 {
184 
185 	if (error != 0) {
186 		printf("getaddr: error %d\n", error);
187 		return;
188 	}
189 	*(bus_addr_t *)arg = segs[0].ds_addr;
190 }
191 
192 static int
193 slot_printf(const struct sdhci_slot *slot, const char * fmt, ...)
194 {
195 	char buf[128];
196 	va_list ap;
197 	int retval;
198 
199 	/*
200 	 * Make sure we print a single line all together rather than in two
201 	 * halves to avoid console gibberish bingo.
202 	 */
203 	va_start(ap, fmt);
204 	retval = vsnprintf(buf, sizeof(buf), fmt, ap);
205 	va_end(ap);
206 
207 	retval += printf("%s-slot%d: %s",
208 	    device_get_nameunit(slot->bus), slot->num, buf);
209 	return (retval);
210 }
211 
212 static int
213 slot_sprintf(const struct sdhci_slot *slot, struct sbuf *s,
214     const char * fmt, ...)
215 {
216 	va_list ap;
217 	int retval;
218 
219 	retval = sbuf_printf(s, "%s-slot%d: ", device_get_nameunit(slot->bus), slot->num);
220 
221 	va_start(ap, fmt);
222 	retval += sbuf_vprintf(s, fmt, ap);
223 	va_end(ap);
224 
225 	return (retval);
226 }
227 
228 static void
229 sdhci_dumpregs_buf(struct sdhci_slot *slot, struct sbuf *s)
230 {
231 	slot_sprintf(slot, s,  "============== REGISTER DUMP ==============\n");
232 
233 	slot_sprintf(slot, s,  "Sys addr: 0x%08x | Version:  0x%08x\n",
234 	    RD4(slot, SDHCI_DMA_ADDRESS), RD2(slot, SDHCI_HOST_VERSION));
235 	slot_sprintf(slot, s,  "Blk size: 0x%08x | Blk cnt:  0x%08x\n",
236 	    RD2(slot, SDHCI_BLOCK_SIZE), RD2(slot, SDHCI_BLOCK_COUNT));
237 	slot_sprintf(slot, s,  "Argument: 0x%08x | Trn mode: 0x%08x\n",
238 	    RD4(slot, SDHCI_ARGUMENT), RD2(slot, SDHCI_TRANSFER_MODE));
239 	slot_sprintf(slot, s,  "Present:  0x%08x | Host ctl: 0x%08x\n",
240 	    RD4(slot, SDHCI_PRESENT_STATE), RD1(slot, SDHCI_HOST_CONTROL));
241 	slot_sprintf(slot, s,  "Power:    0x%08x | Blk gap:  0x%08x\n",
242 	    RD1(slot, SDHCI_POWER_CONTROL), RD1(slot, SDHCI_BLOCK_GAP_CONTROL));
243 	slot_sprintf(slot, s,  "Wake-up:  0x%08x | Clock:    0x%08x\n",
244 	    RD1(slot, SDHCI_WAKE_UP_CONTROL), RD2(slot, SDHCI_CLOCK_CONTROL));
245 	slot_sprintf(slot, s,  "Timeout:  0x%08x | Int stat: 0x%08x\n",
246 	    RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS));
247 	slot_sprintf(slot, s,  "Int enab: 0x%08x | Sig enab: 0x%08x\n",
248 	    RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE));
249 	slot_sprintf(slot, s,  "AC12 err: 0x%08x | Host ctl2:0x%08x\n",
250 	    RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_HOST_CONTROL2));
251 	slot_sprintf(slot, s,  "Caps:     0x%08x | Caps2:    0x%08x\n",
252 	    RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_CAPABILITIES2));
253 	slot_sprintf(slot, s,  "Max curr: 0x%08x | ADMA err: 0x%08x\n",
254 	    RD4(slot, SDHCI_MAX_CURRENT), RD1(slot, SDHCI_ADMA_ERR));
255 	slot_sprintf(slot, s,  "ADMA addr:0x%08x | Slot int: 0x%08x\n",
256 	    RD4(slot, SDHCI_ADMA_ADDRESS_LO), RD2(slot, SDHCI_SLOT_INT_STATUS));
257 
258 	slot_sprintf(slot, s,  "===========================================\n");
259 }
260 
261 static void
262 sdhci_dumpregs(struct sdhci_slot *slot)
263 {
264 	struct sbuf s;
265 
266 	if (sbuf_new(&s, NULL, 1024, SBUF_NOWAIT | SBUF_AUTOEXTEND) == NULL) {
267 		slot_printf(slot, "sdhci_dumpregs: Failed to allocate memory for sbuf\n");
268 		return;
269 	}
270 
271 	sbuf_set_drain(&s, &sbuf_printf_drain, NULL);
272 	sdhci_dumpregs_buf(slot, &s);
273 	sbuf_finish(&s);
274 	sbuf_delete(&s);
275 }
276 
277 static int
278 sdhci_syctl_dumpregs(SYSCTL_HANDLER_ARGS)
279 {
280 	struct sdhci_slot *slot = arg1;
281 	struct sbuf s;
282 
283 	sbuf_new_for_sysctl(&s, NULL, 1024, req);
284 	sbuf_putc(&s, '\n');
285 	sdhci_dumpregs_buf(slot, &s);
286 	sbuf_finish(&s);
287 	sbuf_delete(&s);
288 
289 	return (0);
290 }
291 
292 static void
293 sdhci_dumpcaps_buf(struct sdhci_slot *slot, struct sbuf *s)
294 {
295 	int host_caps = slot->host.caps;
296 	int caps = slot->caps;
297 
298 	slot_sprintf(slot, s,
299 	    "%uMHz%s %s VDD:%s%s%s VCCQ: 3.3V%s%s DRV: B%s%s%s %s %s\n",
300 	    slot->max_clk / 1000000,
301 	    (caps & SDHCI_CAN_DO_HISPD) ? " HS" : "",
302 	    (host_caps & MMC_CAP_8_BIT_DATA) ? "8bits" :
303 	    ((host_caps & MMC_CAP_4_BIT_DATA) ? "4bits" : "1bit"),
304 	    (caps & SDHCI_CAN_VDD_330) ? " 3.3V" : "",
305 	    (caps & SDHCI_CAN_VDD_300) ? " 3.0V" : "",
306 	    ((caps & SDHCI_CAN_VDD_180) &&
307 	    (slot->opt & SDHCI_SLOT_EMBEDDED)) ? " 1.8V" : "",
308 	    (host_caps & MMC_CAP_SIGNALING_180) ? " 1.8V" : "",
309 	    (host_caps & MMC_CAP_SIGNALING_120) ? " 1.2V" : "",
310 	    (host_caps & MMC_CAP_DRIVER_TYPE_A) ? "A" : "",
311 	    (host_caps & MMC_CAP_DRIVER_TYPE_C) ? "C" : "",
312 	    (host_caps & MMC_CAP_DRIVER_TYPE_D) ? "D" : "",
313 	    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO",
314 	    (slot->opt & SDHCI_SLOT_EMBEDDED) ? "embedded" :
315 	    (slot->opt & SDHCI_NON_REMOVABLE) ? "non-removable" :
316 	    "removable");
317 	if (host_caps & (MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 |
318 	    MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE))
319 		slot_sprintf(slot, s, "eMMC:%s%s%s%s\n",
320 		    (host_caps & MMC_CAP_MMC_DDR52) ? " DDR52" : "",
321 		    (host_caps & MMC_CAP_MMC_HS200) ? " HS200" : "",
322 		    (host_caps & MMC_CAP_MMC_HS400) ? " HS400" : "",
323 		    ((host_caps &
324 		    (MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE)) ==
325 		    (MMC_CAP_MMC_HS400 | MMC_CAP_MMC_ENH_STROBE)) ?
326 		    " HS400ES" : "");
327 	if (host_caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
328 	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_SDR104))
329 		slot_sprintf(slot, s, "UHS-I:%s%s%s%s%s\n",
330 		    (host_caps & MMC_CAP_UHS_SDR12) ? " SDR12" : "",
331 		    (host_caps & MMC_CAP_UHS_SDR25) ? " SDR25" : "",
332 		    (host_caps & MMC_CAP_UHS_SDR50) ? " SDR50" : "",
333 		    (host_caps & MMC_CAP_UHS_SDR104) ? " SDR104" : "",
334 		    (host_caps & MMC_CAP_UHS_DDR50) ? " DDR50" : "");
335 	if (slot->opt & SDHCI_TUNING_SUPPORTED)
336 		slot_sprintf(slot, s,
337 		    "Re-tuning count %d secs, mode %d\n",
338 		    slot->retune_count, slot->retune_mode + 1);
339 }
340 
341 static void
342 sdhci_dumpcaps(struct sdhci_slot *slot)
343 {
344 	struct sbuf s;
345 
346 	if (sbuf_new(&s, NULL, 1024, SBUF_NOWAIT | SBUF_AUTOEXTEND) == NULL) {
347 		slot_printf(slot, "sdhci_dumpcaps: Failed to allocate memory for sbuf\n");
348 		return;
349 	}
350 
351 	sbuf_set_drain(&s, &sbuf_printf_drain, NULL);
352 	sdhci_dumpcaps_buf(slot, &s);
353 	sbuf_finish(&s);
354 	sbuf_delete(&s);
355 }
356 
357 static int
358 sdhci_syctl_dumpcaps(SYSCTL_HANDLER_ARGS)
359 {
360 	struct sdhci_slot *slot = arg1;
361 	struct sbuf s;
362 
363 	sbuf_new_for_sysctl(&s, NULL, 1024, req);
364 	sbuf_putc(&s, '\n');
365 	sdhci_dumpcaps_buf(slot, &s);
366 	sbuf_finish(&s);
367 	sbuf_delete(&s);
368 
369 	return (0);
370 }
371 
372 static uint32_t
373 sdhci_tuning_intmask(const struct sdhci_slot *slot)
374 {
375 	uint32_t intmask;
376 
377 	intmask = 0;
378 	if (slot->opt & SDHCI_TUNING_ENABLED) {
379 		intmask |= SDHCI_INT_TUNEERR;
380 		if (slot->retune_mode == SDHCI_RETUNE_MODE_2 ||
381 		    slot->retune_mode == SDHCI_RETUNE_MODE_3)
382 			intmask |= SDHCI_INT_RETUNE;
383 	}
384 	return (intmask);
385 }
386 
387 static void
388 sdhci_init(struct sdhci_slot *slot)
389 {
390 
391 	SDHCI_RESET(slot->bus, slot, SDHCI_RESET_ALL);
392 
393 	/* Enable interrupts. */
394 	slot->intmask = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
395 	    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT | SDHCI_INT_INDEX |
396 	    SDHCI_INT_END_BIT | SDHCI_INT_CRC | SDHCI_INT_TIMEOUT |
397 	    SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
398 	    SDHCI_INT_DMA_END | SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE |
399 	    SDHCI_INT_ACMD12ERR;
400 
401 	if (!(slot->quirks & SDHCI_QUIRK_POLL_CARD_PRESENT) &&
402 	    !(slot->opt & SDHCI_NON_REMOVABLE)) {
403 		slot->intmask |= SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT;
404 	}
405 
406 	WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
407 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
408 }
409 
410 static void
411 sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
412 {
413 	uint32_t clk_base;
414 	uint32_t clk_sel;
415 	uint32_t res;
416 	uint16_t clk;
417 	uint16_t div;
418 	int timeout;
419 
420 	if (clock == slot->clock)
421 		return;
422 	slot->clock = clock;
423 
424 	/* Turn off the clock. */
425 	clk = RD2(slot, SDHCI_CLOCK_CONTROL);
426 	WR2(slot, SDHCI_CLOCK_CONTROL, clk & ~SDHCI_CLOCK_CARD_EN);
427 	/* If no clock requested - leave it so. */
428 	if (clock == 0)
429 		return;
430 
431 	/* Determine the clock base frequency */
432 	clk_base = slot->max_clk;
433 	if (slot->quirks & SDHCI_QUIRK_BCM577XX_400KHZ_CLKSRC) {
434 		clk_sel = RD2(slot, BCM577XX_HOST_CONTROL) &
435 		    BCM577XX_CTRL_CLKSEL_MASK;
436 
437 		/*
438 		 * Select clock source appropriate for the requested frequency.
439 		 */
440 		if ((clk_base / BCM577XX_DEFAULT_MAX_DIVIDER) > clock) {
441 			clk_base = BCM577XX_ALT_CLOCK_BASE;
442 			clk_sel |= (BCM577XX_CTRL_CLKSEL_64MHZ <<
443 			    BCM577XX_CTRL_CLKSEL_SHIFT);
444 		} else {
445 			clk_sel |= (BCM577XX_CTRL_CLKSEL_DEFAULT <<
446 			    BCM577XX_CTRL_CLKSEL_SHIFT);
447 		}
448 
449 		WR2(slot, BCM577XX_HOST_CONTROL, clk_sel);
450 	}
451 
452 	/* Recalculate timeout clock frequency based on the new sd clock. */
453 	if (slot->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)
454 		slot->timeout_clk = slot->clock / 1000;
455 
456 	if (slot->version < SDHCI_SPEC_300) {
457 		/* Looking for highest freq <= clock. */
458 		res = clk_base;
459 		for (div = 1; div < SDHCI_200_MAX_DIVIDER; div <<= 1) {
460 			if (res <= clock)
461 				break;
462 			res >>= 1;
463 		}
464 		/* Divider 1:1 is 0x00, 2:1 is 0x01, 256:1 is 0x80 ... */
465 		div >>= 1;
466 	} else {
467 		/* Version 3.0 divisors are multiples of two up to 1023 * 2 */
468 		if (clock >= clk_base)
469 			div = 0;
470 		else {
471 			for (div = 2; div < SDHCI_300_MAX_DIVIDER; div += 2) {
472 				if ((clk_base / div) <= clock)
473 					break;
474 			}
475 		}
476 		div >>= 1;
477 	}
478 
479 	if (bootverbose || sdhci_debug)
480 		slot_printf(slot, "Divider %d for freq %d (base %d)\n",
481 			div, clock, clk_base);
482 
483 	/* Now we have got divider, set it. */
484 	clk = (div & SDHCI_DIVIDER_MASK) << SDHCI_DIVIDER_SHIFT;
485 	clk |= ((div >> SDHCI_DIVIDER_MASK_LEN) & SDHCI_DIVIDER_HI_MASK)
486 		<< SDHCI_DIVIDER_HI_SHIFT;
487 
488 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
489 	/* Enable clock. */
490 	clk |= SDHCI_CLOCK_INT_EN;
491 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
492 	/* Wait up to 10 ms until it stabilize. */
493 	timeout = 10;
494 	while (!((clk = RD2(slot, SDHCI_CLOCK_CONTROL))
495 		& SDHCI_CLOCK_INT_STABLE)) {
496 		if (timeout == 0) {
497 			slot_printf(slot,
498 			    "Internal clock never stabilised.\n");
499 			sdhci_dumpregs(slot);
500 			return;
501 		}
502 		timeout--;
503 		DELAY(1000);
504 	}
505 	/* Pass clock signal to the bus. */
506 	clk |= SDHCI_CLOCK_CARD_EN;
507 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
508 }
509 
510 static void
511 sdhci_set_power(struct sdhci_slot *slot, u_char power)
512 {
513 	int i;
514 	uint8_t pwr;
515 
516 	if (slot->power == power)
517 		return;
518 
519 	slot->power = power;
520 
521 	/* Turn off the power. */
522 	pwr = 0;
523 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
524 	/* If power down requested - leave it so. */
525 	if (power == 0)
526 		return;
527 	/* Set voltage. */
528 	switch (1 << power) {
529 	case MMC_OCR_LOW_VOLTAGE:
530 		pwr |= SDHCI_POWER_180;
531 		break;
532 	case MMC_OCR_290_300:
533 	case MMC_OCR_300_310:
534 		pwr |= SDHCI_POWER_300;
535 		break;
536 	case MMC_OCR_320_330:
537 	case MMC_OCR_330_340:
538 		pwr |= SDHCI_POWER_330;
539 		break;
540 	}
541 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
542 	/*
543 	 * Turn on VDD1 power.  Note that at least some Intel controllers can
544 	 * fail to enable bus power on the first try after transiting from D3
545 	 * to D0, so we give them up to 2 ms.
546 	 */
547 	pwr |= SDHCI_POWER_ON;
548 	for (i = 0; i < 20; i++) {
549 		WR1(slot, SDHCI_POWER_CONTROL, pwr);
550 		if (RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON)
551 			break;
552 		DELAY(100);
553 	}
554 	if (!(RD1(slot, SDHCI_POWER_CONTROL) & SDHCI_POWER_ON))
555 		slot_printf(slot, "Bus power failed to enable\n");
556 
557 	if (slot->quirks & SDHCI_QUIRK_INTEL_POWER_UP_RESET) {
558 		WR1(slot, SDHCI_POWER_CONTROL, pwr | 0x10);
559 		DELAY(10);
560 		WR1(slot, SDHCI_POWER_CONTROL, pwr);
561 		DELAY(300);
562 	}
563 }
564 
565 static void
566 sdhci_read_block_pio(struct sdhci_slot *slot)
567 {
568 	uint32_t data;
569 	char *buffer;
570 	size_t left;
571 
572 	buffer = slot->curcmd->data->data;
573 	buffer += slot->offset;
574 	/* Transfer one block at a time. */
575 #ifdef MMCCAM
576 	if (slot->curcmd->data->flags & MMC_DATA_BLOCK_SIZE)
577 		left = min(slot->curcmd->data->block_size,
578 		    slot->curcmd->data->len - slot->offset);
579 	else
580 #endif
581 		left = min(512, slot->curcmd->data->len - slot->offset);
582 	slot->offset += left;
583 
584 	/* If we are too fast, broken controllers return zeroes. */
585 	if (slot->quirks & SDHCI_QUIRK_BROKEN_TIMINGS)
586 		DELAY(10);
587 	/* Handle unaligned and aligned buffer cases. */
588 	if ((intptr_t)buffer & 3) {
589 		while (left > 3) {
590 			data = RD4(slot, SDHCI_BUFFER);
591 			buffer[0] = data;
592 			buffer[1] = (data >> 8);
593 			buffer[2] = (data >> 16);
594 			buffer[3] = (data >> 24);
595 			buffer += 4;
596 			left -= 4;
597 		}
598 	} else {
599 		RD_MULTI_4(slot, SDHCI_BUFFER,
600 		    (uint32_t *)buffer, left >> 2);
601 		left &= 3;
602 	}
603 	/* Handle uneven size case. */
604 	if (left > 0) {
605 		data = RD4(slot, SDHCI_BUFFER);
606 		while (left > 0) {
607 			*(buffer++) = data;
608 			data >>= 8;
609 			left--;
610 		}
611 	}
612 }
613 
614 static void
615 sdhci_write_block_pio(struct sdhci_slot *slot)
616 {
617 	uint32_t data = 0;
618 	char *buffer;
619 	size_t left;
620 
621 	buffer = slot->curcmd->data->data;
622 	buffer += slot->offset;
623 	/* Transfer one block at a time. */
624 #ifdef MMCCAM
625 	if (slot->curcmd->data->flags & MMC_DATA_BLOCK_SIZE) {
626 		left = min(slot->curcmd->data->block_size,
627 		    slot->curcmd->data->len - slot->offset);
628 	} else
629 #endif
630 		left = min(512, slot->curcmd->data->len - slot->offset);
631 	slot->offset += left;
632 
633 	/* Handle unaligned and aligned buffer cases. */
634 	if ((intptr_t)buffer & 3) {
635 		while (left > 3) {
636 			data = buffer[0] +
637 			    (buffer[1] << 8) +
638 			    (buffer[2] << 16) +
639 			    (buffer[3] << 24);
640 			left -= 4;
641 			buffer += 4;
642 			WR4(slot, SDHCI_BUFFER, data);
643 		}
644 	} else {
645 		WR_MULTI_4(slot, SDHCI_BUFFER,
646 		    (uint32_t *)buffer, left >> 2);
647 		left &= 3;
648 	}
649 	/* Handle uneven size case. */
650 	if (left > 0) {
651 		while (left > 0) {
652 			data <<= 8;
653 			data += *(buffer++);
654 			left--;
655 		}
656 		WR4(slot, SDHCI_BUFFER, data);
657 	}
658 }
659 
660 static void
661 sdhci_transfer_pio(struct sdhci_slot *slot)
662 {
663 
664 	/* Read as many blocks as possible. */
665 	if (slot->curcmd->data->flags & MMC_DATA_READ) {
666 		while (RD4(slot, SDHCI_PRESENT_STATE) &
667 		    SDHCI_DATA_AVAILABLE) {
668 			sdhci_read_block_pio(slot);
669 			if (slot->offset >= slot->curcmd->data->len)
670 				break;
671 		}
672 	} else {
673 		while (RD4(slot, SDHCI_PRESENT_STATE) &
674 		    SDHCI_SPACE_AVAILABLE) {
675 			sdhci_write_block_pio(slot);
676 			if (slot->offset >= slot->curcmd->data->len)
677 				break;
678 		}
679 	}
680 }
681 
682 static void
683 sdhci_card_task(void *arg, int pending __unused)
684 {
685 	struct sdhci_slot *slot = arg;
686 	device_t d;
687 
688 	SDHCI_LOCK(slot);
689 	if (SDHCI_GET_CARD_PRESENT(slot->bus, slot)) {
690 #ifdef MMCCAM
691 		if (slot->card_present == 0) {
692 #else
693 		if (slot->dev == NULL) {
694 #endif
695 			/* If card is present - attach mmc bus. */
696 			if (bootverbose || sdhci_debug)
697 				slot_printf(slot, "Card inserted\n");
698 #ifdef MMCCAM
699 			slot->card_present = 1;
700 			mmccam_start_discovery(slot->sim);
701 			SDHCI_UNLOCK(slot);
702 #else
703 			d = slot->dev = device_add_child(slot->bus, "mmc", -1);
704 			SDHCI_UNLOCK(slot);
705 			if (d) {
706 				device_set_ivars(d, slot);
707 				(void)device_probe_and_attach(d);
708 			}
709 #endif
710 		} else
711 			SDHCI_UNLOCK(slot);
712 	} else {
713 #ifdef MMCCAM
714 		if (slot->card_present == 1) {
715 #else
716 		if (slot->dev != NULL) {
717 #endif
718 			/* If no card present - detach mmc bus. */
719 			if (bootverbose || sdhci_debug)
720 				slot_printf(slot, "Card removed\n");
721 			d = slot->dev;
722 			slot->dev = NULL;
723 #ifdef MMCCAM
724 			slot->card_present = 0;
725 			mmccam_start_discovery(slot->sim);
726 			SDHCI_UNLOCK(slot);
727 #else
728 			slot->intmask &= ~sdhci_tuning_intmask(slot);
729 			WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
730 			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
731 			slot->opt &= ~SDHCI_TUNING_ENABLED;
732 			SDHCI_UNLOCK(slot);
733 			callout_drain(&slot->retune_callout);
734 			device_delete_child(slot->bus, d);
735 #endif
736 		} else
737 			SDHCI_UNLOCK(slot);
738 	}
739 }
740 
741 static void
742 sdhci_handle_card_present_locked(struct sdhci_slot *slot, bool is_present)
743 {
744 	bool was_present;
745 
746 	/*
747 	 * If there was no card and now there is one, schedule the task to
748 	 * create the child device after a short delay.  The delay is to
749 	 * debounce the card insert (sometimes the card detect pin stabilizes
750 	 * before the other pins have made good contact).
751 	 *
752 	 * If there was a card present and now it's gone, immediately schedule
753 	 * the task to delete the child device.  No debouncing -- gone is gone,
754 	 * because once power is removed, a full card re-init is needed, and
755 	 * that happens by deleting and recreating the child device.
756 	 */
757 #ifdef MMCCAM
758 	was_present = slot->card_present;
759 #else
760 	was_present = slot->dev != NULL;
761 #endif
762 	if (!was_present && is_present) {
763 		taskqueue_enqueue_timeout(taskqueue_swi_giant,
764 		    &slot->card_delayed_task, -SDHCI_INSERT_DELAY_TICKS);
765 	} else if (was_present && !is_present) {
766 		taskqueue_enqueue(taskqueue_swi_giant, &slot->card_task);
767 	}
768 }
769 
770 void
771 sdhci_handle_card_present(struct sdhci_slot *slot, bool is_present)
772 {
773 
774 	SDHCI_LOCK(slot);
775 	sdhci_handle_card_present_locked(slot, is_present);
776 	SDHCI_UNLOCK(slot);
777 }
778 
779 static void
780 sdhci_card_poll(void *arg)
781 {
782 	struct sdhci_slot *slot = arg;
783 
784 	sdhci_handle_card_present(slot,
785 	    SDHCI_GET_CARD_PRESENT(slot->bus, slot));
786 	callout_reset(&slot->card_poll_callout, SDHCI_CARD_PRESENT_TICKS,
787 	    sdhci_card_poll, slot);
788 }
789 
790 static int
791 sdhci_dma_alloc(struct sdhci_slot *slot, uint32_t caps)
792 {
793 	int err;
794 
795 	if (!(slot->quirks & SDHCI_QUIRK_BROKEN_SDMA_BOUNDARY)) {
796 		if (maxphys <= 1024 * 4)
797 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_4K;
798 		else if (maxphys <= 1024 * 8)
799 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_8K;
800 		else if (maxphys <= 1024 * 16)
801 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_16K;
802 		else if (maxphys <= 1024 * 32)
803 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_32K;
804 		else if (maxphys <= 1024 * 64)
805 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_64K;
806 		else if (maxphys <= 1024 * 128)
807 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_128K;
808 		else if (maxphys <= 1024 * 256)
809 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_256K;
810 		else
811 			slot->sdma_boundary = SDHCI_BLKSZ_SDMA_BNDRY_512K;
812 	}
813 	slot->sdma_bbufsz = SDHCI_SDMA_BNDRY_TO_BBUFSZ(slot->sdma_boundary);
814 
815 	/*
816 	 * Allocate the DMA tag for an SDMA bounce buffer.
817 	 * Note that the SDHCI specification doesn't state any alignment
818 	 * constraint for the SDMA system address.  However, controllers
819 	 * typically ignore the SDMA boundary bits in SDHCI_DMA_ADDRESS when
820 	 * forming the actual address of data, requiring the SDMA buffer to
821 	 * be aligned to the SDMA boundary.
822 	 */
823 	err = bus_dma_tag_create(bus_get_dma_tag(slot->bus), slot->sdma_bbufsz,
824 	    0, (caps & SDHCI_CAN_DO_64BIT) ? BUS_SPACE_MAXADDR :
825 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
826 	    slot->sdma_bbufsz, 1, slot->sdma_bbufsz, BUS_DMA_ALLOCNOW,
827 	    NULL, NULL, &slot->dmatag);
828 	if (err != 0) {
829 		slot_printf(slot, "Can't create DMA tag for SDMA\n");
830 		return (err);
831 	}
832 	/* Allocate DMA memory for the SDMA bounce buffer. */
833 	err = bus_dmamem_alloc(slot->dmatag, (void **)&slot->dmamem,
834 	    BUS_DMA_NOWAIT, &slot->dmamap);
835 	if (err != 0) {
836 		slot_printf(slot, "Can't alloc DMA memory for SDMA\n");
837 		bus_dma_tag_destroy(slot->dmatag);
838 		return (err);
839 	}
840 	/* Map the memory of the SDMA bounce buffer. */
841 	err = bus_dmamap_load(slot->dmatag, slot->dmamap,
842 	    (void *)slot->dmamem, slot->sdma_bbufsz, sdhci_getaddr,
843 	    &slot->paddr, 0);
844 	if (err != 0 || slot->paddr == 0) {
845 		slot_printf(slot, "Can't load DMA memory for SDMA\n");
846 		bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
847 		bus_dma_tag_destroy(slot->dmatag);
848 		if (err)
849 			return (err);
850 		else
851 			return (EFAULT);
852 	}
853 
854 	return (0);
855 }
856 
857 static void
858 sdhci_dma_free(struct sdhci_slot *slot)
859 {
860 
861 	bus_dmamap_unload(slot->dmatag, slot->dmamap);
862 	bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
863 	bus_dma_tag_destroy(slot->dmatag);
864 }
865 
866 int
867 sdhci_init_slot(device_t dev, struct sdhci_slot *slot, int num)
868 {
869 	kobjop_desc_t kobj_desc;
870 	kobj_method_t *kobj_method;
871 	uint32_t caps, caps2, freq, host_caps;
872 	int err;
873 	char node_name[8];
874 	struct sysctl_oid *node_oid;
875 
876 	SDHCI_LOCK_INIT(slot);
877 
878 	slot->num = num;
879 	slot->bus = dev;
880 
881 	slot->version = (RD2(slot, SDHCI_HOST_VERSION)
882 		>> SDHCI_SPEC_VER_SHIFT) & SDHCI_SPEC_VER_MASK;
883 	if (slot->quirks & SDHCI_QUIRK_MISSING_CAPS) {
884 		caps = slot->caps;
885 		caps2 = slot->caps2;
886 	} else {
887 		caps = RD4(slot, SDHCI_CAPABILITIES);
888 		if (slot->version >= SDHCI_SPEC_300)
889 			caps2 = RD4(slot, SDHCI_CAPABILITIES2);
890 		else
891 			caps2 = 0;
892 	}
893 	if (slot->version >= SDHCI_SPEC_300) {
894 		if ((caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_REMOVABLE &&
895 		    (caps & SDHCI_SLOTTYPE_MASK) != SDHCI_SLOTTYPE_EMBEDDED) {
896 			slot_printf(slot,
897 			    "Driver doesn't support shared bus slots\n");
898 			SDHCI_LOCK_DESTROY(slot);
899 			return (ENXIO);
900 		} else if ((caps & SDHCI_SLOTTYPE_MASK) ==
901 		    SDHCI_SLOTTYPE_EMBEDDED) {
902 			slot->opt |= SDHCI_SLOT_EMBEDDED | SDHCI_NON_REMOVABLE;
903 		}
904 	}
905 	/* Calculate base clock frequency. */
906 	if (slot->version >= SDHCI_SPEC_300)
907 		freq = (caps & SDHCI_CLOCK_V3_BASE_MASK) >>
908 		    SDHCI_CLOCK_BASE_SHIFT;
909 	else
910 		freq = (caps & SDHCI_CLOCK_BASE_MASK) >>
911 		    SDHCI_CLOCK_BASE_SHIFT;
912 	if (freq != 0)
913 		slot->max_clk = freq * 1000000;
914 	/*
915 	 * If the frequency wasn't in the capabilities and the hardware driver
916 	 * hasn't already set max_clk we're probably not going to work right
917 	 * with an assumption, so complain about it.
918 	 */
919 	if (slot->max_clk == 0) {
920 		slot->max_clk = SDHCI_DEFAULT_MAX_FREQ * 1000000;
921 		slot_printf(slot, "Hardware doesn't specify base clock "
922 		    "frequency, using %dMHz as default.\n",
923 		    SDHCI_DEFAULT_MAX_FREQ);
924 	}
925 	/* Calculate/set timeout clock frequency. */
926 	if (slot->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK) {
927 		slot->timeout_clk = slot->max_clk / 1000;
928 	} else if (slot->quirks & SDHCI_QUIRK_DATA_TIMEOUT_1MHZ) {
929 		slot->timeout_clk = 1000;
930 	} else {
931 		slot->timeout_clk = (caps & SDHCI_TIMEOUT_CLK_MASK) >>
932 		    SDHCI_TIMEOUT_CLK_SHIFT;
933 		if (caps & SDHCI_TIMEOUT_CLK_UNIT)
934 			slot->timeout_clk *= 1000;
935 	}
936 	/*
937 	 * If the frequency wasn't in the capabilities and the hardware driver
938 	 * hasn't already set timeout_clk we'll probably work okay using the
939 	 * max timeout, but still mention it.
940 	 */
941 	if (slot->timeout_clk == 0) {
942 		slot_printf(slot, "Hardware doesn't specify timeout clock "
943 		    "frequency, setting BROKEN_TIMEOUT quirk.\n");
944 		slot->quirks |= SDHCI_QUIRK_BROKEN_TIMEOUT_VAL;
945 	}
946 
947 	slot->host.f_min = SDHCI_MIN_FREQ(slot->bus, slot);
948 	slot->host.f_max = slot->max_clk;
949 	slot->host.host_ocr = 0;
950 	if (caps & SDHCI_CAN_VDD_330)
951 	    slot->host.host_ocr |= MMC_OCR_320_330 | MMC_OCR_330_340;
952 	if (caps & SDHCI_CAN_VDD_300)
953 	    slot->host.host_ocr |= MMC_OCR_290_300 | MMC_OCR_300_310;
954 	/*
955 	 * 1.8V VDD is not supposed to be used for removable cards.  Hardware
956 	 * prior to v3.0 had no way to indicate embedded slots, but did
957 	 * sometimes support 1.8v for non-removable devices.
958 	 */
959 	if ((caps & SDHCI_CAN_VDD_180) && (slot->version < SDHCI_SPEC_300 ||
960 	    (slot->opt & SDHCI_SLOT_EMBEDDED)))
961 	    slot->host.host_ocr |= MMC_OCR_LOW_VOLTAGE;
962 	if (slot->host.host_ocr == 0) {
963 		slot_printf(slot, "Hardware doesn't report any "
964 		    "support voltages.\n");
965 	}
966 
967 	host_caps = slot->host.caps;
968 	host_caps |= MMC_CAP_4_BIT_DATA;
969 	if (caps & SDHCI_CAN_DO_8BITBUS)
970 		host_caps |= MMC_CAP_8_BIT_DATA;
971 	if (caps & SDHCI_CAN_DO_HISPD)
972 		host_caps |= MMC_CAP_HSPEED;
973 	if (slot->quirks & SDHCI_QUIRK_BOOT_NOACC)
974 		host_caps |= MMC_CAP_BOOT_NOACC;
975 	if (slot->quirks & SDHCI_QUIRK_WAIT_WHILE_BUSY)
976 		host_caps |= MMC_CAP_WAIT_WHILE_BUSY;
977 
978 	/* Determine supported UHS-I and eMMC modes. */
979 	if (caps2 & (SDHCI_CAN_SDR50 | SDHCI_CAN_SDR104 | SDHCI_CAN_DDR50))
980 		host_caps |= MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25;
981 	if (caps2 & SDHCI_CAN_SDR104) {
982 		host_caps |= MMC_CAP_UHS_SDR104 | MMC_CAP_UHS_SDR50;
983 		if (!(slot->quirks & SDHCI_QUIRK_BROKEN_MMC_HS200))
984 			host_caps |= MMC_CAP_MMC_HS200;
985 	} else if (caps2 & SDHCI_CAN_SDR50)
986 		host_caps |= MMC_CAP_UHS_SDR50;
987 	if (caps2 & SDHCI_CAN_DDR50 &&
988 	    !(slot->quirks & SDHCI_QUIRK_BROKEN_UHS_DDR50))
989 		host_caps |= MMC_CAP_UHS_DDR50;
990 	if (slot->quirks & SDHCI_QUIRK_MMC_DDR52)
991 		host_caps |= MMC_CAP_MMC_DDR52;
992 	if (slot->quirks & SDHCI_QUIRK_CAPS_BIT63_FOR_MMC_HS400 &&
993 	    caps2 & SDHCI_CAN_MMC_HS400)
994 		host_caps |= MMC_CAP_MMC_HS400;
995 	if (slot->quirks & SDHCI_QUIRK_MMC_HS400_IF_CAN_SDR104 &&
996 	    caps2 & SDHCI_CAN_SDR104)
997 		host_caps |= MMC_CAP_MMC_HS400;
998 
999 	/*
1000 	 * Disable UHS-I and eMMC modes if the set_uhs_timing method is the
1001 	 * default NULL implementation.
1002 	 */
1003 	kobj_desc = &sdhci_set_uhs_timing_desc;
1004 	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
1005 	    kobj_desc);
1006 	if (kobj_method == &kobj_desc->deflt)
1007 		host_caps &= ~(MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
1008 		    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
1009 		    MMC_CAP_MMC_DDR52 | MMC_CAP_MMC_HS200 | MMC_CAP_MMC_HS400);
1010 
1011 #define	SDHCI_CAP_MODES_TUNING(caps2)					\
1012     (((caps2) & SDHCI_TUNE_SDR50 ? MMC_CAP_UHS_SDR50 : 0) |		\
1013     MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 | MMC_CAP_MMC_HS200 |	\
1014     MMC_CAP_MMC_HS400)
1015 
1016 	/*
1017 	 * Disable UHS-I and eMMC modes that require (re-)tuning if either
1018 	 * the tune or re-tune method is the default NULL implementation.
1019 	 */
1020 	kobj_desc = &mmcbr_tune_desc;
1021 	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
1022 	    kobj_desc);
1023 	if (kobj_method == &kobj_desc->deflt)
1024 		goto no_tuning;
1025 	kobj_desc = &mmcbr_retune_desc;
1026 	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
1027 	    kobj_desc);
1028 	if (kobj_method == &kobj_desc->deflt) {
1029 no_tuning:
1030 		host_caps &= ~(SDHCI_CAP_MODES_TUNING(caps2));
1031 	}
1032 
1033 	/* Allocate tuning structures and determine tuning parameters. */
1034 	if (host_caps & SDHCI_CAP_MODES_TUNING(caps2)) {
1035 		slot->opt |= SDHCI_TUNING_SUPPORTED;
1036 		slot->tune_req = malloc(sizeof(*slot->tune_req), M_DEVBUF,
1037 		    M_WAITOK);
1038 		slot->tune_cmd = malloc(sizeof(*slot->tune_cmd), M_DEVBUF,
1039 		    M_WAITOK);
1040 		slot->tune_data = malloc(sizeof(*slot->tune_data), M_DEVBUF,
1041 		    M_WAITOK);
1042 		if (caps2 & SDHCI_TUNE_SDR50)
1043 			slot->opt |= SDHCI_SDR50_NEEDS_TUNING;
1044 		slot->retune_mode = (caps2 & SDHCI_RETUNE_MODES_MASK) >>
1045 		    SDHCI_RETUNE_MODES_SHIFT;
1046 		if (slot->retune_mode == SDHCI_RETUNE_MODE_1) {
1047 			slot->retune_count = (caps2 & SDHCI_RETUNE_CNT_MASK) >>
1048 			    SDHCI_RETUNE_CNT_SHIFT;
1049 			if (slot->retune_count > 0xb) {
1050 				slot_printf(slot, "Unknown re-tuning count "
1051 				    "%x, using 1 sec\n", slot->retune_count);
1052 				slot->retune_count = 1;
1053 			} else if (slot->retune_count != 0)
1054 				slot->retune_count =
1055 				    1 << (slot->retune_count - 1);
1056 		}
1057 	}
1058 
1059 #undef SDHCI_CAP_MODES_TUNING
1060 
1061 	/* Determine supported VCCQ signaling levels. */
1062 	host_caps |= MMC_CAP_SIGNALING_330;
1063 	if (host_caps & (MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25 |
1064 	    MMC_CAP_UHS_SDR50 | MMC_CAP_UHS_DDR50 | MMC_CAP_UHS_SDR104 |
1065 	    MMC_CAP_MMC_DDR52_180 | MMC_CAP_MMC_HS200_180 |
1066 	    MMC_CAP_MMC_HS400_180))
1067 		host_caps |= MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180;
1068 
1069 	/*
1070 	 * Disable 1.2 V and 1.8 V signaling if the switch_vccq method is the
1071 	 * default NULL implementation.  Disable 1.2 V support if it's the
1072 	 * generic SDHCI implementation.
1073 	 */
1074 	kobj_desc = &mmcbr_switch_vccq_desc;
1075 	kobj_method = kobj_lookup_method(((kobj_t)dev)->ops->cls, NULL,
1076 	    kobj_desc);
1077 	if (kobj_method == &kobj_desc->deflt)
1078 		host_caps &= ~(MMC_CAP_SIGNALING_120 | MMC_CAP_SIGNALING_180);
1079 	else if (kobj_method->func == (kobjop_t)sdhci_generic_switch_vccq)
1080 		host_caps &= ~MMC_CAP_SIGNALING_120;
1081 
1082 	/* Determine supported driver types (type B is always mandatory). */
1083 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_A)
1084 		host_caps |= MMC_CAP_DRIVER_TYPE_A;
1085 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_C)
1086 		host_caps |= MMC_CAP_DRIVER_TYPE_C;
1087 	if (caps2 & SDHCI_CAN_DRIVE_TYPE_D)
1088 		host_caps |= MMC_CAP_DRIVER_TYPE_D;
1089 	slot->host.caps = host_caps;
1090 
1091 	/* Decide if we have usable DMA. */
1092 	if (caps & SDHCI_CAN_DO_DMA)
1093 		slot->opt |= SDHCI_HAVE_DMA;
1094 
1095 	if (slot->quirks & SDHCI_QUIRK_BROKEN_DMA)
1096 		slot->opt &= ~SDHCI_HAVE_DMA;
1097 	if (slot->quirks & SDHCI_QUIRK_FORCE_DMA)
1098 		slot->opt |= SDHCI_HAVE_DMA;
1099 	if (slot->quirks & SDHCI_QUIRK_ALL_SLOTS_NON_REMOVABLE)
1100 		slot->opt |= SDHCI_NON_REMOVABLE;
1101 
1102 	/*
1103 	 * Use platform-provided transfer backend
1104 	 * with PIO as a fallback mechanism
1105 	 */
1106 	if (slot->opt & SDHCI_PLATFORM_TRANSFER)
1107 		slot->opt &= ~SDHCI_HAVE_DMA;
1108 
1109 	if (slot->opt & SDHCI_HAVE_DMA) {
1110 		err = sdhci_dma_alloc(slot, caps);
1111 		if (err != 0) {
1112 			if (slot->opt & SDHCI_TUNING_SUPPORTED) {
1113 				free(slot->tune_req, M_DEVBUF);
1114 				free(slot->tune_cmd, M_DEVBUF);
1115 				free(slot->tune_data, M_DEVBUF);
1116 			}
1117 			SDHCI_LOCK_DESTROY(slot);
1118 			return (err);
1119 		}
1120 	}
1121 
1122 	if (bootverbose || sdhci_debug) {
1123 		sdhci_dumpcaps(slot);
1124 		sdhci_dumpregs(slot);
1125 	}
1126 
1127 	slot->timeout = 10;
1128 	SYSCTL_ADD_INT(device_get_sysctl_ctx(slot->bus),
1129 	    SYSCTL_CHILDREN(device_get_sysctl_tree(slot->bus)), OID_AUTO,
1130 	    "timeout", CTLFLAG_RWTUN, &slot->timeout, 0,
1131 	    "Maximum timeout for SDHCI transfers (in secs)");
1132 	TASK_INIT(&slot->card_task, 0, sdhci_card_task, slot);
1133 	TIMEOUT_TASK_INIT(taskqueue_swi_giant, &slot->card_delayed_task, 0,
1134 		sdhci_card_task, slot);
1135 	callout_init(&slot->card_poll_callout, 1);
1136 	callout_init_mtx(&slot->timeout_callout, &slot->mtx, 0);
1137 	callout_init_mtx(&slot->retune_callout, &slot->mtx, 0);
1138 
1139 	if ((slot->quirks & SDHCI_QUIRK_POLL_CARD_PRESENT) &&
1140 	    !(slot->opt & SDHCI_NON_REMOVABLE)) {
1141 		callout_reset(&slot->card_poll_callout,
1142 		    SDHCI_CARD_PRESENT_TICKS, sdhci_card_poll, slot);
1143 	}
1144 
1145 	sdhci_init(slot);
1146 
1147 	snprintf(node_name, sizeof(node_name), "slot%d", slot->num);
1148 
1149 	node_oid = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev),
1150 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1151 	    OID_AUTO, node_name, CTLFLAG_RW, 0, "slot specific node");
1152 
1153 	node_oid = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev),
1154 	    SYSCTL_CHILDREN(node_oid), OID_AUTO, "debug", CTLFLAG_RW, 0,
1155 	    "Debugging node");
1156 
1157 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(node_oid),
1158 	    OID_AUTO, "dumpregs", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1159 	    slot, 0, &sdhci_syctl_dumpregs,
1160 	    "A", "Dump SDHCI registers");
1161 
1162 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(node_oid),
1163 	    OID_AUTO, "dumpcaps", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1164 	    slot, 0, &sdhci_syctl_dumpcaps,
1165 	    "A", "Dump SDHCI capabilites");
1166 
1167 	return (0);
1168 }
1169 
1170 #ifndef MMCCAM
1171 void
1172 sdhci_start_slot(struct sdhci_slot *slot)
1173 {
1174 
1175 	sdhci_card_task(slot, 0);
1176 }
1177 #endif
1178 
1179 int
1180 sdhci_cleanup_slot(struct sdhci_slot *slot)
1181 {
1182 	device_t d;
1183 
1184 	callout_drain(&slot->timeout_callout);
1185 	callout_drain(&slot->card_poll_callout);
1186 	callout_drain(&slot->retune_callout);
1187 	taskqueue_drain(taskqueue_swi_giant, &slot->card_task);
1188 	taskqueue_drain_timeout(taskqueue_swi_giant, &slot->card_delayed_task);
1189 
1190 	SDHCI_LOCK(slot);
1191 	d = slot->dev;
1192 	slot->dev = NULL;
1193 	SDHCI_UNLOCK(slot);
1194 	if (d != NULL)
1195 		device_delete_child(slot->bus, d);
1196 
1197 	SDHCI_LOCK(slot);
1198 	SDHCI_RESET(slot->bus, slot, SDHCI_RESET_ALL);
1199 	SDHCI_UNLOCK(slot);
1200 	if (slot->opt & SDHCI_HAVE_DMA)
1201 		sdhci_dma_free(slot);
1202 	if (slot->opt & SDHCI_TUNING_SUPPORTED) {
1203 		free(slot->tune_req, M_DEVBUF);
1204 		free(slot->tune_cmd, M_DEVBUF);
1205 		free(slot->tune_data, M_DEVBUF);
1206 	}
1207 
1208 	SDHCI_LOCK_DESTROY(slot);
1209 
1210 	return (0);
1211 }
1212 
1213 int
1214 sdhci_generic_suspend(struct sdhci_slot *slot)
1215 {
1216 
1217 	/*
1218 	 * We expect the MMC layer to issue initial tuning after resume.
1219 	 * Otherwise, we'd need to indicate re-tuning including circuit reset
1220 	 * being required at least for re-tuning modes 1 and 2 ourselves.
1221 	 */
1222 	callout_drain(&slot->retune_callout);
1223 	SDHCI_LOCK(slot);
1224 	slot->opt &= ~SDHCI_TUNING_ENABLED;
1225 	SDHCI_RESET(slot->bus, slot, SDHCI_RESET_ALL);
1226 	SDHCI_UNLOCK(slot);
1227 
1228 	return (0);
1229 }
1230 
1231 int
1232 sdhci_generic_resume(struct sdhci_slot *slot)
1233 {
1234 
1235 	SDHCI_LOCK(slot);
1236 	sdhci_init(slot);
1237 	SDHCI_UNLOCK(slot);
1238 
1239 	return (0);
1240 }
1241 
1242 void
1243 sdhci_generic_reset(device_t brdev __unused, struct sdhci_slot *slot,
1244     uint8_t mask)
1245 {
1246 	int timeout;
1247 	uint32_t clock;
1248 
1249 	if (slot->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
1250 		if (!SDHCI_GET_CARD_PRESENT(slot->bus, slot))
1251 			return;
1252 	}
1253 
1254 	/* Some controllers need this kick or reset won't work. */
1255 	if ((mask & SDHCI_RESET_ALL) == 0 &&
1256 	    (slot->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)) {
1257 		/* This is to force an update */
1258 		clock = slot->clock;
1259 		slot->clock = 0;
1260 		sdhci_set_clock(slot, clock);
1261 	}
1262 
1263 	if (mask & SDHCI_RESET_ALL) {
1264 		slot->clock = 0;
1265 		slot->power = 0;
1266 	}
1267 
1268 	WR1(slot, SDHCI_SOFTWARE_RESET, mask);
1269 
1270 	if (slot->quirks & SDHCI_QUIRK_WAITFOR_RESET_ASSERTED) {
1271 		/*
1272 		 * Resets on TI OMAPs and AM335x are incompatible with SDHCI
1273 		 * specification.  The reset bit has internal propagation delay,
1274 		 * so a fast read after write returns 0 even if reset process is
1275 		 * in progress.  The workaround is to poll for 1 before polling
1276 		 * for 0.  In the worst case, if we miss seeing it asserted the
1277 		 * time we spent waiting is enough to ensure the reset finishes.
1278 		 */
1279 		timeout = 10000;
1280 		while ((RD1(slot, SDHCI_SOFTWARE_RESET) & mask) != mask) {
1281 			if (timeout <= 0)
1282 				break;
1283 			timeout--;
1284 			DELAY(1);
1285 		}
1286 	}
1287 
1288 	/* Wait max 100 ms */
1289 	timeout = 10000;
1290 	/* Controller clears the bits when it's done */
1291 	while (RD1(slot, SDHCI_SOFTWARE_RESET) & mask) {
1292 		if (timeout <= 0) {
1293 			slot_printf(slot, "Reset 0x%x never completed.\n",
1294 			    mask);
1295 			sdhci_dumpregs(slot);
1296 			return;
1297 		}
1298 		timeout--;
1299 		DELAY(10);
1300 	}
1301 }
1302 
1303 uint32_t
1304 sdhci_generic_min_freq(device_t brdev __unused, struct sdhci_slot *slot)
1305 {
1306 
1307 	if (slot->version >= SDHCI_SPEC_300)
1308 		return (slot->max_clk / SDHCI_300_MAX_DIVIDER);
1309 	else
1310 		return (slot->max_clk / SDHCI_200_MAX_DIVIDER);
1311 }
1312 
1313 bool
1314 sdhci_generic_get_card_present(device_t brdev __unused, struct sdhci_slot *slot)
1315 {
1316 
1317 	if (slot->opt & SDHCI_NON_REMOVABLE)
1318 		return true;
1319 
1320 	return (RD4(slot, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT);
1321 }
1322 
1323 void
1324 sdhci_generic_set_uhs_timing(device_t brdev __unused, struct sdhci_slot *slot)
1325 {
1326 	const struct mmc_ios *ios;
1327 	uint16_t hostctrl2;
1328 
1329 	if (slot->version < SDHCI_SPEC_300)
1330 		return;
1331 
1332 	SDHCI_ASSERT_LOCKED(slot);
1333 	ios = &slot->host.ios;
1334 	sdhci_set_clock(slot, 0);
1335 	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1336 	hostctrl2 &= ~SDHCI_CTRL2_UHS_MASK;
1337 	if (ios->clock > SD_SDR50_MAX) {
1338 		if (ios->timing == bus_timing_mmc_hs400 ||
1339 		    ios->timing == bus_timing_mmc_hs400es)
1340 			hostctrl2 |= SDHCI_CTRL2_MMC_HS400;
1341 		else
1342 			hostctrl2 |= SDHCI_CTRL2_UHS_SDR104;
1343 	}
1344 	else if (ios->clock > SD_SDR25_MAX)
1345 		hostctrl2 |= SDHCI_CTRL2_UHS_SDR50;
1346 	else if (ios->clock > SD_SDR12_MAX) {
1347 		if (ios->timing == bus_timing_uhs_ddr50 ||
1348 		    ios->timing == bus_timing_mmc_ddr52)
1349 			hostctrl2 |= SDHCI_CTRL2_UHS_DDR50;
1350 		else
1351 			hostctrl2 |= SDHCI_CTRL2_UHS_SDR25;
1352 	} else if (ios->clock > SD_MMC_CARD_ID_FREQUENCY)
1353 		hostctrl2 |= SDHCI_CTRL2_UHS_SDR12;
1354 	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2);
1355 	sdhci_set_clock(slot, ios->clock);
1356 }
1357 
1358 int
1359 sdhci_generic_update_ios(device_t brdev, device_t reqdev)
1360 {
1361 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1362 	struct mmc_ios *ios = &slot->host.ios;
1363 
1364 	SDHCI_LOCK(slot);
1365 	/* Do full reset on bus power down to clear from any state. */
1366 	if (ios->power_mode == power_off) {
1367 		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
1368 		sdhci_init(slot);
1369 	}
1370 	/* Configure the bus. */
1371 	sdhci_set_clock(slot, ios->clock);
1372 	sdhci_set_power(slot, (ios->power_mode == power_off) ? 0 : ios->vdd);
1373 	if (ios->bus_width == bus_width_8) {
1374 		slot->hostctrl |= SDHCI_CTRL_8BITBUS;
1375 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
1376 	} else if (ios->bus_width == bus_width_4) {
1377 		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
1378 		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
1379 	} else if (ios->bus_width == bus_width_1) {
1380 		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
1381 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
1382 	} else {
1383 		panic("Invalid bus width: %d", ios->bus_width);
1384 	}
1385 	if (ios->clock > SD_SDR12_MAX &&
1386 	    !(slot->quirks & SDHCI_QUIRK_DONT_SET_HISPD_BIT))
1387 		slot->hostctrl |= SDHCI_CTRL_HISPD;
1388 	else
1389 		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
1390 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
1391 	SDHCI_SET_UHS_TIMING(brdev, slot);
1392 	/* Some controllers like reset after bus changes. */
1393 	if (slot->quirks & SDHCI_QUIRK_RESET_ON_IOS)
1394 		SDHCI_RESET(slot->bus, slot,
1395 		    SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1396 
1397 	SDHCI_UNLOCK(slot);
1398 	return (0);
1399 }
1400 
1401 int
1402 sdhci_generic_switch_vccq(device_t brdev __unused, device_t reqdev)
1403 {
1404 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1405 	enum mmc_vccq vccq;
1406 	int err;
1407 	uint16_t hostctrl2;
1408 
1409 	if (slot->version < SDHCI_SPEC_300)
1410 		return (0);
1411 
1412 	err = 0;
1413 	vccq = slot->host.ios.vccq;
1414 	SDHCI_LOCK(slot);
1415 	sdhci_set_clock(slot, 0);
1416 	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1417 	switch (vccq) {
1418 	case vccq_330:
1419 		if (!(hostctrl2 & SDHCI_CTRL2_S18_ENABLE))
1420 			goto done;
1421 		hostctrl2 &= ~SDHCI_CTRL2_S18_ENABLE;
1422 		WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2);
1423 		DELAY(5000);
1424 		hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1425 		if (!(hostctrl2 & SDHCI_CTRL2_S18_ENABLE))
1426 			goto done;
1427 		err = EAGAIN;
1428 		break;
1429 	case vccq_180:
1430 		if (!(slot->host.caps & MMC_CAP_SIGNALING_180)) {
1431 			err = EINVAL;
1432 			goto done;
1433 		}
1434 		if (hostctrl2 & SDHCI_CTRL2_S18_ENABLE)
1435 			goto done;
1436 		hostctrl2 |= SDHCI_CTRL2_S18_ENABLE;
1437 		WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2);
1438 		DELAY(5000);
1439 		hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1440 		if (hostctrl2 & SDHCI_CTRL2_S18_ENABLE)
1441 			goto done;
1442 		err = EAGAIN;
1443 		break;
1444 	default:
1445 		slot_printf(slot,
1446 		    "Attempt to set unsupported signaling voltage\n");
1447 		err = EINVAL;
1448 		break;
1449 	}
1450 done:
1451 	sdhci_set_clock(slot, slot->host.ios.clock);
1452 	SDHCI_UNLOCK(slot);
1453 	return (err);
1454 }
1455 
1456 int
1457 sdhci_generic_tune(device_t brdev __unused, device_t reqdev, bool hs400)
1458 {
1459 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1460 	const struct mmc_ios *ios = &slot->host.ios;
1461 	struct mmc_command *tune_cmd;
1462 	struct mmc_data *tune_data;
1463 	uint32_t opcode;
1464 	int err;
1465 
1466 	if (!(slot->opt & SDHCI_TUNING_SUPPORTED))
1467 		return (0);
1468 
1469 	slot->retune_ticks = slot->retune_count * hz;
1470 	opcode = MMC_SEND_TUNING_BLOCK;
1471 	SDHCI_LOCK(slot);
1472 	switch (ios->timing) {
1473 	case bus_timing_mmc_hs400:
1474 		slot_printf(slot, "HS400 must be tuned in HS200 mode\n");
1475 		SDHCI_UNLOCK(slot);
1476 		return (EINVAL);
1477 	case bus_timing_mmc_hs200:
1478 		/*
1479 		 * In HS400 mode, controllers use the data strobe line to
1480 		 * latch data from the devices so periodic re-tuning isn't
1481 		 * expected to be required.
1482 		 */
1483 		if (hs400)
1484 			slot->retune_ticks = 0;
1485 		opcode = MMC_SEND_TUNING_BLOCK_HS200;
1486 		break;
1487 	case bus_timing_uhs_ddr50:
1488 	case bus_timing_uhs_sdr104:
1489 		break;
1490 	case bus_timing_uhs_sdr50:
1491 		if (slot->opt & SDHCI_SDR50_NEEDS_TUNING)
1492 			break;
1493 		/* FALLTHROUGH */
1494 	default:
1495 		SDHCI_UNLOCK(slot);
1496 		return (0);
1497 	}
1498 
1499 	tune_cmd = slot->tune_cmd;
1500 	memset(tune_cmd, 0, sizeof(*tune_cmd));
1501 	tune_cmd->opcode = opcode;
1502 	tune_cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1503 	tune_data = tune_cmd->data = slot->tune_data;
1504 	memset(tune_data, 0, sizeof(*tune_data));
1505 	tune_data->len = (opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
1506 	    ios->bus_width == bus_width_8) ? MMC_TUNING_LEN_HS200 :
1507 	    MMC_TUNING_LEN;
1508 	tune_data->flags = MMC_DATA_READ;
1509 	tune_data->mrq = tune_cmd->mrq = slot->tune_req;
1510 
1511 	slot->opt &= ~SDHCI_TUNING_ENABLED;
1512 	err = sdhci_exec_tuning(slot, true);
1513 	if (err == 0) {
1514 		slot->opt |= SDHCI_TUNING_ENABLED;
1515 		slot->intmask |= sdhci_tuning_intmask(slot);
1516 		WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
1517 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1518 		if (slot->retune_ticks) {
1519 			callout_reset(&slot->retune_callout, slot->retune_ticks,
1520 			    sdhci_retune, slot);
1521 		}
1522 	}
1523 	SDHCI_UNLOCK(slot);
1524 	return (err);
1525 }
1526 
1527 int
1528 sdhci_generic_retune(device_t brdev __unused, device_t reqdev, bool reset)
1529 {
1530 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1531 	int err;
1532 
1533 	if (!(slot->opt & SDHCI_TUNING_ENABLED))
1534 		return (0);
1535 
1536 	/* HS400 must be tuned in HS200 mode. */
1537 	if (slot->host.ios.timing == bus_timing_mmc_hs400)
1538 		return (EINVAL);
1539 
1540 	SDHCI_LOCK(slot);
1541 	err = sdhci_exec_tuning(slot, reset);
1542 	/*
1543 	 * There are two ways sdhci_exec_tuning() can fail:
1544 	 * EBUSY should not actually happen when requests are only issued
1545 	 *	 with the host properly acquired, and
1546 	 * EIO   re-tuning failed (but it did work initially).
1547 	 *
1548 	 * In both cases, we should retry at later point if periodic re-tuning
1549 	 * is enabled.  Note that due to slot->retune_req not being cleared in
1550 	 * these failure cases, the MMC layer should trigger another attempt at
1551 	 * re-tuning with the next request anyway, though.
1552 	 */
1553 	if (slot->retune_ticks) {
1554 		callout_reset(&slot->retune_callout, slot->retune_ticks,
1555 		    sdhci_retune, slot);
1556 	}
1557 	SDHCI_UNLOCK(slot);
1558 	return (err);
1559 }
1560 
1561 static int
1562 sdhci_exec_tuning(struct sdhci_slot *slot, bool reset)
1563 {
1564 	struct mmc_request *tune_req;
1565 	struct mmc_command *tune_cmd;
1566 	int i;
1567 	uint32_t intmask;
1568 	uint16_t hostctrl2;
1569 	u_char opt;
1570 
1571 	SDHCI_ASSERT_LOCKED(slot);
1572 	if (slot->req != NULL)
1573 		return (EBUSY);
1574 
1575 	/* Tuning doesn't work with DMA enabled. */
1576 	opt = slot->opt;
1577 	slot->opt = opt & ~SDHCI_HAVE_DMA;
1578 
1579 	/*
1580 	 * Ensure that as documented, SDHCI_INT_DATA_AVAIL is the only
1581 	 * kind of interrupt we receive in response to a tuning request.
1582 	 */
1583 	intmask = slot->intmask;
1584 	slot->intmask = SDHCI_INT_DATA_AVAIL;
1585 	WR4(slot, SDHCI_INT_ENABLE, SDHCI_INT_DATA_AVAIL);
1586 	WR4(slot, SDHCI_SIGNAL_ENABLE, SDHCI_INT_DATA_AVAIL);
1587 
1588 	hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1589 	if (reset)
1590 		hostctrl2 &= ~SDHCI_CTRL2_SAMPLING_CLOCK;
1591 	else
1592 		hostctrl2 |= SDHCI_CTRL2_SAMPLING_CLOCK;
1593 	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 | SDHCI_CTRL2_EXEC_TUNING);
1594 
1595 	tune_req = slot->tune_req;
1596 	tune_cmd = slot->tune_cmd;
1597 	for (i = 0; i < MMC_TUNING_MAX; i++) {
1598 		memset(tune_req, 0, sizeof(*tune_req));
1599 		tune_req->cmd = tune_cmd;
1600 		tune_req->done = sdhci_req_wakeup;
1601 		tune_req->done_data = slot;
1602 		slot->req = tune_req;
1603 		slot->flags = 0;
1604 		sdhci_start(slot);
1605 		while (!(tune_req->flags & MMC_REQ_DONE))
1606 			msleep(tune_req, &slot->mtx, 0, "sdhciet", 0);
1607 		if (!(tune_req->flags & MMC_TUNE_DONE))
1608 			break;
1609 		hostctrl2 = RD2(slot, SDHCI_HOST_CONTROL2);
1610 		if (!(hostctrl2 & SDHCI_CTRL2_EXEC_TUNING))
1611 			break;
1612 		if (tune_cmd->opcode == MMC_SEND_TUNING_BLOCK)
1613 			DELAY(1000);
1614 	}
1615 
1616 	/*
1617 	 * Restore DMA usage and interrupts.
1618 	 * Note that the interrupt aggregation code might have cleared
1619 	 * SDHCI_INT_DMA_END and/or SDHCI_INT_RESPONSE in slot->intmask
1620 	 * and SDHCI_SIGNAL_ENABLE respectively so ensure SDHCI_INT_ENABLE
1621 	 * doesn't lose these.
1622 	 */
1623 	slot->opt = opt;
1624 	slot->intmask = intmask;
1625 	WR4(slot, SDHCI_INT_ENABLE, intmask | SDHCI_INT_DMA_END |
1626 	    SDHCI_INT_RESPONSE);
1627 	WR4(slot, SDHCI_SIGNAL_ENABLE, intmask);
1628 
1629 	if ((hostctrl2 & (SDHCI_CTRL2_EXEC_TUNING |
1630 	    SDHCI_CTRL2_SAMPLING_CLOCK)) == SDHCI_CTRL2_SAMPLING_CLOCK) {
1631 		slot->retune_req = 0;
1632 		return (0);
1633 	}
1634 
1635 	slot_printf(slot, "Tuning failed, using fixed sampling clock\n");
1636 	WR2(slot, SDHCI_HOST_CONTROL2, hostctrl2 & ~(SDHCI_CTRL2_EXEC_TUNING |
1637 	    SDHCI_CTRL2_SAMPLING_CLOCK));
1638 	SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1639 	return (EIO);
1640 }
1641 
1642 static void
1643 sdhci_retune(void *arg)
1644 {
1645 	struct sdhci_slot *slot = arg;
1646 
1647 	slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
1648 }
1649 
1650 #ifdef MMCCAM
1651 static void
1652 sdhci_req_done(struct sdhci_slot *slot)
1653 {
1654 	union ccb *ccb;
1655 
1656 	if (__predict_false(sdhci_debug > 1))
1657 		slot_printf(slot, "%s\n", __func__);
1658 	if (slot->ccb != NULL && slot->curcmd != NULL) {
1659 		callout_stop(&slot->timeout_callout);
1660 		ccb = slot->ccb;
1661 		slot->ccb = NULL;
1662 		slot->curcmd = NULL;
1663 
1664 		/* Tell CAM the request is finished */
1665 		struct ccb_mmcio *mmcio;
1666 		mmcio = &ccb->mmcio;
1667 
1668 		ccb->ccb_h.status =
1669 		    (mmcio->cmd.error == 0 ? CAM_REQ_CMP : CAM_REQ_CMP_ERR);
1670 		xpt_done(ccb);
1671 	}
1672 }
1673 #else
1674 static void
1675 sdhci_req_done(struct sdhci_slot *slot)
1676 {
1677 	struct mmc_request *req;
1678 
1679 	if (slot->req != NULL && slot->curcmd != NULL) {
1680 		callout_stop(&slot->timeout_callout);
1681 		req = slot->req;
1682 		slot->req = NULL;
1683 		slot->curcmd = NULL;
1684 		req->done(req);
1685 	}
1686 }
1687 #endif
1688 
1689 static void
1690 sdhci_req_wakeup(struct mmc_request *req)
1691 {
1692 
1693 	req->flags |= MMC_REQ_DONE;
1694 	wakeup(req);
1695 }
1696 
1697 static void
1698 sdhci_timeout(void *arg)
1699 {
1700 	struct sdhci_slot *slot = arg;
1701 
1702 	if (slot->curcmd != NULL) {
1703 		slot_printf(slot, "Controller timeout\n");
1704 		sdhci_dumpregs(slot);
1705 		SDHCI_RESET(slot->bus, slot,
1706 		    SDHCI_RESET_CMD | SDHCI_RESET_DATA);
1707 		slot->curcmd->error = MMC_ERR_TIMEOUT;
1708 		sdhci_req_done(slot);
1709 	} else {
1710 		slot_printf(slot, "Spurious timeout - no active command\n");
1711 	}
1712 }
1713 
1714 static void
1715 sdhci_set_transfer_mode(struct sdhci_slot *slot, const struct mmc_data *data)
1716 {
1717 	uint16_t mode;
1718 
1719 	if (data == NULL)
1720 		return;
1721 
1722 	mode = SDHCI_TRNS_BLK_CNT_EN;
1723 	if (data->len > 512 || data->block_count > 1) {
1724 		mode |= SDHCI_TRNS_MULTI;
1725 		if (data->block_count == 0 && __predict_true(
1726 #ifdef MMCCAM
1727 		    slot->ccb->mmcio.stop.opcode == MMC_STOP_TRANSMISSION &&
1728 #else
1729 		    slot->req->stop != NULL &&
1730 #endif
1731 		    !(slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP)))
1732 			mode |= SDHCI_TRNS_ACMD12;
1733 	}
1734 	if (data->flags & MMC_DATA_READ)
1735 		mode |= SDHCI_TRNS_READ;
1736 	if (slot->flags & SDHCI_USE_DMA)
1737 		mode |= SDHCI_TRNS_DMA;
1738 
1739 	WR2(slot, SDHCI_TRANSFER_MODE, mode);
1740 }
1741 
1742 static void
1743 sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
1744 {
1745 	int flags, timeout;
1746 	uint32_t mask;
1747 
1748 	slot->curcmd = cmd;
1749 	slot->cmd_done = 0;
1750 
1751 	cmd->error = MMC_ERR_NONE;
1752 
1753 	/* This flags combination is not supported by controller. */
1754 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
1755 		slot_printf(slot, "Unsupported response type!\n");
1756 		cmd->error = MMC_ERR_FAILED;
1757 		sdhci_req_done(slot);
1758 		return;
1759 	}
1760 
1761 	/*
1762 	 * Do not issue command if there is no card, clock or power.
1763 	 * Controller will not detect timeout without clock active.
1764 	 */
1765 	if (!SDHCI_GET_CARD_PRESENT(slot->bus, slot) ||
1766 	    slot->power == 0 ||
1767 	    slot->clock == 0) {
1768 		slot_printf(slot,
1769 			    "Cannot issue a command (power=%d clock=%d)\n",
1770 			    slot->power, slot->clock);
1771 		cmd->error = MMC_ERR_FAILED;
1772 		sdhci_req_done(slot);
1773 		return;
1774 	}
1775 	/* Always wait for free CMD bus. */
1776 	mask = SDHCI_CMD_INHIBIT;
1777 	/* Wait for free DAT if we have data or busy signal. */
1778 	if (cmd->data != NULL || (cmd->flags & MMC_RSP_BUSY))
1779 		mask |= SDHCI_DAT_INHIBIT;
1780 	/*
1781 	 * We shouldn't wait for DAT for stop commands or CMD19/CMD21.  Note
1782 	 * that these latter are also special in that SDHCI_CMD_DATA should
1783 	 * be set below but no actual data is ever read from the controller.
1784 	*/
1785 #ifdef MMCCAM
1786 	if (cmd == &slot->ccb->mmcio.stop ||
1787 #else
1788 	if (cmd == slot->req->stop ||
1789 #endif
1790 	    __predict_false(cmd->opcode == MMC_SEND_TUNING_BLOCK ||
1791 	    cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))
1792 		mask &= ~SDHCI_DAT_INHIBIT;
1793 	/*
1794 	 *  Wait for bus no more then 250 ms.  Typically there will be no wait
1795 	 *  here at all, but when writing a crash dump we may be bypassing the
1796 	 *  host platform's interrupt handler, and in some cases that handler
1797 	 *  may be working around hardware quirks such as not respecting r1b
1798 	 *  busy indications.  In those cases, this wait-loop serves the purpose
1799 	 *  of waiting for the prior command and data transfers to be done, and
1800 	 *  SD cards are allowed to take up to 250ms for write and erase ops.
1801 	 *  (It's usually more like 20-30ms in the real world.)
1802 	 */
1803 	timeout = 250;
1804 	while (mask & RD4(slot, SDHCI_PRESENT_STATE)) {
1805 		if (timeout == 0) {
1806 			slot_printf(slot, "Controller never released "
1807 			    "inhibit bit(s).\n");
1808 			sdhci_dumpregs(slot);
1809 			cmd->error = MMC_ERR_FAILED;
1810 			sdhci_req_done(slot);
1811 			return;
1812 		}
1813 		timeout--;
1814 		DELAY(1000);
1815 	}
1816 
1817 	/* Prepare command flags. */
1818 	if (!(cmd->flags & MMC_RSP_PRESENT))
1819 		flags = SDHCI_CMD_RESP_NONE;
1820 	else if (cmd->flags & MMC_RSP_136)
1821 		flags = SDHCI_CMD_RESP_LONG;
1822 	else if (cmd->flags & MMC_RSP_BUSY)
1823 		flags = SDHCI_CMD_RESP_SHORT_BUSY;
1824 	else
1825 		flags = SDHCI_CMD_RESP_SHORT;
1826 	if (cmd->flags & MMC_RSP_CRC)
1827 		flags |= SDHCI_CMD_CRC;
1828 	if (cmd->flags & MMC_RSP_OPCODE)
1829 		flags |= SDHCI_CMD_INDEX;
1830 	if (cmd->data != NULL)
1831 		flags |= SDHCI_CMD_DATA;
1832 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
1833 		flags |= SDHCI_CMD_TYPE_ABORT;
1834 	/* Prepare data. */
1835 	sdhci_start_data(slot, cmd->data);
1836 	/*
1837 	 * Interrupt aggregation: To reduce total number of interrupts
1838 	 * group response interrupt with data interrupt when possible.
1839 	 * If there going to be data interrupt, mask response one.
1840 	 */
1841 	if (slot->data_done == 0) {
1842 		WR4(slot, SDHCI_SIGNAL_ENABLE,
1843 		    slot->intmask &= ~SDHCI_INT_RESPONSE);
1844 	}
1845 	/* Set command argument. */
1846 	WR4(slot, SDHCI_ARGUMENT, cmd->arg);
1847 	/* Set data transfer mode. */
1848 	sdhci_set_transfer_mode(slot, cmd->data);
1849 	if (__predict_false(sdhci_debug > 1))
1850 		slot_printf(slot, "Starting command opcode %#04x flags %#04x\n",
1851 		    cmd->opcode, flags);
1852 
1853 	/* Start command. */
1854 	WR2(slot, SDHCI_COMMAND_FLAGS, (cmd->opcode << 8) | (flags & 0xff));
1855 	/* Start timeout callout. */
1856 	callout_reset(&slot->timeout_callout, slot->timeout * hz,
1857 	    sdhci_timeout, slot);
1858 }
1859 
1860 static void
1861 sdhci_finish_command(struct sdhci_slot *slot)
1862 {
1863 	int i;
1864 	uint32_t val;
1865 	uint8_t extra;
1866 
1867 	if (__predict_false(sdhci_debug > 1))
1868 		slot_printf(slot, "%s: called, err %d flags %#04x\n",
1869 		    __func__, slot->curcmd->error, slot->curcmd->flags);
1870 	slot->cmd_done = 1;
1871 	/*
1872 	 * Interrupt aggregation: Restore command interrupt.
1873 	 * Main restore point for the case when command interrupt
1874 	 * happened first.
1875 	 */
1876 	if (__predict_true(slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK &&
1877 	    slot->curcmd->opcode != MMC_SEND_TUNING_BLOCK_HS200))
1878 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |=
1879 		    SDHCI_INT_RESPONSE);
1880 	/* In case of error - reset host and return. */
1881 	if (slot->curcmd->error) {
1882 		if (slot->curcmd->error == MMC_ERR_BADCRC)
1883 			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
1884 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD);
1885 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_DATA);
1886 		sdhci_start(slot);
1887 		return;
1888 	}
1889 	/* If command has response - fetch it. */
1890 	if (slot->curcmd->flags & MMC_RSP_PRESENT) {
1891 		if (slot->curcmd->flags & MMC_RSP_136) {
1892 			/* CRC is stripped so we need one byte shift. */
1893 			extra = 0;
1894 			for (i = 0; i < 4; i++) {
1895 				val = RD4(slot, SDHCI_RESPONSE + i * 4);
1896 				if (slot->quirks &
1897 				    SDHCI_QUIRK_DONT_SHIFT_RESPONSE)
1898 					slot->curcmd->resp[3 - i] = val;
1899 				else {
1900 					slot->curcmd->resp[3 - i] =
1901 					    (val << 8) | extra;
1902 					extra = val >> 24;
1903 				}
1904 			}
1905 		} else
1906 			slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE);
1907 	}
1908 	if (__predict_false(sdhci_debug > 1))
1909 		slot_printf(slot, "Resp: %#04x %#04x %#04x %#04x\n",
1910 		    slot->curcmd->resp[0], slot->curcmd->resp[1],
1911 		    slot->curcmd->resp[2], slot->curcmd->resp[3]);
1912 
1913 	/* If data ready - finish. */
1914 	if (slot->data_done)
1915 		sdhci_start(slot);
1916 }
1917 
1918 static void
1919 sdhci_start_data(struct sdhci_slot *slot, const struct mmc_data *data)
1920 {
1921 	uint32_t blkcnt, blksz, current_timeout, sdma_bbufsz, target_timeout;
1922 	uint8_t div;
1923 
1924 	if (data == NULL && (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
1925 		slot->data_done = 1;
1926 		return;
1927 	}
1928 
1929 	slot->data_done = 0;
1930 
1931 	/* Calculate and set data timeout.*/
1932 	/* XXX: We should have this from mmc layer, now assume 1 sec. */
1933 	if (slot->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL) {
1934 		div = 0xE;
1935 	} else {
1936 		target_timeout = 1000000;
1937 		div = 0;
1938 		current_timeout = (1 << 13) * 1000 / slot->timeout_clk;
1939 		while (current_timeout < target_timeout && div < 0xE) {
1940 			++div;
1941 			current_timeout <<= 1;
1942 		}
1943 		/* Compensate for an off-by-one error in the CaFe chip.*/
1944 		if (div < 0xE &&
1945 		    (slot->quirks & SDHCI_QUIRK_INCR_TIMEOUT_CONTROL)) {
1946 			++div;
1947 		}
1948 	}
1949 	WR1(slot, SDHCI_TIMEOUT_CONTROL, div);
1950 
1951 	if (data == NULL)
1952 		return;
1953 
1954 	/* Use DMA if possible. */
1955 	if ((slot->opt & SDHCI_HAVE_DMA))
1956 		slot->flags |= SDHCI_USE_DMA;
1957 	/* If data is small, broken DMA may return zeroes instead of data. */
1958 	if ((slot->quirks & SDHCI_QUIRK_BROKEN_TIMINGS) &&
1959 	    (data->len <= 512))
1960 		slot->flags &= ~SDHCI_USE_DMA;
1961 	/* Some controllers require even block sizes. */
1962 	if ((slot->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE) &&
1963 	    ((data->len) & 0x3))
1964 		slot->flags &= ~SDHCI_USE_DMA;
1965 	/* Load DMA buffer. */
1966 	if (slot->flags & SDHCI_USE_DMA) {
1967 		sdma_bbufsz = slot->sdma_bbufsz;
1968 		if (data->flags & MMC_DATA_READ)
1969 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1970 			    BUS_DMASYNC_PREREAD);
1971 		else {
1972 			memcpy(slot->dmamem, data->data, ulmin(data->len,
1973 			    sdma_bbufsz));
1974 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1975 			    BUS_DMASYNC_PREWRITE);
1976 		}
1977 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
1978 		/*
1979 		 * Interrupt aggregation: Mask border interrupt for the last
1980 		 * bounce buffer and unmask otherwise.
1981 		 */
1982 		if (data->len == sdma_bbufsz)
1983 			slot->intmask &= ~SDHCI_INT_DMA_END;
1984 		else
1985 			slot->intmask |= SDHCI_INT_DMA_END;
1986 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1987 	}
1988 	/* Current data offset for both PIO and DMA. */
1989 	slot->offset = 0;
1990 #ifdef MMCCAM
1991 	if (data->flags & MMC_DATA_BLOCK_SIZE) {
1992 		/* Set block size and request border interrupts on the SDMA boundary. */
1993 		blksz = SDHCI_MAKE_BLKSZ(slot->sdma_boundary, data->block_size);
1994 		blkcnt = data->block_count;
1995 		if (__predict_false(sdhci_debug > 0))
1996 			slot_printf(slot, "SDIO Custom block params: blksz: "
1997 			    "%#10x, blk cnt: %#10x\n", blksz, blkcnt);
1998 	} else
1999 #endif
2000 	{
2001 		/* Set block size and request border interrupts on the SDMA boundary. */
2002 		blksz = SDHCI_MAKE_BLKSZ(slot->sdma_boundary, ulmin(data->len, 512));
2003 		blkcnt = howmany(data->len, 512);
2004 	}
2005 
2006 	WR2(slot, SDHCI_BLOCK_SIZE, blksz);
2007 	WR2(slot, SDHCI_BLOCK_COUNT, blkcnt);
2008 	if (__predict_false(sdhci_debug > 1))
2009 		slot_printf(slot, "Blk size: 0x%08x | Blk cnt:  0x%08x\n",
2010 		    blksz, blkcnt);
2011 }
2012 
2013 void
2014 sdhci_finish_data(struct sdhci_slot *slot)
2015 {
2016 	struct mmc_data *data = slot->curcmd->data;
2017 	size_t left;
2018 
2019 	/* Interrupt aggregation: Restore command interrupt.
2020 	 * Auxiliary restore point for the case when data interrupt
2021 	 * happened first. */
2022 	if (!slot->cmd_done) {
2023 		WR4(slot, SDHCI_SIGNAL_ENABLE,
2024 		    slot->intmask |= SDHCI_INT_RESPONSE);
2025 	}
2026 	/* Unload rest of data from DMA buffer. */
2027 	if (!slot->data_done && (slot->flags & SDHCI_USE_DMA) &&
2028 	    slot->curcmd->data != NULL) {
2029 		if (data->flags & MMC_DATA_READ) {
2030 			left = data->len - slot->offset;
2031 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2032 			    BUS_DMASYNC_POSTREAD);
2033 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
2034 			    ulmin(left, slot->sdma_bbufsz));
2035 		} else
2036 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2037 			    BUS_DMASYNC_POSTWRITE);
2038 	}
2039 	slot->data_done = 1;
2040 	/* If there was error - reset the host. */
2041 	if (slot->curcmd->error) {
2042 		if (slot->curcmd->error == MMC_ERR_BADCRC)
2043 			slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
2044 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD);
2045 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_DATA);
2046 		sdhci_start(slot);
2047 		return;
2048 	}
2049 	/* If we already have command response - finish. */
2050 	if (slot->cmd_done)
2051 		sdhci_start(slot);
2052 }
2053 
2054 #ifdef MMCCAM
2055 static void
2056 sdhci_start(struct sdhci_slot *slot)
2057 {
2058 	union ccb *ccb;
2059 	struct ccb_mmcio *mmcio;
2060 
2061 	ccb = slot->ccb;
2062 	if (ccb == NULL)
2063 		return;
2064 
2065 	mmcio = &ccb->mmcio;
2066 	if (!(slot->flags & CMD_STARTED)) {
2067 		slot->flags |= CMD_STARTED;
2068 		sdhci_start_command(slot, &mmcio->cmd);
2069 		return;
2070 	}
2071 
2072 	/*
2073 	 * Old stack doesn't use this!
2074 	 * Enabling this code causes significant performance degradation
2075 	 * and IRQ storms on BBB, Wandboard behaves fine.
2076 	 * Not using this code does no harm...
2077 	if (!(slot->flags & STOP_STARTED) && mmcio->stop.opcode != 0) {
2078 		slot->flags |= STOP_STARTED;
2079 		sdhci_start_command(slot, &mmcio->stop);
2080 		return;
2081 	}
2082 	*/
2083 	if (__predict_false(sdhci_debug > 1))
2084 		slot_printf(slot, "result: %d\n", mmcio->cmd.error);
2085 	if (mmcio->cmd.error == 0 &&
2086 	    (slot->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)) {
2087 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD);
2088 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_DATA);
2089 	}
2090 
2091 	sdhci_req_done(slot);
2092 }
2093 #else
2094 static void
2095 sdhci_start(struct sdhci_slot *slot)
2096 {
2097 	const struct mmc_request *req;
2098 
2099 	req = slot->req;
2100 	if (req == NULL)
2101 		return;
2102 
2103 	if (!(slot->flags & CMD_STARTED)) {
2104 		slot->flags |= CMD_STARTED;
2105 		sdhci_start_command(slot, req->cmd);
2106 		return;
2107 	}
2108 	if ((slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP) &&
2109 	    !(slot->flags & STOP_STARTED) && req->stop) {
2110 		slot->flags |= STOP_STARTED;
2111 		sdhci_start_command(slot, req->stop);
2112 		return;
2113 	}
2114 	if (__predict_false(sdhci_debug > 1))
2115 		slot_printf(slot, "result: %d\n", req->cmd->error);
2116 	if (!req->cmd->error &&
2117 	    ((slot->curcmd == req->stop &&
2118 	     (slot->quirks & SDHCI_QUIRK_BROKEN_AUTO_STOP)) ||
2119 	     (slot->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST))) {
2120 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD);
2121 		SDHCI_RESET(slot->bus, slot, SDHCI_RESET_DATA);
2122 	}
2123 
2124 	sdhci_req_done(slot);
2125 }
2126 #endif
2127 
2128 int
2129 sdhci_generic_request(device_t brdev __unused, device_t reqdev,
2130     struct mmc_request *req)
2131 {
2132 	struct sdhci_slot *slot = device_get_ivars(reqdev);
2133 
2134 	SDHCI_LOCK(slot);
2135 	if (slot->req != NULL) {
2136 		SDHCI_UNLOCK(slot);
2137 		return (EBUSY);
2138 	}
2139 	if (__predict_false(sdhci_debug > 1)) {
2140 		slot_printf(slot,
2141 		    "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
2142 		    req->cmd->opcode, req->cmd->arg, req->cmd->flags,
2143 		    (req->cmd->data)?(u_int)req->cmd->data->len:0,
2144 		    (req->cmd->data)?req->cmd->data->flags:0);
2145 	}
2146 	slot->req = req;
2147 	slot->flags = 0;
2148 	sdhci_start(slot);
2149 	SDHCI_UNLOCK(slot);
2150 	if (dumping) {
2151 		while (slot->req != NULL) {
2152 			sdhci_generic_intr(slot);
2153 			DELAY(10);
2154 		}
2155 	}
2156 	return (0);
2157 }
2158 
2159 int
2160 sdhci_generic_get_ro(device_t brdev __unused, device_t reqdev)
2161 {
2162 	struct sdhci_slot *slot = device_get_ivars(reqdev);
2163 	uint32_t val;
2164 
2165 	SDHCI_LOCK(slot);
2166 	val = RD4(slot, SDHCI_PRESENT_STATE);
2167 	SDHCI_UNLOCK(slot);
2168 	return (!(val & SDHCI_WRITE_PROTECT));
2169 }
2170 
2171 int
2172 sdhci_generic_acquire_host(device_t brdev __unused, device_t reqdev)
2173 {
2174 	struct sdhci_slot *slot = device_get_ivars(reqdev);
2175 	int err = 0;
2176 
2177 	SDHCI_LOCK(slot);
2178 	while (slot->bus_busy)
2179 		msleep(slot, &slot->mtx, 0, "sdhciah", 0);
2180 	slot->bus_busy++;
2181 	/* Activate led. */
2182 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl |= SDHCI_CTRL_LED);
2183 	SDHCI_UNLOCK(slot);
2184 	return (err);
2185 }
2186 
2187 int
2188 sdhci_generic_release_host(device_t brdev __unused, device_t reqdev)
2189 {
2190 	struct sdhci_slot *slot = device_get_ivars(reqdev);
2191 
2192 	SDHCI_LOCK(slot);
2193 	/* Deactivate led. */
2194 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl &= ~SDHCI_CTRL_LED);
2195 	slot->bus_busy--;
2196 	wakeup(slot);
2197 	SDHCI_UNLOCK(slot);
2198 	return (0);
2199 }
2200 
2201 static void
2202 sdhci_cmd_irq(struct sdhci_slot *slot, uint32_t intmask)
2203 {
2204 
2205 	if (!slot->curcmd) {
2206 		slot_printf(slot, "Got command interrupt 0x%08x, but "
2207 		    "there is no active command.\n", intmask);
2208 		sdhci_dumpregs(slot);
2209 		return;
2210 	}
2211 	if (intmask & SDHCI_INT_TIMEOUT)
2212 		slot->curcmd->error = MMC_ERR_TIMEOUT;
2213 	else if (intmask & SDHCI_INT_CRC)
2214 		slot->curcmd->error = MMC_ERR_BADCRC;
2215 	else if (intmask & (SDHCI_INT_END_BIT | SDHCI_INT_INDEX))
2216 		slot->curcmd->error = MMC_ERR_FIFO;
2217 
2218 	sdhci_finish_command(slot);
2219 }
2220 
2221 static void
2222 sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask)
2223 {
2224 	struct mmc_data *data;
2225 	size_t left;
2226 	uint32_t sdma_bbufsz;
2227 
2228 	if (!slot->curcmd) {
2229 		slot_printf(slot, "Got data interrupt 0x%08x, but "
2230 		    "there is no active command.\n", intmask);
2231 		sdhci_dumpregs(slot);
2232 		return;
2233 	}
2234 	if (slot->curcmd->data == NULL &&
2235 	    (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
2236 		slot_printf(slot, "Got data interrupt 0x%08x, but "
2237 		    "there is no active data operation.\n",
2238 		    intmask);
2239 		sdhci_dumpregs(slot);
2240 		return;
2241 	}
2242 	if (intmask & SDHCI_INT_DATA_TIMEOUT)
2243 		slot->curcmd->error = MMC_ERR_TIMEOUT;
2244 	else if (intmask & (SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_END_BIT))
2245 		slot->curcmd->error = MMC_ERR_BADCRC;
2246 	if (slot->curcmd->data == NULL &&
2247 	    (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
2248 	    SDHCI_INT_DMA_END))) {
2249 		slot_printf(slot, "Got data interrupt 0x%08x, but "
2250 		    "there is busy-only command.\n", intmask);
2251 		sdhci_dumpregs(slot);
2252 		slot->curcmd->error = MMC_ERR_INVALID;
2253 	}
2254 	if (slot->curcmd->error) {
2255 		/* No need to continue after any error. */
2256 		goto done;
2257 	}
2258 
2259 	/* Handle tuning completion interrupt. */
2260 	if (__predict_false((intmask & SDHCI_INT_DATA_AVAIL) &&
2261 	    (slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK ||
2262 	    slot->curcmd->opcode == MMC_SEND_TUNING_BLOCK_HS200))) {
2263 		slot->req->flags |= MMC_TUNE_DONE;
2264 		sdhci_finish_command(slot);
2265 		sdhci_finish_data(slot);
2266 		return;
2267 	}
2268 	/* Handle PIO interrupt. */
2269 	if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL)) {
2270 		if ((slot->opt & SDHCI_PLATFORM_TRANSFER) &&
2271 		    SDHCI_PLATFORM_WILL_HANDLE(slot->bus, slot)) {
2272 			SDHCI_PLATFORM_START_TRANSFER(slot->bus, slot,
2273 			    &intmask);
2274 			slot->flags |= PLATFORM_DATA_STARTED;
2275 		} else
2276 			sdhci_transfer_pio(slot);
2277 	}
2278 	/* Handle DMA border. */
2279 	if (intmask & SDHCI_INT_DMA_END) {
2280 		data = slot->curcmd->data;
2281 		sdma_bbufsz = slot->sdma_bbufsz;
2282 
2283 		/* Unload DMA buffer ... */
2284 		left = data->len - slot->offset;
2285 		if (data->flags & MMC_DATA_READ) {
2286 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2287 			    BUS_DMASYNC_POSTREAD);
2288 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
2289 			    ulmin(left, sdma_bbufsz));
2290 		} else {
2291 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2292 			    BUS_DMASYNC_POSTWRITE);
2293 		}
2294 		/* ... and reload it again. */
2295 		slot->offset += sdma_bbufsz;
2296 		left = data->len - slot->offset;
2297 		if (data->flags & MMC_DATA_READ) {
2298 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2299 			    BUS_DMASYNC_PREREAD);
2300 		} else {
2301 			memcpy(slot->dmamem, (u_char*)data->data + slot->offset,
2302 			    ulmin(left, sdma_bbufsz));
2303 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
2304 			    BUS_DMASYNC_PREWRITE);
2305 		}
2306 		/*
2307 		 * Interrupt aggregation: Mask border interrupt for the last
2308 		 * bounce buffer.
2309 		 */
2310 		if (left == sdma_bbufsz) {
2311 			slot->intmask &= ~SDHCI_INT_DMA_END;
2312 			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
2313 		}
2314 		/* Restart DMA. */
2315 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
2316 	}
2317 	/* We have got all data. */
2318 	if (intmask & SDHCI_INT_DATA_END) {
2319 		if (slot->flags & PLATFORM_DATA_STARTED) {
2320 			slot->flags &= ~PLATFORM_DATA_STARTED;
2321 			SDHCI_PLATFORM_FINISH_TRANSFER(slot->bus, slot);
2322 		} else
2323 			sdhci_finish_data(slot);
2324 	}
2325 done:
2326 	if (slot->curcmd != NULL && slot->curcmd->error != 0) {
2327 		if (slot->flags & PLATFORM_DATA_STARTED) {
2328 			slot->flags &= ~PLATFORM_DATA_STARTED;
2329 			SDHCI_PLATFORM_FINISH_TRANSFER(slot->bus, slot);
2330 		} else
2331 			sdhci_finish_data(slot);
2332 	}
2333 }
2334 
2335 static void
2336 sdhci_acmd_irq(struct sdhci_slot *slot, uint16_t acmd_err)
2337 {
2338 
2339 	if (!slot->curcmd) {
2340 		slot_printf(slot, "Got AutoCMD12 error 0x%04x, but "
2341 		    "there is no active command.\n", acmd_err);
2342 		sdhci_dumpregs(slot);
2343 		return;
2344 	}
2345 	slot_printf(slot, "Got AutoCMD12 error 0x%04x\n", acmd_err);
2346 	SDHCI_RESET(slot->bus, slot, SDHCI_RESET_CMD);
2347 }
2348 
2349 void
2350 sdhci_generic_intr(struct sdhci_slot *slot)
2351 {
2352 	uint32_t intmask, present;
2353 	uint16_t val16;
2354 
2355 	SDHCI_LOCK(slot);
2356 	/* Read slot interrupt status. */
2357 	intmask = RD4(slot, SDHCI_INT_STATUS);
2358 	if (intmask == 0 || intmask == 0xffffffff) {
2359 		SDHCI_UNLOCK(slot);
2360 		return;
2361 	}
2362 	if (__predict_false(sdhci_debug > 2))
2363 		slot_printf(slot, "Interrupt %#x\n", intmask);
2364 
2365 	/* Handle tuning error interrupt. */
2366 	if (__predict_false(intmask & SDHCI_INT_TUNEERR)) {
2367 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_TUNEERR);
2368 		slot_printf(slot, "Tuning error indicated\n");
2369 		slot->retune_req |= SDHCI_RETUNE_REQ_RESET;
2370 		if (slot->curcmd) {
2371 			slot->curcmd->error = MMC_ERR_BADCRC;
2372 			sdhci_finish_command(slot);
2373 		}
2374 	}
2375 	/* Handle re-tuning interrupt. */
2376 	if (__predict_false(intmask & SDHCI_INT_RETUNE))
2377 		slot->retune_req |= SDHCI_RETUNE_REQ_NEEDED;
2378 	/* Handle card presence interrupts. */
2379 	if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
2380 		present = (intmask & SDHCI_INT_CARD_INSERT) != 0;
2381 		slot->intmask &=
2382 		    ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
2383 		slot->intmask |= present ? SDHCI_INT_CARD_REMOVE :
2384 		    SDHCI_INT_CARD_INSERT;
2385 		WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
2386 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
2387 		WR4(slot, SDHCI_INT_STATUS, intmask &
2388 		    (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE));
2389 		sdhci_handle_card_present_locked(slot, present);
2390 	}
2391 	/* Handle command interrupts. */
2392 	if (intmask & SDHCI_INT_CMD_MASK) {
2393 		WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_CMD_MASK);
2394 		sdhci_cmd_irq(slot, intmask & SDHCI_INT_CMD_MASK);
2395 	}
2396 	/* Handle data interrupts. */
2397 	if (intmask & SDHCI_INT_DATA_MASK) {
2398 		WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_DATA_MASK);
2399 		/* Don't call data_irq in case of errored command. */
2400 		if ((intmask & SDHCI_INT_CMD_ERROR_MASK) == 0)
2401 			sdhci_data_irq(slot, intmask & SDHCI_INT_DATA_MASK);
2402 	}
2403 	/* Handle AutoCMD12 error interrupt. */
2404 	if (intmask & SDHCI_INT_ACMD12ERR) {
2405 		/* Clearing SDHCI_INT_ACMD12ERR may clear SDHCI_ACMD12_ERR. */
2406 		val16 = RD2(slot, SDHCI_ACMD12_ERR);
2407 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR);
2408 		sdhci_acmd_irq(slot, val16);
2409 	}
2410 	/* Handle bus power interrupt. */
2411 	if (intmask & SDHCI_INT_BUS_POWER) {
2412 		WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER);
2413 		slot_printf(slot, "Card is consuming too much power!\n");
2414 	}
2415 	intmask &= ~(SDHCI_INT_ERROR | SDHCI_INT_TUNEERR | SDHCI_INT_RETUNE |
2416 	    SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | SDHCI_INT_CMD_MASK |
2417 	    SDHCI_INT_DATA_MASK | SDHCI_INT_ACMD12ERR | SDHCI_INT_BUS_POWER);
2418 	/* The rest is unknown. */
2419 	if (intmask) {
2420 		WR4(slot, SDHCI_INT_STATUS, intmask);
2421 		slot_printf(slot, "Unexpected interrupt 0x%08x.\n",
2422 		    intmask);
2423 		sdhci_dumpregs(slot);
2424 	}
2425 
2426 	SDHCI_UNLOCK(slot);
2427 }
2428 
2429 int
2430 sdhci_generic_read_ivar(device_t bus, device_t child, int which,
2431     uintptr_t *result)
2432 {
2433 	const struct sdhci_slot *slot = device_get_ivars(child);
2434 
2435 	switch (which) {
2436 	default:
2437 		return (EINVAL);
2438 	case MMCBR_IVAR_BUS_MODE:
2439 		*result = slot->host.ios.bus_mode;
2440 		break;
2441 	case MMCBR_IVAR_BUS_WIDTH:
2442 		*result = slot->host.ios.bus_width;
2443 		break;
2444 	case MMCBR_IVAR_CHIP_SELECT:
2445 		*result = slot->host.ios.chip_select;
2446 		break;
2447 	case MMCBR_IVAR_CLOCK:
2448 		*result = slot->host.ios.clock;
2449 		break;
2450 	case MMCBR_IVAR_F_MIN:
2451 		*result = slot->host.f_min;
2452 		break;
2453 	case MMCBR_IVAR_F_MAX:
2454 		*result = slot->host.f_max;
2455 		break;
2456 	case MMCBR_IVAR_HOST_OCR:
2457 		*result = slot->host.host_ocr;
2458 		break;
2459 	case MMCBR_IVAR_MODE:
2460 		*result = slot->host.mode;
2461 		break;
2462 	case MMCBR_IVAR_OCR:
2463 		*result = slot->host.ocr;
2464 		break;
2465 	case MMCBR_IVAR_POWER_MODE:
2466 		*result = slot->host.ios.power_mode;
2467 		break;
2468 	case MMCBR_IVAR_VDD:
2469 		*result = slot->host.ios.vdd;
2470 		break;
2471 	case MMCBR_IVAR_RETUNE_REQ:
2472 		if (slot->opt & SDHCI_TUNING_ENABLED) {
2473 			if (slot->retune_req & SDHCI_RETUNE_REQ_RESET) {
2474 				*result = retune_req_reset;
2475 				break;
2476 			}
2477 			if (slot->retune_req & SDHCI_RETUNE_REQ_NEEDED) {
2478 				*result = retune_req_normal;
2479 				break;
2480 			}
2481 		}
2482 		*result = retune_req_none;
2483 		break;
2484 	case MMCBR_IVAR_VCCQ:
2485 		*result = slot->host.ios.vccq;
2486 		break;
2487 	case MMCBR_IVAR_CAPS:
2488 		*result = slot->host.caps;
2489 		break;
2490 	case MMCBR_IVAR_TIMING:
2491 		*result = slot->host.ios.timing;
2492 		break;
2493 	case MMCBR_IVAR_MAX_DATA:
2494 		/*
2495 		 * Re-tuning modes 1 and 2 restrict the maximum data length
2496 		 * per read/write command to 4 MiB.
2497 		 */
2498 		if (slot->opt & SDHCI_TUNING_ENABLED &&
2499 		    (slot->retune_mode == SDHCI_RETUNE_MODE_1 ||
2500 		    slot->retune_mode == SDHCI_RETUNE_MODE_2)) {
2501 			*result = 4 * 1024 * 1024 / MMC_SECTOR_SIZE;
2502 			break;
2503 		}
2504 		*result = 65535;
2505 		break;
2506 	case MMCBR_IVAR_MAX_BUSY_TIMEOUT:
2507 		/*
2508 		 * Currently, sdhci_start_data() hardcodes 1 s for all CMDs.
2509 		 */
2510 		*result = 1000000;
2511 		break;
2512 	}
2513 	return (0);
2514 }
2515 
2516 int
2517 sdhci_generic_write_ivar(device_t bus, device_t child, int which,
2518     uintptr_t value)
2519 {
2520 	struct sdhci_slot *slot = device_get_ivars(child);
2521 	uint32_t clock, max_clock;
2522 	int i;
2523 
2524 	if (sdhci_debug > 1)
2525 		slot_printf(slot, "%s: var=%d\n", __func__, which);
2526 	switch (which) {
2527 	default:
2528 		return (EINVAL);
2529 	case MMCBR_IVAR_BUS_MODE:
2530 		slot->host.ios.bus_mode = value;
2531 		break;
2532 	case MMCBR_IVAR_BUS_WIDTH:
2533 		slot->host.ios.bus_width = value;
2534 		break;
2535 	case MMCBR_IVAR_CHIP_SELECT:
2536 		slot->host.ios.chip_select = value;
2537 		break;
2538 	case MMCBR_IVAR_CLOCK:
2539 		if (value > 0) {
2540 			max_clock = slot->max_clk;
2541 			clock = max_clock;
2542 
2543 			if (slot->version < SDHCI_SPEC_300) {
2544 				for (i = 0; i < SDHCI_200_MAX_DIVIDER;
2545 				    i <<= 1) {
2546 					if (clock <= value)
2547 						break;
2548 					clock >>= 1;
2549 				}
2550 			} else {
2551 				for (i = 0; i < SDHCI_300_MAX_DIVIDER;
2552 				    i += 2) {
2553 					if (clock <= value)
2554 						break;
2555 					clock = max_clock / (i + 2);
2556 				}
2557 			}
2558 
2559 			slot->host.ios.clock = clock;
2560 		} else
2561 			slot->host.ios.clock = 0;
2562 		break;
2563 	case MMCBR_IVAR_MODE:
2564 		slot->host.mode = value;
2565 		break;
2566 	case MMCBR_IVAR_OCR:
2567 		slot->host.ocr = value;
2568 		break;
2569 	case MMCBR_IVAR_POWER_MODE:
2570 		slot->host.ios.power_mode = value;
2571 		break;
2572 	case MMCBR_IVAR_VDD:
2573 		slot->host.ios.vdd = value;
2574 		break;
2575 	case MMCBR_IVAR_VCCQ:
2576 		slot->host.ios.vccq = value;
2577 		break;
2578 	case MMCBR_IVAR_TIMING:
2579 		slot->host.ios.timing = value;
2580 		break;
2581 	case MMCBR_IVAR_CAPS:
2582 	case MMCBR_IVAR_HOST_OCR:
2583 	case MMCBR_IVAR_F_MIN:
2584 	case MMCBR_IVAR_F_MAX:
2585 	case MMCBR_IVAR_MAX_DATA:
2586 	case MMCBR_IVAR_RETUNE_REQ:
2587 		return (EINVAL);
2588 	}
2589 	return (0);
2590 }
2591 
2592 #ifdef MMCCAM
2593 void
2594 sdhci_start_slot(struct sdhci_slot *slot)
2595 {
2596 
2597 	if ((slot->devq = cam_simq_alloc(1)) == NULL)
2598 		goto fail;
2599 
2600 	mtx_init(&slot->sim_mtx, "sdhcisim", NULL, MTX_DEF);
2601 	slot->sim = cam_sim_alloc(sdhci_cam_action, sdhci_cam_poll,
2602 	    "sdhci_slot", slot, device_get_unit(slot->bus),
2603 	    &slot->sim_mtx, 1, 1, slot->devq);
2604 
2605 	if (slot->sim == NULL) {
2606 		cam_simq_free(slot->devq);
2607 		slot_printf(slot, "cannot allocate CAM SIM\n");
2608 		goto fail;
2609 	}
2610 
2611 	mtx_lock(&slot->sim_mtx);
2612 	if (xpt_bus_register(slot->sim, slot->bus, 0) != 0) {
2613 		slot_printf(slot, "cannot register SCSI pass-through bus\n");
2614 		cam_sim_free(slot->sim, FALSE);
2615 		cam_simq_free(slot->devq);
2616 		mtx_unlock(&slot->sim_mtx);
2617 		goto fail;
2618 	}
2619 	mtx_unlock(&slot->sim_mtx);
2620 
2621 	/* End CAM-specific init */
2622 	slot->card_present = 0;
2623 	sdhci_card_task(slot, 0);
2624 	return;
2625 
2626 fail:
2627 	if (slot->sim != NULL) {
2628 		mtx_lock(&slot->sim_mtx);
2629 		xpt_bus_deregister(cam_sim_path(slot->sim));
2630 		cam_sim_free(slot->sim, FALSE);
2631 		mtx_unlock(&slot->sim_mtx);
2632 	}
2633 
2634 	if (slot->devq != NULL)
2635 		cam_simq_free(slot->devq);
2636 }
2637 
2638 void
2639 sdhci_cam_action(struct cam_sim *sim, union ccb *ccb)
2640 {
2641 	struct sdhci_slot *slot;
2642 
2643 	slot = cam_sim_softc(sim);
2644 	if (slot == NULL) {
2645 		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2646 		xpt_done(ccb);
2647 		return;
2648 	}
2649 
2650 	mtx_assert(&slot->sim_mtx, MA_OWNED);
2651 
2652 	switch (ccb->ccb_h.func_code) {
2653 	case XPT_PATH_INQ:
2654 		mmc_path_inq(&ccb->cpi, "Deglitch Networks", sim, maxphys);
2655 		break;
2656 
2657 	case XPT_MMC_GET_TRAN_SETTINGS:
2658 	case XPT_GET_TRAN_SETTINGS:
2659 	{
2660 		struct ccb_trans_settings *cts = &ccb->cts;
2661 		uint32_t max_data;
2662 
2663 		if (sdhci_debug > 1)
2664 			slot_printf(slot, "Got XPT_GET_TRAN_SETTINGS\n");
2665 
2666 		cts->protocol = PROTO_MMCSD;
2667 		cts->protocol_version = 1;
2668 		cts->transport = XPORT_MMCSD;
2669 		cts->transport_version = 1;
2670 		cts->xport_specific.valid = 0;
2671 		cts->proto_specific.mmc.host_ocr = slot->host.host_ocr;
2672 		cts->proto_specific.mmc.host_f_min = slot->host.f_min;
2673 		cts->proto_specific.mmc.host_f_max = slot->host.f_max;
2674 		cts->proto_specific.mmc.host_caps = slot->host.caps;
2675 		/*
2676 		 * Re-tuning modes 1 and 2 restrict the maximum data length
2677 		 * per read/write command to 4 MiB.
2678 		 */
2679 		if (slot->opt & SDHCI_TUNING_ENABLED &&
2680 		    (slot->retune_mode == SDHCI_RETUNE_MODE_1 ||
2681 		    slot->retune_mode == SDHCI_RETUNE_MODE_2)) {
2682 			max_data = 4 * 1024 * 1024 / MMC_SECTOR_SIZE;
2683 		} else {
2684 			max_data = 65535;
2685 		}
2686 		cts->proto_specific.mmc.host_max_data = max_data;
2687 
2688 		memcpy(&cts->proto_specific.mmc.ios, &slot->host.ios, sizeof(struct mmc_ios));
2689 		ccb->ccb_h.status = CAM_REQ_CMP;
2690 		break;
2691 	}
2692 	case XPT_MMC_SET_TRAN_SETTINGS:
2693 	case XPT_SET_TRAN_SETTINGS:
2694 		if (sdhci_debug > 1)
2695 			slot_printf(slot, "Got XPT_SET_TRAN_SETTINGS\n");
2696 		sdhci_cam_settran_settings(slot, ccb);
2697 		ccb->ccb_h.status = CAM_REQ_CMP;
2698 		break;
2699 	case XPT_RESET_BUS:
2700 		if (sdhci_debug > 1)
2701 			slot_printf(slot, "Got XPT_RESET_BUS, ACK it...\n");
2702 		ccb->ccb_h.status = CAM_REQ_CMP;
2703 		break;
2704 	case XPT_MMC_IO:
2705 		/*
2706 		 * Here is the HW-dependent part of
2707 		 * sending the command to the underlying h/w
2708 		 * At some point in the future an interrupt comes.
2709 		 * Then the request will be marked as completed.
2710 		 */
2711 		if (__predict_false(sdhci_debug > 1))
2712 			slot_printf(slot, "Got XPT_MMC_IO\n");
2713 		ccb->ccb_h.status = CAM_REQ_INPROG;
2714 
2715 		sdhci_cam_request(cam_sim_softc(sim), ccb);
2716 		return;
2717 	default:
2718 		ccb->ccb_h.status = CAM_REQ_INVALID;
2719 		break;
2720 	}
2721 	xpt_done(ccb);
2722 	return;
2723 }
2724 
2725 void
2726 sdhci_cam_poll(struct cam_sim *sim)
2727 {
2728 	return;
2729 }
2730 
2731 static int
2732 sdhci_cam_get_possible_host_clock(const struct sdhci_slot *slot,
2733     int proposed_clock)
2734 {
2735 	int max_clock, clock, i;
2736 
2737 	if (proposed_clock == 0)
2738 		return 0;
2739 	max_clock = slot->max_clk;
2740 	clock = max_clock;
2741 
2742 	if (slot->version < SDHCI_SPEC_300) {
2743 		for (i = 0; i < SDHCI_200_MAX_DIVIDER; i <<= 1) {
2744 			if (clock <= proposed_clock)
2745 				break;
2746 			clock >>= 1;
2747 		}
2748 	} else {
2749 		for (i = 0; i < SDHCI_300_MAX_DIVIDER; i += 2) {
2750 			if (clock <= proposed_clock)
2751 				break;
2752 			clock = max_clock / (i + 2);
2753 		}
2754 	}
2755 	return clock;
2756 }
2757 
2758 static int
2759 sdhci_cam_settran_settings(struct sdhci_slot *slot, union ccb *ccb)
2760 {
2761 	struct mmc_ios *ios;
2762 	const struct mmc_ios *new_ios;
2763 	const struct ccb_trans_settings_mmc *cts;
2764 
2765 	ios = &slot->host.ios;
2766 	cts = &ccb->cts.proto_specific.mmc;
2767 	new_ios = &cts->ios;
2768 
2769 	/* Update only requested fields */
2770 	if (cts->ios_valid & MMC_CLK) {
2771 		ios->clock = sdhci_cam_get_possible_host_clock(slot, new_ios->clock);
2772 		if (sdhci_debug > 1)
2773 			slot_printf(slot, "Clock => %d\n", ios->clock);
2774 	}
2775 	if (cts->ios_valid & MMC_VDD) {
2776 		ios->vdd = new_ios->vdd;
2777 		if (sdhci_debug > 1)
2778 			slot_printf(slot, "VDD => %d\n", ios->vdd);
2779 	}
2780 	if (cts->ios_valid & MMC_CS) {
2781 		ios->chip_select = new_ios->chip_select;
2782 		if (sdhci_debug > 1)
2783 			slot_printf(slot, "CS => %d\n", ios->chip_select);
2784 	}
2785 	if (cts->ios_valid & MMC_BW) {
2786 		ios->bus_width = new_ios->bus_width;
2787 		if (sdhci_debug > 1)
2788 			slot_printf(slot, "Bus width => %d\n", ios->bus_width);
2789 	}
2790 	if (cts->ios_valid & MMC_PM) {
2791 		ios->power_mode = new_ios->power_mode;
2792 		if (sdhci_debug > 1)
2793 			slot_printf(slot, "Power mode => %d\n", ios->power_mode);
2794 	}
2795 	if (cts->ios_valid & MMC_BT) {
2796 		ios->timing = new_ios->timing;
2797 		if (sdhci_debug > 1)
2798 			slot_printf(slot, "Timing => %d\n", ios->timing);
2799 	}
2800 	if (cts->ios_valid & MMC_BM) {
2801 		ios->bus_mode = new_ios->bus_mode;
2802 		if (sdhci_debug > 1)
2803 			slot_printf(slot, "Bus mode => %d\n", ios->bus_mode);
2804 	}
2805 	if (cts->ios_valid & MMC_VCCQ) {
2806 		ios->vccq = new_ios->vccq;
2807 		if (sdhci_debug > 1)
2808 			slot_printf(slot, "VCCQ => %d\n", ios->vccq);
2809 	}
2810 
2811 	/* XXX Provide a way to call a chip-specific IOS update, required for TI */
2812 	return (sdhci_cam_update_ios(slot));
2813 }
2814 
2815 static int
2816 sdhci_cam_update_ios(struct sdhci_slot *slot)
2817 {
2818 	struct mmc_ios *ios = &slot->host.ios;
2819 
2820 	if (sdhci_debug > 1)
2821 		slot_printf(slot, "%s: power_mode=%d, clk=%d, bus_width=%d, timing=%d\n",
2822 		    __func__, ios->power_mode, ios->clock, ios->bus_width, ios->timing);
2823 	SDHCI_LOCK(slot);
2824 	/* Do full reset on bus power down to clear from any state. */
2825 	if (ios->power_mode == power_off) {
2826 		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
2827 		sdhci_init(slot);
2828 	}
2829 	/* Configure the bus. */
2830 	sdhci_set_clock(slot, ios->clock);
2831 	sdhci_set_power(slot, (ios->power_mode == power_off) ? 0 : ios->vdd);
2832 	if (ios->bus_width == bus_width_8) {
2833 		slot->hostctrl |= SDHCI_CTRL_8BITBUS;
2834 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
2835 	} else if (ios->bus_width == bus_width_4) {
2836 		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
2837 		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
2838 	} else if (ios->bus_width == bus_width_1) {
2839 		slot->hostctrl &= ~SDHCI_CTRL_8BITBUS;
2840 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
2841 	} else {
2842 		panic("Invalid bus width: %d", ios->bus_width);
2843 	}
2844 	if (ios->timing == bus_timing_hs &&
2845 	    !(slot->quirks & SDHCI_QUIRK_DONT_SET_HISPD_BIT))
2846 		slot->hostctrl |= SDHCI_CTRL_HISPD;
2847 	else
2848 		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
2849 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
2850 	/* Some controllers like reset after bus changes. */
2851 	if(slot->quirks & SDHCI_QUIRK_RESET_ON_IOS)
2852 		SDHCI_RESET(slot->bus, slot,
2853 		    SDHCI_RESET_CMD | SDHCI_RESET_DATA);
2854 
2855 	SDHCI_UNLOCK(slot);
2856 	return (0);
2857 }
2858 
2859 static int
2860 sdhci_cam_request(struct sdhci_slot *slot, union ccb *ccb)
2861 {
2862 	const struct ccb_mmcio *mmcio;
2863 
2864 	mmcio = &ccb->mmcio;
2865 
2866 	SDHCI_LOCK(slot);
2867 /*	if (slot->req != NULL) {
2868 		SDHCI_UNLOCK(slot);
2869 		return (EBUSY);
2870 	}
2871 */
2872 	if (__predict_false(sdhci_debug > 1)) {
2873 		slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x "
2874 		    "blksz=%zu blkcnt=%zu\n",
2875 		    mmcio->cmd.opcode, mmcio->cmd.arg, mmcio->cmd.flags,
2876 		    mmcio->cmd.data != NULL ? (unsigned int) mmcio->cmd.data->len : 0,
2877 		    mmcio->cmd.data != NULL ? mmcio->cmd.data->flags : 0,
2878 		    mmcio->cmd.data != NULL ? mmcio->cmd.data->block_size : 0,
2879 		    mmcio->cmd.data != NULL ? mmcio->cmd.data->block_count : 0);
2880 	}
2881 	if (mmcio->cmd.data != NULL) {
2882 		if (mmcio->cmd.data->len == 0 || mmcio->cmd.data->flags == 0)
2883 			panic("data->len = %d, data->flags = %d -- something is b0rked",
2884 			    (int)mmcio->cmd.data->len, mmcio->cmd.data->flags);
2885 	}
2886 	slot->ccb = ccb;
2887 	slot->flags = 0;
2888 	sdhci_start(slot);
2889 	SDHCI_UNLOCK(slot);
2890 	if (dumping) {
2891 		while (slot->ccb != NULL) {
2892 			sdhci_generic_intr(slot);
2893 			DELAY(10);
2894 		}
2895 	}
2896 	return (0);
2897 }
2898 #endif /* MMCCAM */
2899 
2900 MODULE_VERSION(sdhci, SDHCI_VERSION);
2901