xref: /freebsd/sys/dev/sdhci/sdhci.c (revision 33644623554bb0fc57ed3c7d874193a498679b22)
1 /*-
2  * Copyright (c) 2008 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28 
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/bus.h>
32 #include <sys/conf.h>
33 #include <sys/kernel.h>
34 #include <sys/lock.h>
35 #include <sys/module.h>
36 #include <sys/mutex.h>
37 #include <sys/resource.h>
38 #include <sys/rman.h>
39 #include <sys/taskqueue.h>
40 
41 #include <dev/pci/pcireg.h>
42 #include <dev/pci/pcivar.h>
43 
44 #include <machine/bus.h>
45 #include <machine/resource.h>
46 #include <machine/stdarg.h>
47 
48 #include <dev/mmc/bridge.h>
49 #include <dev/mmc/mmcreg.h>
50 #include <dev/mmc/mmcbrvar.h>
51 
52 #include "mmcbr_if.h"
53 #include "sdhci.h"
54 
55 #define DMA_BLOCK_SIZE	4096
56 #define DMA_BOUNDARY	0	/* DMA reload every 4K */
57 
58 /* Controller doesn't honor resets unless we touch the clock register */
59 #define SDHCI_QUIRK_CLOCK_BEFORE_RESET			(1<<0)
60 /* Controller really supports DMA */
61 #define SDHCI_QUIRK_FORCE_DMA				(1<<1)
62 /* Controller has unusable DMA engine */
63 #define SDHCI_QUIRK_BROKEN_DMA				(1<<2)
64 /* Controller doesn't like to be reset when there is no card inserted. */
65 #define SDHCI_QUIRK_NO_CARD_NO_RESET			(1<<3)
66 /* Controller has flaky internal state so reset it on each ios change */
67 #define SDHCI_QUIRK_RESET_ON_IOS			(1<<4)
68 /* Controller can only DMA chunk sizes that are a multiple of 32 bits */
69 #define SDHCI_QUIRK_32BIT_DMA_SIZE			(1<<5)
70 /* Controller needs to be reset after each request to stay stable */
71 #define SDHCI_QUIRK_RESET_AFTER_REQUEST			(1<<6)
72 /* Controller has an off-by-one issue with timeout value */
73 #define SDHCI_QUIRK_INCR_TIMEOUT_CONTROL		(1<<7)
74 /* Controller has broken read timings */
75 #define SDHCI_QUIRK_BROKEN_TIMINGS			(1<<8)
76 
77 static const struct sdhci_device {
78 	uint32_t	model;
79 	uint16_t	subvendor;
80 	char		*desc;
81 	u_int		quirks;
82 } sdhci_devices[] = {
83 	{ 0x08221180, 	0xffff,	"RICOH R5C822 SD",
84 	    SDHCI_QUIRK_FORCE_DMA },
85 	{ 0x8034104c, 	0xffff, "TI XX21/XX11 SD",
86 	    SDHCI_QUIRK_FORCE_DMA },
87 	{ 0x05501524, 	0xffff, "ENE CB712 SD",
88 	    SDHCI_QUIRK_BROKEN_TIMINGS },
89 	{ 0x05511524, 	0xffff, "ENE CB712 SD 2",
90 	    SDHCI_QUIRK_BROKEN_TIMINGS },
91 	{ 0x07501524, 	0xffff, "ENE CB714 SD",
92 	    SDHCI_QUIRK_RESET_ON_IOS |
93 	    SDHCI_QUIRK_BROKEN_TIMINGS },
94 	{ 0x07511524, 	0xffff, "ENE CB714 SD 2",
95 	    SDHCI_QUIRK_RESET_ON_IOS |
96 	    SDHCI_QUIRK_BROKEN_TIMINGS },
97 	{ 0x410111ab, 	0xffff, "Marvell CaFe SD",
98 	    SDHCI_QUIRK_INCR_TIMEOUT_CONTROL },
99 	{ 0x2381197B, 	0xffff,	"JMicron JMB38X SD",
100 	    SDHCI_QUIRK_32BIT_DMA_SIZE |
101 	    SDHCI_QUIRK_RESET_AFTER_REQUEST },
102 	{ 0,		0xffff,	NULL,
103 	    0 }
104 };
105 
106 struct sdhci_softc;
107 
108 struct sdhci_slot {
109 	struct sdhci_softc	*sc;
110 	device_t	dev;		/* Slot device */
111 	u_char		num;		/* Slot number */
112 	u_char		opt;		/* Slot options */
113 #define SDHCI_HAVE_DMA		1
114 	uint32_t	max_clk;	/* Max possible freq */
115 	uint32_t	timeout_clk;	/* Timeout freq */
116 	struct resource	*mem_res;	/* Memory resource */
117 	int		mem_rid;
118 	bus_dma_tag_t 	dmatag;
119 	bus_dmamap_t 	dmamap;
120 	u_char		*dmamem;
121 	bus_addr_t	paddr;		/* DMA buffer address */
122 	struct task	card_task;	/* Card presence check task */
123 	struct callout	card_callout;	/* Card insert delay callout */
124 	struct mmc_host host;		/* Host parameters */
125 	struct mmc_request *req;	/* Current request */
126 	struct mmc_command *curcmd;	/* Current command of current request */
127 
128 	uint32_t	intmask;	/* Current interrupt mask */
129 	uint32_t	clock;		/* Current clock freq. */
130 	size_t		offset;		/* Data buffer offset */
131 	uint8_t		hostctrl;	/* Current host control register */
132 	u_char		power;		/* Current power */
133 	u_char		bus_busy;	/* Bus busy status */
134 	u_char		cmd_done;	/* CMD command part done flag */
135 	u_char		data_done;	/* DAT command part done flag */
136 	u_char		flags;		/* Request execution flags */
137 #define CMD_STARTED		1
138 #define STOP_STARTED		2
139 #define SDHCI_USE_DMA		4	/* Use DMA for this req. */
140 	struct mtx	mtx;		/* Slot mutex */
141 };
142 
143 struct sdhci_softc {
144 	device_t	dev;		/* Controller device */
145 	u_int		quirks;		/* Chip specific quirks */
146 	struct resource *irq_res;	/* IRQ resource */
147 	int 		irq_rid;
148 	void 		*intrhand;	/* Interrupt handle */
149 
150 	int		num_slots;	/* Number of slots on this controller */
151 	struct sdhci_slot slots[6];
152 };
153 
154 static inline uint8_t
155 RD1(struct sdhci_slot *slot, bus_size_t off)
156 {
157 	bus_barrier(slot->mem_res, 0, 0xFF,
158 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
159 	return bus_read_1(slot->mem_res, off);
160 }
161 
162 static inline void
163 WR1(struct sdhci_slot *slot, bus_size_t off, uint8_t val)
164 {
165 	bus_barrier(slot->mem_res, 0, 0xFF,
166 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
167 	bus_write_1(slot->mem_res, off, val);
168 }
169 
170 static inline uint16_t
171 RD2(struct sdhci_slot *slot, bus_size_t off)
172 {
173 	bus_barrier(slot->mem_res, 0, 0xFF,
174 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
175 	return bus_read_2(slot->mem_res, off);
176 }
177 
178 static inline void
179 WR2(struct sdhci_slot *slot, bus_size_t off, uint16_t val)
180 {
181 	bus_barrier(slot->mem_res, 0, 0xFF,
182 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
183 	bus_write_2(slot->mem_res, off, val);
184 }
185 
186 static inline uint32_t
187 RD4(struct sdhci_slot *slot, bus_size_t off)
188 {
189 	bus_barrier(slot->mem_res, 0, 0xFF,
190 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
191 	return bus_read_4(slot->mem_res, off);
192 }
193 
194 static inline void
195 WR4(struct sdhci_slot *slot, bus_size_t off, uint32_t val)
196 {
197 	bus_barrier(slot->mem_res, 0, 0xFF,
198 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
199 	bus_write_4(slot->mem_res, off, val);
200 }
201 
202 /* bus entry points */
203 static int sdhci_probe(device_t dev);
204 static int sdhci_attach(device_t dev);
205 static int sdhci_detach(device_t dev);
206 static void sdhci_intr(void *);
207 
208 static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock);
209 static void sdhci_start(struct sdhci_slot *slot);
210 static void sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data);
211 
212 static void sdhci_card_task(void *, int);
213 
214 /* helper routines */
215 #define SDHCI_LOCK(_slot)		mtx_lock(&(_slot)->mtx)
216 #define	SDHCI_UNLOCK(_slot)		mtx_unlock(&(_slot)->mtx)
217 #define SDHCI_LOCK_INIT(_slot) \
218 	mtx_init(&_slot->mtx, "SD slot mtx", "sdhci", MTX_DEF)
219 #define SDHCI_LOCK_DESTROY(_slot)	mtx_destroy(&_slot->mtx);
220 #define SDHCI_ASSERT_LOCKED(_slot)	mtx_assert(&_slot->mtx, MA_OWNED);
221 #define SDHCI_ASSERT_UNLOCKED(_slot)	mtx_assert(&_slot->mtx, MA_NOTOWNED);
222 
223 static int
224 slot_printf(struct sdhci_slot *slot, const char * fmt, ...)
225 {
226 	va_list ap;
227 	int retval;
228 
229     	retval = printf("%s-slot%d: ",
230 	    device_get_nameunit(slot->sc->dev), slot->num);
231 
232 	va_start(ap, fmt);
233 	retval += vprintf(fmt, ap);
234 	va_end(ap);
235 	return (retval);
236 }
237 
238 static void
239 sdhci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
240 {
241 	if (error != 0) {
242 		printf("getaddr: error %d\n", error);
243 		return;
244 	}
245 	*(bus_addr_t *)arg = segs[0].ds_addr;
246 }
247 
248 static void
249 sdhci_dumpregs(struct sdhci_slot *slot)
250 {
251 	slot_printf(slot,
252 	    "============== REGISTER DUMP ==============\n");
253 
254 	slot_printf(slot, "Sys addr: 0x%08x | Version:  0x%08x\n",
255 	    RD4(slot, SDHCI_DMA_ADDRESS), RD2(slot, SDHCI_HOST_VERSION));
256 	slot_printf(slot, "Blk size: 0x%08x | Blk cnt:  0x%08x\n",
257 	    RD2(slot, SDHCI_BLOCK_SIZE), RD2(slot, SDHCI_BLOCK_COUNT));
258 	slot_printf(slot, "Argument: 0x%08x | Trn mode: 0x%08x\n",
259 	    RD4(slot, SDHCI_ARGUMENT), RD2(slot, SDHCI_TRANSFER_MODE));
260 	slot_printf(slot, "Present:  0x%08x | Host ctl: 0x%08x\n",
261 	    RD4(slot, SDHCI_PRESENT_STATE), RD1(slot, SDHCI_HOST_CONTROL));
262 	slot_printf(slot, "Power:    0x%08x | Blk gap:  0x%08x\n",
263 	    RD1(slot, SDHCI_POWER_CONTROL), RD1(slot, SDHCI_BLOCK_GAP_CONTROL));
264 	slot_printf(slot, "Wake-up:  0x%08x | Clock:    0x%08x\n",
265 	    RD1(slot, SDHCI_WAKE_UP_CONTROL), RD2(slot, SDHCI_CLOCK_CONTROL));
266 	slot_printf(slot, "Timeout:  0x%08x | Int stat: 0x%08x\n",
267 	    RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS));
268 	slot_printf(slot, "Int enab: 0x%08x | Sig enab: 0x%08x\n",
269 	    RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE));
270 	slot_printf(slot, "AC12 err: 0x%08x | Slot int: 0x%08x\n",
271 	    RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_SLOT_INT_STATUS));
272 	slot_printf(slot, "Caps:     0x%08x | Max curr: 0x%08x\n",
273 	    RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_MAX_CURRENT));
274 
275 	slot_printf(slot,
276 	    "===========================================\n");
277 }
278 
279 static void
280 sdhci_reset(struct sdhci_slot *slot, uint8_t mask)
281 {
282 	int timeout;
283 	uint8_t res;
284 
285 	if (slot->sc->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
286 		if (!(RD4(slot, SDHCI_PRESENT_STATE) &
287 			SDHCI_CARD_PRESENT))
288 			return;
289 	}
290 
291 	/* Some controllers need this kick or reset won't work. */
292 	if ((mask & SDHCI_RESET_ALL) == 0 &&
293 	    (slot->sc->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)) {
294 		uint32_t clock;
295 
296 		/* This is to force an update */
297 		clock = slot->clock;
298 		slot->clock = 0;
299 		sdhci_set_clock(slot, clock);
300 	}
301 
302 	WR1(slot, SDHCI_SOFTWARE_RESET, mask);
303 
304 	if (mask & SDHCI_RESET_ALL)
305 		slot->clock = 0;
306 
307 	/* Wait max 100 ms */
308 	timeout = 100;
309 	/* Controller clears the bits when it's done */
310 	while ((res = RD1(slot, SDHCI_SOFTWARE_RESET)) & mask) {
311 		if (timeout == 0) {
312 			slot_printf(slot,
313 			    "Reset 0x%x never completed - 0x%x.\n",
314 			    (int)mask, (int)res);
315 			sdhci_dumpregs(slot);
316 			return;
317 		}
318 		timeout--;
319 		DELAY(1000);
320 	}
321 }
322 
323 static void
324 sdhci_init(struct sdhci_slot *slot)
325 {
326 
327 	sdhci_reset(slot, SDHCI_RESET_ALL);
328 
329 	/* Enable interrupts. */
330 	slot->intmask = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
331 	    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT | SDHCI_INT_INDEX |
332 	    SDHCI_INT_END_BIT | SDHCI_INT_CRC | SDHCI_INT_TIMEOUT |
333 	    SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT |
334 	    SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
335 	    SDHCI_INT_DMA_END | SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE |
336 	    SDHCI_INT_ACMD12ERR;
337 	WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
338 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
339 }
340 
341 static void
342 sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
343 {
344 	uint32_t res;
345 	uint16_t clk;
346 	int timeout;
347 
348 	if (clock == slot->clock)
349 		return;
350 	slot->clock = clock;
351 
352 	/* Turn off the clock. */
353 	WR2(slot, SDHCI_CLOCK_CONTROL, 0);
354 	/* If no clock requested - left it so. */
355 	if (clock == 0)
356 		return;
357 	/* Looking for highest freq <= clock. */
358 	res = slot->max_clk;
359 	for (clk = 1; clk < 256; clk <<= 1) {
360 		if (res <= clock)
361 			break;
362 		res >>= 1;
363 	}
364 	/* Divider 1:1 is 0x00, 2:1 is 0x01, 256:1 is 0x80 ... */
365 	clk >>= 1;
366 	/* Now we have got divider, set it. */
367 	clk <<= SDHCI_DIVIDER_SHIFT;
368 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
369 	/* Enable clock. */
370 	clk |= SDHCI_CLOCK_INT_EN;
371 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
372 	/* Wait up to 10 ms until it stabilize. */
373 	timeout = 10;
374 	while (!((clk = RD2(slot, SDHCI_CLOCK_CONTROL))
375 		& SDHCI_CLOCK_INT_STABLE)) {
376 		if (timeout == 0) {
377 			slot_printf(slot,
378 			    "Internal clock never stabilised.\n");
379 			sdhci_dumpregs(slot);
380 			return;
381 		}
382 		timeout--;
383 		DELAY(1000);
384 	}
385 	/* Pass clock signal to the bus. */
386 	clk |= SDHCI_CLOCK_CARD_EN;
387 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
388 }
389 
390 static void
391 sdhci_set_power(struct sdhci_slot *slot, u_char power)
392 {
393 	uint8_t pwr;
394 
395 	if (slot->power == power)
396 		return;
397 	slot->power = power;
398 
399 	/* Turn off the power. */
400 	pwr = 0;
401 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
402 	/* If power down requested - left it so. */
403 	if (power == 0)
404 		return;
405 	/* Set voltage. */
406 	switch (1 << power) {
407 	case MMC_OCR_LOW_VOLTAGE:
408 		pwr |= SDHCI_POWER_180;
409 		break;
410 	case MMC_OCR_290_300:
411 	case MMC_OCR_300_310:
412 		pwr |= SDHCI_POWER_300;
413 		break;
414 	case MMC_OCR_320_330:
415 	case MMC_OCR_330_340:
416 		pwr |= SDHCI_POWER_330;
417 		break;
418 	}
419 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
420 	/* Turn on the power. */
421 	pwr |= SDHCI_POWER_ON;
422 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
423 }
424 
425 static void
426 sdhci_read_block_pio(struct sdhci_slot *slot)
427 {
428 	uint32_t data;
429 	char *buffer;
430 	size_t left;
431 
432 	buffer = slot->curcmd->data->data;
433 	buffer += slot->offset;
434 	/* Transfer one block at a time. */
435 	left = min(512, slot->curcmd->data->len - slot->offset);
436 	slot->offset += left;
437 
438 	/* If we are too fast, broken controllers return zeroes. */
439 	if (slot->sc->quirks & SDHCI_QUIRK_BROKEN_TIMINGS)
440 		DELAY(10);
441 	/* Handle unalligned and alligned buffer cases. */
442 	if ((intptr_t)buffer & 3) {
443 		while (left > 3) {
444 			data = RD4(slot, SDHCI_BUFFER);
445 			buffer[0] = data;
446 			buffer[1] = (data >> 8);
447 			buffer[2] = (data >> 16);
448 			buffer[3] = (data >> 24);
449 			buffer += 4;
450 			left -= 4;
451 		}
452 	} else {
453 		bus_read_multi_stream_4(slot->mem_res, SDHCI_BUFFER,
454 		    (uint32_t *)buffer, left >> 2);
455 		left &= 3;
456 	}
457 	/* Handle uneven size case. */
458 	if (left > 0) {
459 		data = RD4(slot, SDHCI_BUFFER);
460 		while (left > 0) {
461 			*(buffer++) = data;
462 			data >>= 8;
463 			left--;
464 		}
465 	}
466 }
467 
468 static void
469 sdhci_write_block_pio(struct sdhci_slot *slot)
470 {
471 	uint32_t data = 0;
472 	char *buffer;
473 	size_t left;
474 
475 	buffer = slot->curcmd->data->data;
476 	buffer += slot->offset;
477 	/* Transfer one block at a time. */
478 	left = min(512, slot->curcmd->data->len - slot->offset);
479 	slot->offset += left;
480 
481 	/* Handle unalligned and alligned buffer cases. */
482 	if ((intptr_t)buffer & 3) {
483 		while (left > 3) {
484 			data = buffer[0] +
485 			    (buffer[1] << 8) +
486 			    (buffer[2] << 16) +
487 			    (buffer[3] << 24);
488 			left -= 4;
489 			buffer += 4;
490 			WR4(slot, SDHCI_BUFFER, data);
491 		}
492 	} else {
493 		bus_write_multi_stream_4(slot->mem_res, SDHCI_BUFFER,
494 		    (uint32_t *)buffer, left >> 2);
495 		left &= 3;
496 	}
497 	/* Handle uneven size case. */
498 	if (left > 0) {
499 		while (left > 0) {
500 			data <<= 8;
501 			data += *(buffer++);
502 			left--;
503 		}
504 		WR4(slot, SDHCI_BUFFER, data);
505 	}
506 }
507 
508 static void
509 sdhci_transfer_pio(struct sdhci_slot *slot)
510 {
511 
512 	/* Read as many blocks as possible. */
513 	if (slot->curcmd->data->flags & MMC_DATA_READ) {
514 		while (RD4(slot, SDHCI_PRESENT_STATE) &
515 		    SDHCI_DATA_AVAILABLE) {
516 			sdhci_read_block_pio(slot);
517 			if (slot->offset >= slot->curcmd->data->len)
518 				break;
519 		}
520 	} else {
521 		while (RD4(slot, SDHCI_PRESENT_STATE) &
522 		    SDHCI_SPACE_AVAILABLE) {
523 			sdhci_write_block_pio(slot);
524 			if (slot->offset >= slot->curcmd->data->len)
525 				break;
526 		}
527 	}
528 }
529 
530 static void
531 sdhci_card_delay(void *arg)
532 {
533 	struct sdhci_slot *slot = arg;
534 
535 	taskqueue_enqueue(taskqueue_swi_giant, &slot->card_task);
536 }
537 
538 static void
539 sdhci_card_task(void *arg, int pending)
540 {
541 	struct sdhci_slot *slot = arg;
542 
543 	SDHCI_LOCK(slot);
544 	if (RD4(slot, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT) {
545 		if (slot->dev == NULL) {
546 			/* If card is present - attach mmc bus. */
547 			slot->dev = device_add_child(slot->sc->dev, "mmc", -1);
548 			device_set_ivars(slot->dev, slot);
549 			SDHCI_UNLOCK(slot);
550 			device_probe_and_attach(slot->dev);
551 		} else
552 			SDHCI_UNLOCK(slot);
553 	} else {
554 		if (slot->dev != NULL) {
555 			/* If no card present - detach mmc bus. */
556 			device_t d = slot->dev;
557 			slot->dev = NULL;
558 			SDHCI_UNLOCK(slot);
559 			device_delete_child(slot->sc->dev, d);
560 		} else
561 			SDHCI_UNLOCK(slot);
562 	}
563 }
564 
565 static int
566 sdhci_probe(device_t dev)
567 {
568 	uint32_t model;
569 	uint16_t subvendor;
570 	uint8_t class, subclass;
571 	int i, result;
572 
573 	model = (uint32_t)pci_get_device(dev) << 16;
574 	model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
575 	subvendor = pci_get_subvendor(dev);
576 	class = pci_get_class(dev);
577 	subclass = pci_get_subclass(dev);
578 
579 	result = ENXIO;
580 	for (i = 0; sdhci_devices[i].model != 0; i++) {
581 		if (sdhci_devices[i].model == model &&
582 		    (sdhci_devices[i].subvendor == 0xffff ||
583 		    sdhci_devices[i].subvendor == subvendor)) {
584 			device_set_desc(dev, sdhci_devices[i].desc);
585 			result = BUS_PROBE_DEFAULT;
586 			break;
587 		}
588 	}
589 	if (result == ENXIO && class == PCIC_BASEPERIPH &&
590 	    subclass == PCIS_BASEPERIPH_SDHC) {
591 		device_set_desc(dev, "Generic SD HCI");
592 		result = BUS_PROBE_GENERIC;
593 	}
594 
595 	return (result);
596 }
597 
598 static int
599 sdhci_attach(device_t dev)
600 {
601 	struct sdhci_softc *sc = device_get_softc(dev);
602 	uint32_t model;
603 	uint16_t subvendor;
604 	uint8_t class, subclass, progif;
605 	int err, slots, bar, i;
606 
607 	sc->dev = dev;
608 	model = (uint32_t)pci_get_device(dev) << 16;
609 	model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
610 	subvendor = pci_get_subvendor(dev);
611 	class = pci_get_class(dev);
612 	subclass = pci_get_subclass(dev);
613 	progif = pci_get_progif(dev);
614 	/* Apply chip specific quirks. */
615 	for (i = 0; sdhci_devices[i].model != 0; i++) {
616 		if (sdhci_devices[i].model == model &&
617 		    (sdhci_devices[i].subvendor == 0xffff ||
618 		    sdhci_devices[i].subvendor == subvendor)) {
619 			sc->quirks = sdhci_devices[i].quirks;
620 			break;
621 		}
622 	}
623 	/* Read slots info from PCI registers. */
624 	slots = pci_read_config(dev, PCI_SLOT_INFO, 1);
625 	bar = PCI_SLOT_INFO_FIRST_BAR(slots);
626 	slots = PCI_SLOT_INFO_SLOTS(slots);
627 	if (slots > 6 || bar > 5) {
628 		device_printf(dev, "Incorrect slots information (%d, %d).\n",
629 		    slots, bar);
630 		return (EINVAL);
631 	}
632 	/* Allocate IRQ. */
633 	sc->irq_rid = 0;
634 	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid,
635 	    RF_SHAREABLE | RF_ACTIVE);
636 	if (sc->irq_res == NULL) {
637 		device_printf(dev, "Can't allocate IRQ\n");
638 		return (ENOMEM);
639 	}
640 	/* Scan all slots. */
641 	for (i = 0; i < slots; i++) {
642 		struct sdhci_slot *slot = &sc->slots[sc->num_slots];
643 		uint32_t caps;
644 
645 		SDHCI_LOCK_INIT(slot);
646 		slot->sc = sc;
647 		slot->num = sc->num_slots;
648 		/* Allocate memory. */
649 		slot->mem_rid = PCIR_BAR(bar + i);
650 		slot->mem_res = bus_alloc_resource(dev,
651 		    SYS_RES_MEMORY, &slot->mem_rid, 0ul, ~0ul, 0x100, RF_ACTIVE);
652 		if (slot->mem_res == NULL) {
653 			device_printf(dev, "Can't allocate memory\n");
654 			SDHCI_LOCK_DESTROY(slot);
655 			continue;
656 		}
657 		/* Allocate DMA tag. */
658 		err = bus_dma_tag_create(bus_get_dma_tag(dev),
659 		    DMA_BLOCK_SIZE, 0, BUS_SPACE_MAXADDR_32BIT,
660 		    BUS_SPACE_MAXADDR, NULL, NULL,
661 		    DMA_BLOCK_SIZE, 1, DMA_BLOCK_SIZE,
662 		    BUS_DMA_ALLOCNOW, NULL, NULL,
663 		    &slot->dmatag);
664 		if (err != 0) {
665 			device_printf(dev, "Can't create DMA tag\n");
666 			SDHCI_LOCK_DESTROY(slot);
667 			continue;
668 		}
669 		/* Allocate DMA memory. */
670 		err = bus_dmamem_alloc(slot->dmatag, (void **)&slot->dmamem,
671 		    BUS_DMA_NOWAIT, &slot->dmamap);
672 		if (err != 0) {
673 			device_printf(dev, "Can't alloc DMA memory\n");
674 			SDHCI_LOCK_DESTROY(slot);
675 			continue;
676 		}
677 		/* Map the memory. */
678 		err = bus_dmamap_load(slot->dmatag, slot->dmamap,
679 		    (void *)slot->dmamem, DMA_BLOCK_SIZE,
680 		    sdhci_getaddr, &slot->paddr, 0);
681 		if (err != 0 || slot->paddr == 0) {
682 			device_printf(dev, "Can't load DMA memory\n");
683 			SDHCI_LOCK_DESTROY(slot);
684 			continue;
685 		}
686 		/* Initialize slot. */
687 		sdhci_init(slot);
688 		caps = RD4(slot, SDHCI_CAPABILITIES);
689 		/* Calculate base clock frequency. */
690 		slot->max_clk =
691 			(caps & SDHCI_CLOCK_BASE_MASK) >> SDHCI_CLOCK_BASE_SHIFT;
692 		if (slot->max_clk == 0) {
693 			device_printf(dev, "Hardware doesn't specify base clock "
694 			    "frequency.\n");
695 		}
696 		slot->max_clk *= 1000000;
697 		/* Calculate timeout clock frequency. */
698 		slot->timeout_clk =
699 			(caps & SDHCI_TIMEOUT_CLK_MASK) >> SDHCI_TIMEOUT_CLK_SHIFT;
700 		if (slot->timeout_clk == 0) {
701 			device_printf(dev, "Hardware doesn't specify timeout clock "
702 			    "frequency.\n");
703 		}
704 		if (caps & SDHCI_TIMEOUT_CLK_UNIT)
705 			slot->timeout_clk *= 1000;
706 
707 		slot->host.f_min = slot->max_clk / 256;
708 		slot->host.f_max = slot->max_clk;
709 		slot->host.host_ocr = 0;
710 		if (caps & SDHCI_CAN_VDD_330)
711 		    slot->host.host_ocr |= MMC_OCR_320_330 | MMC_OCR_330_340;
712 		if (caps & SDHCI_CAN_VDD_300)
713 		    slot->host.host_ocr |= MMC_OCR_290_300 | MMC_OCR_300_310;
714 		if (caps & SDHCI_CAN_VDD_180)
715 		    slot->host.host_ocr |= MMC_OCR_LOW_VOLTAGE;
716 		if (slot->host.host_ocr == 0) {
717 			device_printf(dev, "Hardware doesn't report any "
718 			    "support voltages.\n");
719 		}
720 		slot->host.caps = MMC_CAP_4_BIT_DATA;
721 		if (caps & SDHCI_CAN_DO_HISPD)
722 			slot->host.caps |= MMC_CAP_HSPEED;
723 		/* Decide if we have usable DMA. */
724 		if (caps & SDHCI_CAN_DO_DMA)
725 			slot->opt |= SDHCI_HAVE_DMA;
726 		if (class == PCIC_BASEPERIPH &&
727 		    subclass == PCIS_BASEPERIPH_SDHC &&
728 		    progif != PCI_SDHCI_IFDMA)
729 			slot->opt &= ~SDHCI_HAVE_DMA;
730 		if (sc->quirks & SDHCI_QUIRK_BROKEN_DMA)
731 			slot->opt &= ~SDHCI_HAVE_DMA;
732 		if (sc->quirks & SDHCI_QUIRK_FORCE_DMA)
733 			slot->opt |= SDHCI_HAVE_DMA;
734 
735 		if (bootverbose) {
736 			slot_printf(slot, "%uMHz%s 4bits%s%s%s %s\n",
737 			    slot->max_clk / 1000000,
738 			    (caps & SDHCI_CAN_DO_HISPD) ? " HS" : "",
739 			    (caps & SDHCI_CAN_VDD_330) ? " 3.3V" : "",
740 			    (caps & SDHCI_CAN_VDD_300) ? " 3.0V" : "",
741 			    (caps & SDHCI_CAN_VDD_180) ? " 1.8V" : "",
742 			    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO");
743 			sdhci_dumpregs(slot);
744 		}
745 
746 		TASK_INIT(&slot->card_task, 0, sdhci_card_task, slot);
747 		callout_init(&slot->card_callout, 1);
748 		sc->num_slots++;
749 	}
750 	device_printf(dev, "%d slot(s) allocated\n", sc->num_slots);
751 	/* Activate the interrupt */
752 	err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
753 	    NULL, sdhci_intr, sc, &sc->intrhand);
754 	if (err)
755 		device_printf(dev, "Can't setup IRQ\n");
756 	pci_enable_busmaster(dev);
757 	/* Process cards detection. */
758 	for (i = 0; i < sc->num_slots; i++) {
759 		struct sdhci_slot *slot = &sc->slots[i];
760 
761 		sdhci_card_task(slot, 0);
762 	}
763 
764 	return (0);
765 }
766 
767 static int
768 sdhci_detach(device_t dev)
769 {
770 	struct sdhci_softc *sc = device_get_softc(dev);
771 	int i;
772 
773 	bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
774 	bus_release_resource(dev, SYS_RES_IRQ,
775 	    sc->irq_rid, sc->irq_res);
776 
777 	for (i = 0; i < sc->num_slots; i++) {
778 		struct sdhci_slot *slot = &sc->slots[i];
779 		device_t d;
780 
781 		callout_drain(&slot->card_callout);
782 		taskqueue_drain(taskqueue_swi_giant, &slot->card_task);
783 
784 		SDHCI_LOCK(slot);
785 		d = slot->dev;
786 		slot->dev = NULL;
787 		SDHCI_UNLOCK(slot);
788 		if (d != NULL)
789 			device_delete_child(dev, d);
790 
791 		SDHCI_LOCK(slot);
792 		sdhci_reset(slot, SDHCI_RESET_ALL);
793 		SDHCI_UNLOCK(slot);
794 		bus_dmamap_unload(slot->dmatag, slot->dmamap);
795 		bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
796 		bus_dma_tag_destroy(slot->dmatag);
797 		bus_release_resource(dev, SYS_RES_MEMORY,
798 		    slot->mem_rid, slot->mem_res);
799 		SDHCI_LOCK_DESTROY(slot);
800 	}
801 	return (0);
802 }
803 
804 static int
805 sdhci_update_ios(device_t brdev, device_t reqdev)
806 {
807 	struct sdhci_slot *slot = device_get_ivars(reqdev);
808 	struct mmc_ios *ios = &slot->host.ios;
809 
810 	SDHCI_LOCK(slot);
811 	/* Do full reset on bus power down to clear from any state. */
812 	if (ios->power_mode == power_off) {
813 		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
814 		sdhci_init(slot);
815 	}
816 	/* Configure the bus. */
817 	sdhci_set_clock(slot, ios->clock);
818 	sdhci_set_power(slot, (ios->power_mode == power_off)?0:ios->vdd);
819 	if (ios->bus_width == bus_width_4)
820 		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
821 	else
822 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
823 	if (ios->timing == bus_timing_hs)
824 		slot->hostctrl |= SDHCI_CTRL_HISPD;
825 	else
826 		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
827 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
828 	/* Some controllers like reset after bus changes. */
829 	if(slot->sc->quirks & SDHCI_QUIRK_RESET_ON_IOS)
830 		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
831 
832 	SDHCI_UNLOCK(slot);
833 	return (0);
834 }
835 
836 static void
837 sdhci_set_transfer_mode(struct sdhci_slot *slot,
838 	struct mmc_data *data)
839 {
840 	uint16_t mode;
841 
842 	if (data == NULL)
843 		return;
844 
845 	mode = SDHCI_TRNS_BLK_CNT_EN;
846 	if (data->len > 512)
847 		mode |= SDHCI_TRNS_MULTI;
848 	if (data->flags & MMC_DATA_READ)
849 		mode |= SDHCI_TRNS_READ;
850 	if (slot->req->stop)
851 		mode |= SDHCI_TRNS_ACMD12;
852 	if (slot->flags & SDHCI_USE_DMA)
853 		mode |= SDHCI_TRNS_DMA;
854 
855 	WR2(slot, SDHCI_TRANSFER_MODE, mode);
856 }
857 
858 static void
859 sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
860 {
861 	struct mmc_request *req = slot->req;
862 	int flags, timeout;
863 	uint32_t mask, state;
864 
865 	slot->curcmd = cmd;
866 	slot->cmd_done = 0;
867 
868 	cmd->error = MMC_ERR_NONE;
869 
870 	/* This flags combination is not supported by controller. */
871 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
872 		slot_printf(slot, "Unsupported response type!\n");
873 		cmd->error = MMC_ERR_FAILED;
874 		slot->req = NULL;
875 		slot->curcmd = NULL;
876 		req->done(req);
877 		return;
878 	}
879 
880 	/* Read controller present state. */
881 	state = RD4(slot, SDHCI_PRESENT_STATE);
882 	/* Do not issue command if there is no card. */
883 	if ((state & SDHCI_CARD_PRESENT) == 0) {
884 		cmd->error = MMC_ERR_FAILED;
885 		slot->req = NULL;
886 		slot->curcmd = NULL;
887 		req->done(req);
888 		return;
889 	}
890 	/* Always wait for free CMD bus. */
891 	mask = SDHCI_CMD_INHIBIT;
892 	/* Wait for free DAT if we have data or busy signal. */
893 	if (cmd->data || (cmd->flags & MMC_RSP_BUSY))
894 		mask |= SDHCI_DAT_INHIBIT;
895 	/* We shouldn't wait for DAT for stop commands. */
896 	if (cmd == slot->req->stop)
897 		mask &= ~SDHCI_DAT_INHIBIT;
898 	/* Wait for bus no more then 10 ms. */
899 	timeout = 10;
900 	while (state & mask) {
901 		if (timeout == 0) {
902 			slot_printf(slot, "Controller never released "
903 			    "inhibit bit(s).\n");
904 			sdhci_dumpregs(slot);
905 			cmd->error = MMC_ERR_FAILED;
906 			slot->req = NULL;
907 			slot->curcmd = NULL;
908 			req->done(req);
909 			return;
910 		}
911 		timeout--;
912 		DELAY(1000);
913 		state = RD4(slot, SDHCI_PRESENT_STATE);
914 	}
915 
916 	/* Prepare command flags. */
917 	if (!(cmd->flags & MMC_RSP_PRESENT))
918 		flags = SDHCI_CMD_RESP_NONE;
919 	else if (cmd->flags & MMC_RSP_136)
920 		flags = SDHCI_CMD_RESP_LONG;
921 	else if (cmd->flags & MMC_RSP_BUSY)
922 		flags = SDHCI_CMD_RESP_SHORT_BUSY;
923 	else
924 		flags = SDHCI_CMD_RESP_SHORT;
925 	if (cmd->flags & MMC_RSP_CRC)
926 		flags |= SDHCI_CMD_CRC;
927 	if (cmd->flags & MMC_RSP_OPCODE)
928 		flags |= SDHCI_CMD_INDEX;
929 	if (cmd->data)
930 		flags |= SDHCI_CMD_DATA;
931 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
932 		flags |= SDHCI_CMD_TYPE_ABORT;
933 	/* Prepare data. */
934 	sdhci_start_data(slot, cmd->data);
935 	/*
936 	 * Interrupt aggregation: To reduce total number of interrupts
937 	 * group response interrupt with data interrupt when possible.
938 	 * If there going to be data interrupt, mask response one.
939 	 */
940 	if (slot->data_done == 0) {
941 		WR4(slot, SDHCI_SIGNAL_ENABLE,
942 		    slot->intmask &= ~SDHCI_INT_RESPONSE);
943 	}
944 	/* Set command argument. */
945 	WR4(slot, SDHCI_ARGUMENT, cmd->arg);
946 	/* Set data transfer mode. */
947 	sdhci_set_transfer_mode(slot, cmd->data);
948 	/* Set command flags. */
949 	WR1(slot, SDHCI_COMMAND_FLAGS, flags);
950 	/* Start command. */
951 	WR1(slot, SDHCI_COMMAND, cmd->opcode);
952 }
953 
954 static void
955 sdhci_finish_command(struct sdhci_slot *slot)
956 {
957 	int i;
958 
959 	slot->cmd_done = 1;
960 	/* Interrupt aggregation: Restore command interrupt.
961 	 * Main restore point for the case when command interrupt
962 	 * happened first. */
963 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= SDHCI_INT_RESPONSE);
964 	/* In case of error - reset host and return. */
965 	if (slot->curcmd->error) {
966 		sdhci_reset(slot, SDHCI_RESET_CMD);
967 		sdhci_reset(slot, SDHCI_RESET_DATA);
968 		sdhci_start(slot);
969 		return;
970 	}
971 	/* If command has response - fetch it. */
972 	if (slot->curcmd->flags & MMC_RSP_PRESENT) {
973 		if (slot->curcmd->flags & MMC_RSP_136) {
974 			/* CRC is stripped so we need one byte shift. */
975 			uint8_t extra = 0;
976 			for (i = 0; i < 4; i++) {
977 				uint32_t val = RD4(slot, SDHCI_RESPONSE + i * 4);
978 				slot->curcmd->resp[3 - i] = (val << 8) + extra;
979 				extra = val >> 24;
980 			}
981 		} else
982 			slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE);
983 	}
984 	/* If data ready - finish. */
985 	if (slot->data_done)
986 		sdhci_start(slot);
987 }
988 
989 static void
990 sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data)
991 {
992 	uint32_t target_timeout, current_timeout;
993 	uint8_t div;
994 
995 	if (data == NULL && (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
996 		slot->data_done = 1;
997 		return;
998 	}
999 
1000 	slot->data_done = 0;
1001 
1002 	/* Calculate and set data timeout.*/
1003 	/* XXX: We should have this from mmc layer, now assume 1 sec. */
1004 	target_timeout = 1000000;
1005 	div = 0;
1006 	current_timeout = (1 << 13) * 1000 / slot->timeout_clk;
1007 	while (current_timeout < target_timeout) {
1008 		div++;
1009 		current_timeout <<= 1;
1010 		if (div >= 0xF)
1011 			break;
1012 	}
1013 	/* Compensate for an off-by-one error in the CaFe chip.*/
1014 	if (slot->sc->quirks & SDHCI_QUIRK_INCR_TIMEOUT_CONTROL)
1015 		div++;
1016 	if (div >= 0xF) {
1017 		slot_printf(slot, "Timeout too large!\n");
1018 		div = 0xE;
1019 	}
1020 	WR1(slot, SDHCI_TIMEOUT_CONTROL, div);
1021 
1022 	if (data == NULL)
1023 		return;
1024 
1025 	/* Use DMA if possible. */
1026 	if ((slot->opt & SDHCI_HAVE_DMA))
1027 		slot->flags |= SDHCI_USE_DMA;
1028 	/* If data is small, broken DMA may return zeroes instead of data, */
1029 	if ((slot->sc->quirks & SDHCI_QUIRK_BROKEN_TIMINGS) &&
1030 	    (data->len <= 512))
1031 		slot->flags &= ~SDHCI_USE_DMA;
1032 	/* Some controllers require even block sizes. */
1033 	if ((slot->sc->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE) &&
1034 	    ((data->len) & 0x3))
1035 		slot->flags &= ~SDHCI_USE_DMA;
1036 	/* Load DMA buffer. */
1037 	if (slot->flags & SDHCI_USE_DMA) {
1038 		if (data->flags & MMC_DATA_READ)
1039 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_PREREAD);
1040 		else {
1041 			memcpy(slot->dmamem, data->data,
1042 			    (data->len < DMA_BLOCK_SIZE)?data->len:DMA_BLOCK_SIZE);
1043 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_PREWRITE);
1044 		}
1045 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
1046 		/* Interrupt aggregation: Mask border interrupt
1047 		 * for the last page and unmask else. */
1048 		if (data->len == DMA_BLOCK_SIZE)
1049 			slot->intmask &= ~SDHCI_INT_DMA_END;
1050 		else
1051 			slot->intmask |= SDHCI_INT_DMA_END;
1052 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1053 	}
1054 	/* Current data offset for both PIO and DMA. */
1055 	slot->offset = 0;
1056 	/* Set block size and request IRQ on 4K border. */
1057 	WR2(slot, SDHCI_BLOCK_SIZE,
1058 	    SDHCI_MAKE_BLKSZ(DMA_BOUNDARY, (data->len < 512)?data->len:512));
1059 	/* Set block count. */
1060 	WR2(slot, SDHCI_BLOCK_COUNT, (data->len + 511) / 512);
1061 }
1062 
1063 static void
1064 sdhci_finish_data(struct sdhci_slot *slot)
1065 {
1066 	struct mmc_data *data = slot->curcmd->data;
1067 
1068 	slot->data_done = 1;
1069 	/* Interrupt aggregation: Restore command interrupt.
1070 	 * Auxillary restore point for the case when data interrupt
1071 	 * happened first. */
1072 	if (!slot->cmd_done) {
1073 		WR4(slot, SDHCI_SIGNAL_ENABLE,
1074 		    slot->intmask |= SDHCI_INT_RESPONSE);
1075 	}
1076 	/* Unload rest of data from DMA buffer. */
1077 	if (slot->flags & SDHCI_USE_DMA) {
1078 		if (data->flags & MMC_DATA_READ) {
1079 			size_t left = data->len - slot->offset;
1080 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_POSTREAD);
1081 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
1082 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1083 		} else
1084 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_POSTWRITE);
1085 	}
1086 	/* If there was error - reset the host. */
1087 	if (slot->curcmd->error) {
1088 		sdhci_reset(slot, SDHCI_RESET_CMD);
1089 		sdhci_reset(slot, SDHCI_RESET_DATA);
1090 		sdhci_start(slot);
1091 		return;
1092 	}
1093 	/* If we already have command response - finish. */
1094 	if (slot->cmd_done)
1095 		sdhci_start(slot);
1096 }
1097 
1098 static void
1099 sdhci_start(struct sdhci_slot *slot)
1100 {
1101 	struct mmc_request *req;
1102 
1103 	req = slot->req;
1104 	if (req == NULL)
1105 		return;
1106 
1107 	if (!(slot->flags & CMD_STARTED)) {
1108 		slot->flags |= CMD_STARTED;
1109 		sdhci_start_command(slot, req->cmd);
1110 		return;
1111 	}
1112 /* 	We don't need this until using Auto-CMD12 feature
1113 	if (!(slot->flags & STOP_STARTED) && req->stop) {
1114 		slot->flags |= STOP_STARTED;
1115 		sdhci_start_command(slot, req->stop);
1116 		return;
1117 	}
1118 */
1119 	if (req->cmd->error) {
1120 		if (bootverbose) {
1121 			slot_printf(slot,
1122 			    "Command error %d (opcode %u arg %u flags %u "
1123 			    "dlen %u dflags %u)\n",
1124 			    req->cmd->error, req->cmd->opcode, req->cmd->arg,
1125 			    req->cmd->flags,
1126 			    (req->cmd->data)?(u_int)req->cmd->data->len:0,
1127 			    (req->cmd->data)?(u_int)req->cmd->data->flags:0);
1128 		}
1129 	} else if (slot->sc->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST) {
1130 		sdhci_reset(slot, SDHCI_RESET_CMD);
1131 		sdhci_reset(slot, SDHCI_RESET_DATA);
1132 	}
1133 
1134 	/* We must be done -- bad idea to do this while locked? */
1135 	slot->req = NULL;
1136 	slot->curcmd = NULL;
1137 	req->done(req);
1138 }
1139 
1140 static int
1141 sdhci_request(device_t brdev, device_t reqdev, struct mmc_request *req)
1142 {
1143 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1144 
1145 	SDHCI_LOCK(slot);
1146 	if (slot->req != NULL) {
1147 		SDHCI_UNLOCK(slot);
1148 		return (EBUSY);
1149 	}
1150 /*	printf("%s cmd op %u arg %u flags %u data %ju\n", __func__,
1151     	    req->cmd->opcode, req->cmd->arg, req->cmd->flags,
1152     	    (req->cmd->data)?req->cmd->data->len:0); */
1153 	slot->req = req;
1154 	slot->flags = 0;
1155 	sdhci_start(slot);
1156 	SDHCI_UNLOCK(slot);
1157 	return (0);
1158 }
1159 
1160 static int
1161 sdhci_get_ro(device_t brdev, device_t reqdev)
1162 {
1163 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1164 	uint32_t val;
1165 
1166 	SDHCI_LOCK(slot);
1167 	val = RD4(slot, SDHCI_PRESENT_STATE);
1168 	SDHCI_UNLOCK(slot);
1169 	return (!(val & SDHCI_WRITE_PROTECT));
1170 }
1171 
1172 static int
1173 sdhci_acquire_host(device_t brdev, device_t reqdev)
1174 {
1175 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1176 	int err = 0;
1177 
1178 	SDHCI_LOCK(slot);
1179 	while (slot->bus_busy)
1180 		msleep(slot, &slot->mtx, PZERO, "sdhciah", hz / 5);
1181 	slot->bus_busy++;
1182 	/* Activate led. */
1183 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl |= SDHCI_CTRL_LED);
1184 	SDHCI_UNLOCK(slot);
1185 	return (err);
1186 }
1187 
1188 static int
1189 sdhci_release_host(device_t brdev, device_t reqdev)
1190 {
1191 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1192 
1193 	SDHCI_LOCK(slot);
1194 	/* Deactivate led. */
1195 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl &= ~SDHCI_CTRL_LED);
1196 	slot->bus_busy--;
1197 	wakeup(slot);
1198 	SDHCI_UNLOCK(slot);
1199 	return (0);
1200 }
1201 
1202 static void
1203 sdhci_cmd_irq(struct sdhci_slot *slot, uint32_t intmask)
1204 {
1205 
1206 	if (!slot->curcmd) {
1207 		slot_printf(slot, "Got command interrupt 0x%08x, but "
1208 		    "there is no active command.\n", intmask);
1209 		sdhci_dumpregs(slot);
1210 		return;
1211 	}
1212 	if (intmask & SDHCI_INT_TIMEOUT)
1213 		slot->curcmd->error = MMC_ERR_TIMEOUT;
1214 	else if (intmask & SDHCI_INT_CRC)
1215 		slot->curcmd->error = MMC_ERR_BADCRC;
1216 	else if (intmask & (SDHCI_INT_END_BIT | SDHCI_INT_INDEX))
1217 		slot->curcmd->error = MMC_ERR_FIFO;
1218 
1219 	sdhci_finish_command(slot);
1220 }
1221 
1222 static void
1223 sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask)
1224 {
1225 
1226 	if (!slot->curcmd) {
1227 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1228 		    "there is no active command.\n", intmask);
1229 		sdhci_dumpregs(slot);
1230 		return;
1231 	}
1232 	if (slot->curcmd->data == NULL &&
1233 	    (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
1234 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1235 		    "there is no active data operation.\n",
1236 		    intmask);
1237 		sdhci_dumpregs(slot);
1238 		return;
1239 	}
1240 	if (intmask & SDHCI_INT_DATA_TIMEOUT)
1241 		slot->curcmd->error = MMC_ERR_TIMEOUT;
1242 	else if (intmask & (SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_END_BIT))
1243 		slot->curcmd->error = MMC_ERR_BADCRC;
1244 	if (slot->curcmd->data == NULL &&
1245 	    (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
1246 	    SDHCI_INT_DMA_END))) {
1247 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1248 		    "there is busy-only command.\n", intmask);
1249 		sdhci_dumpregs(slot);
1250 		slot->curcmd->error = MMC_ERR_INVALID;
1251 	}
1252 	if (slot->curcmd->error) {
1253 		/* No need to continue after any error. */
1254 		sdhci_finish_data(slot);
1255 		return;
1256 	}
1257 
1258 	/* Handle PIO interrupt. */
1259 	if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
1260 		sdhci_transfer_pio(slot);
1261 	/* Handle DMA border. */
1262 	if (intmask & SDHCI_INT_DMA_END) {
1263 		struct mmc_data *data = slot->curcmd->data;
1264 		size_t left;
1265 
1266 		/* Unload DMA buffer... */
1267 		left = data->len - slot->offset;
1268 		if (data->flags & MMC_DATA_READ) {
1269 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1270 			    BUS_DMASYNC_POSTREAD);
1271 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
1272 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1273 		} else {
1274 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1275 			    BUS_DMASYNC_POSTWRITE);
1276 		}
1277 		/* ... and reload it again. */
1278 		slot->offset += DMA_BLOCK_SIZE;
1279 		left = data->len - slot->offset;
1280 		if (data->flags & MMC_DATA_READ) {
1281 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1282 			    BUS_DMASYNC_PREREAD);
1283 		} else {
1284 			memcpy(slot->dmamem, (u_char*)data->data + slot->offset,
1285 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1286 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1287 			    BUS_DMASYNC_PREWRITE);
1288 		}
1289 		/* Interrupt aggregation: Mask border interrupt
1290 		 * for the last page. */
1291 		if (left == DMA_BLOCK_SIZE) {
1292 			slot->intmask &= ~SDHCI_INT_DMA_END;
1293 			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1294 		}
1295 		/* Restart DMA. */
1296 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
1297 	}
1298 	/* We have got all data. */
1299 	if (intmask & SDHCI_INT_DATA_END)
1300 		sdhci_finish_data(slot);
1301 }
1302 
1303 static void
1304 sdhci_acmd_irq(struct sdhci_slot *slot)
1305 {
1306 	uint16_t err;
1307 
1308 	err = RD4(slot, SDHCI_ACMD12_ERR);
1309 	if (!slot->curcmd) {
1310 		slot_printf(slot, "Got AutoCMD12 error 0x%04x, but "
1311 		    "there is no active command.\n", err);
1312 		sdhci_dumpregs(slot);
1313 		return;
1314 	}
1315 	slot_printf(slot, "Got AutoCMD12 error 0x%04x\n", err);
1316 	sdhci_reset(slot, SDHCI_RESET_CMD);
1317 }
1318 
1319 static void
1320 sdhci_intr(void *arg)
1321 {
1322 	struct sdhci_softc *sc = (struct sdhci_softc *)arg;
1323 	int i;
1324 
1325 	for (i = 0; i < sc->num_slots; i++) {
1326 		struct sdhci_slot *slot = &sc->slots[i];
1327 		uint32_t intmask;
1328 
1329 		SDHCI_LOCK(slot);
1330 		/* Read slot interrupt status. */
1331 		intmask = RD4(slot, SDHCI_INT_STATUS);
1332 		if (intmask == 0 || intmask == 0xffffffff) {
1333 			SDHCI_UNLOCK(slot);
1334 			continue;
1335 		}
1336 /*
1337 		slot_printf(slot, "got interrupt %x\n", intmask);
1338 */
1339 		/* Handle card presence interrupts. */
1340 		if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
1341 			WR4(slot, SDHCI_INT_STATUS, intmask &
1342 			    (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE));
1343 
1344 			if (intmask & SDHCI_INT_CARD_REMOVE) {
1345 				if (bootverbose)
1346 					slot_printf(slot, "Card removed\n");
1347 				callout_stop(&slot->card_callout);
1348 				taskqueue_enqueue(taskqueue_swi_giant,
1349 				    &slot->card_task);
1350 			}
1351 			if (intmask & SDHCI_INT_CARD_INSERT) {
1352 				if (bootverbose)
1353 					slot_printf(slot, "Card inserted\n");
1354 				callout_reset(&slot->card_callout, hz / 2,
1355 				    sdhci_card_delay, slot);
1356 			}
1357 			intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
1358 		}
1359 		/* Handle command interrupts. */
1360 		if (intmask & SDHCI_INT_CMD_MASK) {
1361 			WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_CMD_MASK);
1362 			sdhci_cmd_irq(slot, intmask & SDHCI_INT_CMD_MASK);
1363 		}
1364 		/* Handle data interrupts. */
1365 		if (intmask & SDHCI_INT_DATA_MASK) {
1366 			WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_DATA_MASK);
1367 			sdhci_data_irq(slot, intmask & SDHCI_INT_DATA_MASK);
1368 		}
1369 		/* Handle AutoCMD12 error interrupt. */
1370 		if (intmask & SDHCI_INT_ACMD12ERR) {
1371 			WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR);
1372 			sdhci_acmd_irq(slot);
1373 		}
1374 		intmask &= ~(SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK);
1375 		intmask &= ~SDHCI_INT_ACMD12ERR;
1376 		intmask &= ~SDHCI_INT_ERROR;
1377 		/* Handle bus power interrupt. */
1378 		if (intmask & SDHCI_INT_BUS_POWER) {
1379 			WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER);
1380 			slot_printf(slot,
1381 			    "Card is consuming too much power!\n");
1382 			intmask &= ~SDHCI_INT_BUS_POWER;
1383 		}
1384 		/* The rest is unknown. */
1385 		if (intmask) {
1386 			WR4(slot, SDHCI_INT_STATUS, intmask);
1387 			slot_printf(slot, "Unexpected interrupt 0x%08x.\n",
1388 			    intmask);
1389 			sdhci_dumpregs(slot);
1390 		}
1391 
1392 		SDHCI_UNLOCK(slot);
1393 	}
1394 }
1395 
1396 static int
1397 sdhci_read_ivar(device_t bus, device_t child, int which, u_char *result)
1398 {
1399 	struct sdhci_slot *slot = device_get_ivars(child);
1400 
1401 	switch (which) {
1402 	default:
1403 		return (EINVAL);
1404 	case MMCBR_IVAR_BUS_MODE:
1405 		*(int *)result = slot->host.ios.bus_mode;
1406 		break;
1407 	case MMCBR_IVAR_BUS_WIDTH:
1408 		*(int *)result = slot->host.ios.bus_width;
1409 		break;
1410 	case MMCBR_IVAR_CHIP_SELECT:
1411 		*(int *)result = slot->host.ios.chip_select;
1412 		break;
1413 	case MMCBR_IVAR_CLOCK:
1414 		*(int *)result = slot->host.ios.clock;
1415 		break;
1416 	case MMCBR_IVAR_F_MIN:
1417 		*(int *)result = slot->host.f_min;
1418 		break;
1419 	case MMCBR_IVAR_F_MAX:
1420 		*(int *)result = slot->host.f_max;
1421 		break;
1422 	case MMCBR_IVAR_HOST_OCR:
1423 		*(int *)result = slot->host.host_ocr;
1424 		break;
1425 	case MMCBR_IVAR_MODE:
1426 		*(int *)result = slot->host.mode;
1427 		break;
1428 	case MMCBR_IVAR_OCR:
1429 		*(int *)result = slot->host.ocr;
1430 		break;
1431 	case MMCBR_IVAR_POWER_MODE:
1432 		*(int *)result = slot->host.ios.power_mode;
1433 		break;
1434 	case MMCBR_IVAR_VDD:
1435 		*(int *)result = slot->host.ios.vdd;
1436 		break;
1437 	case MMCBR_IVAR_CAPS:
1438 		*(int *)result = slot->host.caps;
1439 		break;
1440 	case MMCBR_IVAR_TIMING:
1441 		*(int *)result = slot->host.ios.timing;
1442 		break;
1443 	case MMCBR_IVAR_MAX_DATA:
1444 		*(int *)result = 65535;
1445 		break;
1446 	}
1447 	return (0);
1448 }
1449 
1450 static int
1451 sdhci_write_ivar(device_t bus, device_t child, int which, uintptr_t value)
1452 {
1453 	struct sdhci_slot *slot = device_get_ivars(child);
1454 
1455 	switch (which) {
1456 	default:
1457 		return (EINVAL);
1458 	case MMCBR_IVAR_BUS_MODE:
1459 		slot->host.ios.bus_mode = value;
1460 		break;
1461 	case MMCBR_IVAR_BUS_WIDTH:
1462 		slot->host.ios.bus_width = value;
1463 		break;
1464 	case MMCBR_IVAR_CHIP_SELECT:
1465 		slot->host.ios.chip_select = value;
1466 		break;
1467 	case MMCBR_IVAR_CLOCK:
1468 		if (value > 0) {
1469 			uint32_t clock = slot->max_clk;
1470 			int i;
1471 
1472 			for (i = 0; i < 8; i++) {
1473 				if (clock <= value)
1474 					break;
1475 				clock >>= 1;
1476 			}
1477 			slot->host.ios.clock = clock;
1478 		} else
1479 			slot->host.ios.clock = 0;
1480 		break;
1481 	case MMCBR_IVAR_MODE:
1482 		slot->host.mode = value;
1483 		break;
1484 	case MMCBR_IVAR_OCR:
1485 		slot->host.ocr = value;
1486 		break;
1487 	case MMCBR_IVAR_POWER_MODE:
1488 		slot->host.ios.power_mode = value;
1489 		break;
1490 	case MMCBR_IVAR_VDD:
1491 		slot->host.ios.vdd = value;
1492 		break;
1493 	case MMCBR_IVAR_TIMING:
1494 		slot->host.ios.timing = value;
1495 		break;
1496 	case MMCBR_IVAR_CAPS:
1497 	case MMCBR_IVAR_HOST_OCR:
1498 	case MMCBR_IVAR_F_MIN:
1499 	case MMCBR_IVAR_F_MAX:
1500 	case MMCBR_IVAR_MAX_DATA:
1501 		return (EINVAL);
1502 	}
1503 	return (0);
1504 }
1505 
1506 static device_method_t sdhci_methods[] = {
1507 	/* device_if */
1508 	DEVMETHOD(device_probe, sdhci_probe),
1509 	DEVMETHOD(device_attach, sdhci_attach),
1510 	DEVMETHOD(device_detach, sdhci_detach),
1511 
1512 	/* Bus interface */
1513 	DEVMETHOD(bus_read_ivar,	sdhci_read_ivar),
1514 	DEVMETHOD(bus_write_ivar,	sdhci_write_ivar),
1515 
1516 	/* mmcbr_if */
1517 	DEVMETHOD(mmcbr_update_ios, sdhci_update_ios),
1518 	DEVMETHOD(mmcbr_request, sdhci_request),
1519 	DEVMETHOD(mmcbr_get_ro, sdhci_get_ro),
1520 	DEVMETHOD(mmcbr_acquire_host, sdhci_acquire_host),
1521 	DEVMETHOD(mmcbr_release_host, sdhci_release_host),
1522 
1523 	{0, 0},
1524 };
1525 
1526 static driver_t sdhci_driver = {
1527 	"sdhci",
1528 	sdhci_methods,
1529 	sizeof(struct sdhci_softc),
1530 };
1531 static devclass_t sdhci_devclass;
1532 
1533 
1534 DRIVER_MODULE(sdhci, pci, sdhci_driver, sdhci_devclass, 0, 0);
1535