xref: /freebsd/sys/dev/sdhci/sdhci.c (revision bb15ca603fa442c72dde3f3cb8b46db6970e3950)
1 /*-
2  * Copyright (c) 2008 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28 
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/bus.h>
32 #include <sys/conf.h>
33 #include <sys/kernel.h>
34 #include <sys/lock.h>
35 #include <sys/module.h>
36 #include <sys/mutex.h>
37 #include <sys/resource.h>
38 #include <sys/rman.h>
39 #include <sys/sysctl.h>
40 #include <sys/taskqueue.h>
41 
42 #include <dev/pci/pcireg.h>
43 #include <dev/pci/pcivar.h>
44 
45 #include <machine/bus.h>
46 #include <machine/resource.h>
47 #include <machine/stdarg.h>
48 
49 #include <dev/mmc/bridge.h>
50 #include <dev/mmc/mmcreg.h>
51 #include <dev/mmc/mmcbrvar.h>
52 
53 #include "mmcbr_if.h"
54 #include "sdhci.h"
55 
56 #define DMA_BLOCK_SIZE	4096
57 #define DMA_BOUNDARY	0	/* DMA reload every 4K */
58 
59 /* Controller doesn't honor resets unless we touch the clock register */
60 #define SDHCI_QUIRK_CLOCK_BEFORE_RESET			(1<<0)
61 /* Controller really supports DMA */
62 #define SDHCI_QUIRK_FORCE_DMA				(1<<1)
63 /* Controller has unusable DMA engine */
64 #define SDHCI_QUIRK_BROKEN_DMA				(1<<2)
65 /* Controller doesn't like to be reset when there is no card inserted. */
66 #define SDHCI_QUIRK_NO_CARD_NO_RESET			(1<<3)
67 /* Controller has flaky internal state so reset it on each ios change */
68 #define SDHCI_QUIRK_RESET_ON_IOS			(1<<4)
69 /* Controller can only DMA chunk sizes that are a multiple of 32 bits */
70 #define SDHCI_QUIRK_32BIT_DMA_SIZE			(1<<5)
71 /* Controller needs to be reset after each request to stay stable */
72 #define SDHCI_QUIRK_RESET_AFTER_REQUEST			(1<<6)
73 /* Controller has an off-by-one issue with timeout value */
74 #define SDHCI_QUIRK_INCR_TIMEOUT_CONTROL		(1<<7)
75 /* Controller has broken read timings */
76 #define SDHCI_QUIRK_BROKEN_TIMINGS			(1<<8)
77 
78 static const struct sdhci_device {
79 	uint32_t	model;
80 	uint16_t	subvendor;
81 	char		*desc;
82 	u_int		quirks;
83 } sdhci_devices[] = {
84 	{ 0x08221180, 	0xffff,	"RICOH R5C822 SD",
85 	    SDHCI_QUIRK_FORCE_DMA },
86 	{ 0xe8221180, 	0xffff,	"RICOH SD",
87 	    SDHCI_QUIRK_FORCE_DMA },
88 	{ 0x8034104c, 	0xffff, "TI XX21/XX11 SD",
89 	    SDHCI_QUIRK_FORCE_DMA },
90 	{ 0x05501524, 	0xffff, "ENE CB712 SD",
91 	    SDHCI_QUIRK_BROKEN_TIMINGS },
92 	{ 0x05511524, 	0xffff, "ENE CB712 SD 2",
93 	    SDHCI_QUIRK_BROKEN_TIMINGS },
94 	{ 0x07501524, 	0xffff, "ENE CB714 SD",
95 	    SDHCI_QUIRK_RESET_ON_IOS |
96 	    SDHCI_QUIRK_BROKEN_TIMINGS },
97 	{ 0x07511524, 	0xffff, "ENE CB714 SD 2",
98 	    SDHCI_QUIRK_RESET_ON_IOS |
99 	    SDHCI_QUIRK_BROKEN_TIMINGS },
100 	{ 0x410111ab, 	0xffff, "Marvell CaFe SD",
101 	    SDHCI_QUIRK_INCR_TIMEOUT_CONTROL },
102 	{ 0x2381197B, 	0xffff,	"JMicron JMB38X SD",
103 	    SDHCI_QUIRK_32BIT_DMA_SIZE |
104 	    SDHCI_QUIRK_RESET_AFTER_REQUEST },
105 	{ 0,		0xffff,	NULL,
106 	    0 }
107 };
108 
109 struct sdhci_softc;
110 
111 struct sdhci_slot {
112 	struct sdhci_softc	*sc;
113 	device_t	dev;		/* Slot device */
114 	u_char		num;		/* Slot number */
115 	u_char		opt;		/* Slot options */
116 #define SDHCI_HAVE_DMA		1
117 	uint32_t	max_clk;	/* Max possible freq */
118 	uint32_t	timeout_clk;	/* Timeout freq */
119 	struct resource	*mem_res;	/* Memory resource */
120 	int		mem_rid;
121 	bus_dma_tag_t 	dmatag;
122 	bus_dmamap_t 	dmamap;
123 	u_char		*dmamem;
124 	bus_addr_t	paddr;		/* DMA buffer address */
125 	struct task	card_task;	/* Card presence check task */
126 	struct callout	card_callout;	/* Card insert delay callout */
127 	struct mmc_host host;		/* Host parameters */
128 	struct mmc_request *req;	/* Current request */
129 	struct mmc_command *curcmd;	/* Current command of current request */
130 
131 	uint32_t	intmask;	/* Current interrupt mask */
132 	uint32_t	clock;		/* Current clock freq. */
133 	size_t		offset;		/* Data buffer offset */
134 	uint8_t		hostctrl;	/* Current host control register */
135 	u_char		power;		/* Current power */
136 	u_char		bus_busy;	/* Bus busy status */
137 	u_char		cmd_done;	/* CMD command part done flag */
138 	u_char		data_done;	/* DAT command part done flag */
139 	u_char		flags;		/* Request execution flags */
140 #define CMD_STARTED		1
141 #define STOP_STARTED		2
142 #define SDHCI_USE_DMA		4	/* Use DMA for this req. */
143 	struct mtx	mtx;		/* Slot mutex */
144 };
145 
146 struct sdhci_softc {
147 	device_t	dev;		/* Controller device */
148 	u_int		quirks;		/* Chip specific quirks */
149 	struct resource *irq_res;	/* IRQ resource */
150 	int 		irq_rid;
151 	void 		*intrhand;	/* Interrupt handle */
152 
153 	int		num_slots;	/* Number of slots on this controller */
154 	struct sdhci_slot slots[6];
155 };
156 
157 static SYSCTL_NODE(_hw, OID_AUTO, sdhci, CTLFLAG_RD, 0, "sdhci driver");
158 
159 int	sdhci_debug;
160 TUNABLE_INT("hw.sdhci.debug", &sdhci_debug);
161 SYSCTL_INT(_hw_sdhci, OID_AUTO, debug, CTLFLAG_RW, &sdhci_debug, 0, "Debug level");
162 
163 static inline uint8_t
164 RD1(struct sdhci_slot *slot, bus_size_t off)
165 {
166 	bus_barrier(slot->mem_res, 0, 0xFF,
167 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
168 	return bus_read_1(slot->mem_res, off);
169 }
170 
171 static inline void
172 WR1(struct sdhci_slot *slot, bus_size_t off, uint8_t val)
173 {
174 	bus_barrier(slot->mem_res, 0, 0xFF,
175 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
176 	bus_write_1(slot->mem_res, off, val);
177 }
178 
179 static inline uint16_t
180 RD2(struct sdhci_slot *slot, bus_size_t off)
181 {
182 	bus_barrier(slot->mem_res, 0, 0xFF,
183 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
184 	return bus_read_2(slot->mem_res, off);
185 }
186 
187 static inline void
188 WR2(struct sdhci_slot *slot, bus_size_t off, uint16_t val)
189 {
190 	bus_barrier(slot->mem_res, 0, 0xFF,
191 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
192 	bus_write_2(slot->mem_res, off, val);
193 }
194 
195 static inline uint32_t
196 RD4(struct sdhci_slot *slot, bus_size_t off)
197 {
198 	bus_barrier(slot->mem_res, 0, 0xFF,
199 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
200 	return bus_read_4(slot->mem_res, off);
201 }
202 
203 static inline void
204 WR4(struct sdhci_slot *slot, bus_size_t off, uint32_t val)
205 {
206 	bus_barrier(slot->mem_res, 0, 0xFF,
207 	    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
208 	bus_write_4(slot->mem_res, off, val);
209 }
210 
211 /* bus entry points */
212 static int sdhci_probe(device_t dev);
213 static int sdhci_attach(device_t dev);
214 static int sdhci_detach(device_t dev);
215 static void sdhci_intr(void *);
216 
217 static void sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock);
218 static void sdhci_start(struct sdhci_slot *slot);
219 static void sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data);
220 
221 static void sdhci_card_task(void *, int);
222 
223 /* helper routines */
224 #define SDHCI_LOCK(_slot)		mtx_lock(&(_slot)->mtx)
225 #define	SDHCI_UNLOCK(_slot)		mtx_unlock(&(_slot)->mtx)
226 #define SDHCI_LOCK_INIT(_slot) \
227 	mtx_init(&_slot->mtx, "SD slot mtx", "sdhci", MTX_DEF)
228 #define SDHCI_LOCK_DESTROY(_slot)	mtx_destroy(&_slot->mtx);
229 #define SDHCI_ASSERT_LOCKED(_slot)	mtx_assert(&_slot->mtx, MA_OWNED);
230 #define SDHCI_ASSERT_UNLOCKED(_slot)	mtx_assert(&_slot->mtx, MA_NOTOWNED);
231 
232 static int
233 slot_printf(struct sdhci_slot *slot, const char * fmt, ...)
234 {
235 	va_list ap;
236 	int retval;
237 
238     	retval = printf("%s-slot%d: ",
239 	    device_get_nameunit(slot->sc->dev), slot->num);
240 
241 	va_start(ap, fmt);
242 	retval += vprintf(fmt, ap);
243 	va_end(ap);
244 	return (retval);
245 }
246 
247 static void
248 sdhci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
249 {
250 	if (error != 0) {
251 		printf("getaddr: error %d\n", error);
252 		return;
253 	}
254 	*(bus_addr_t *)arg = segs[0].ds_addr;
255 }
256 
257 static void
258 sdhci_dumpregs(struct sdhci_slot *slot)
259 {
260 	slot_printf(slot,
261 	    "============== REGISTER DUMP ==============\n");
262 
263 	slot_printf(slot, "Sys addr: 0x%08x | Version:  0x%08x\n",
264 	    RD4(slot, SDHCI_DMA_ADDRESS), RD2(slot, SDHCI_HOST_VERSION));
265 	slot_printf(slot, "Blk size: 0x%08x | Blk cnt:  0x%08x\n",
266 	    RD2(slot, SDHCI_BLOCK_SIZE), RD2(slot, SDHCI_BLOCK_COUNT));
267 	slot_printf(slot, "Argument: 0x%08x | Trn mode: 0x%08x\n",
268 	    RD4(slot, SDHCI_ARGUMENT), RD2(slot, SDHCI_TRANSFER_MODE));
269 	slot_printf(slot, "Present:  0x%08x | Host ctl: 0x%08x\n",
270 	    RD4(slot, SDHCI_PRESENT_STATE), RD1(slot, SDHCI_HOST_CONTROL));
271 	slot_printf(slot, "Power:    0x%08x | Blk gap:  0x%08x\n",
272 	    RD1(slot, SDHCI_POWER_CONTROL), RD1(slot, SDHCI_BLOCK_GAP_CONTROL));
273 	slot_printf(slot, "Wake-up:  0x%08x | Clock:    0x%08x\n",
274 	    RD1(slot, SDHCI_WAKE_UP_CONTROL), RD2(slot, SDHCI_CLOCK_CONTROL));
275 	slot_printf(slot, "Timeout:  0x%08x | Int stat: 0x%08x\n",
276 	    RD1(slot, SDHCI_TIMEOUT_CONTROL), RD4(slot, SDHCI_INT_STATUS));
277 	slot_printf(slot, "Int enab: 0x%08x | Sig enab: 0x%08x\n",
278 	    RD4(slot, SDHCI_INT_ENABLE), RD4(slot, SDHCI_SIGNAL_ENABLE));
279 	slot_printf(slot, "AC12 err: 0x%08x | Slot int: 0x%08x\n",
280 	    RD2(slot, SDHCI_ACMD12_ERR), RD2(slot, SDHCI_SLOT_INT_STATUS));
281 	slot_printf(slot, "Caps:     0x%08x | Max curr: 0x%08x\n",
282 	    RD4(slot, SDHCI_CAPABILITIES), RD4(slot, SDHCI_MAX_CURRENT));
283 
284 	slot_printf(slot,
285 	    "===========================================\n");
286 }
287 
288 static void
289 sdhci_reset(struct sdhci_slot *slot, uint8_t mask)
290 {
291 	int timeout;
292 	uint8_t res;
293 
294 	if (slot->sc->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
295 		if (!(RD4(slot, SDHCI_PRESENT_STATE) &
296 			SDHCI_CARD_PRESENT))
297 			return;
298 	}
299 
300 	/* Some controllers need this kick or reset won't work. */
301 	if ((mask & SDHCI_RESET_ALL) == 0 &&
302 	    (slot->sc->quirks & SDHCI_QUIRK_CLOCK_BEFORE_RESET)) {
303 		uint32_t clock;
304 
305 		/* This is to force an update */
306 		clock = slot->clock;
307 		slot->clock = 0;
308 		sdhci_set_clock(slot, clock);
309 	}
310 
311 	WR1(slot, SDHCI_SOFTWARE_RESET, mask);
312 
313 	if (mask & SDHCI_RESET_ALL) {
314 		slot->clock = 0;
315 		slot->power = 0;
316 	}
317 
318 	/* Wait max 100 ms */
319 	timeout = 100;
320 	/* Controller clears the bits when it's done */
321 	while ((res = RD1(slot, SDHCI_SOFTWARE_RESET)) & mask) {
322 		if (timeout == 0) {
323 			slot_printf(slot,
324 			    "Reset 0x%x never completed - 0x%x.\n",
325 			    (int)mask, (int)res);
326 			sdhci_dumpregs(slot);
327 			return;
328 		}
329 		timeout--;
330 		DELAY(1000);
331 	}
332 }
333 
334 static void
335 sdhci_init(struct sdhci_slot *slot)
336 {
337 
338 	sdhci_reset(slot, SDHCI_RESET_ALL);
339 
340 	/* Enable interrupts. */
341 	slot->intmask = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
342 	    SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_TIMEOUT | SDHCI_INT_INDEX |
343 	    SDHCI_INT_END_BIT | SDHCI_INT_CRC | SDHCI_INT_TIMEOUT |
344 	    SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT |
345 	    SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
346 	    SDHCI_INT_DMA_END | SDHCI_INT_DATA_END | SDHCI_INT_RESPONSE |
347 	    SDHCI_INT_ACMD12ERR;
348 	WR4(slot, SDHCI_INT_ENABLE, slot->intmask);
349 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
350 }
351 
352 static void
353 sdhci_set_clock(struct sdhci_slot *slot, uint32_t clock)
354 {
355 	uint32_t res;
356 	uint16_t clk;
357 	int timeout;
358 
359 	if (clock == slot->clock)
360 		return;
361 	slot->clock = clock;
362 
363 	/* Turn off the clock. */
364 	WR2(slot, SDHCI_CLOCK_CONTROL, 0);
365 	/* If no clock requested - left it so. */
366 	if (clock == 0)
367 		return;
368 	/* Looking for highest freq <= clock. */
369 	res = slot->max_clk;
370 	for (clk = 1; clk < 256; clk <<= 1) {
371 		if (res <= clock)
372 			break;
373 		res >>= 1;
374 	}
375 	/* Divider 1:1 is 0x00, 2:1 is 0x01, 256:1 is 0x80 ... */
376 	clk >>= 1;
377 	/* Now we have got divider, set it. */
378 	clk <<= SDHCI_DIVIDER_SHIFT;
379 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
380 	/* Enable clock. */
381 	clk |= SDHCI_CLOCK_INT_EN;
382 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
383 	/* Wait up to 10 ms until it stabilize. */
384 	timeout = 10;
385 	while (!((clk = RD2(slot, SDHCI_CLOCK_CONTROL))
386 		& SDHCI_CLOCK_INT_STABLE)) {
387 		if (timeout == 0) {
388 			slot_printf(slot,
389 			    "Internal clock never stabilised.\n");
390 			sdhci_dumpregs(slot);
391 			return;
392 		}
393 		timeout--;
394 		DELAY(1000);
395 	}
396 	/* Pass clock signal to the bus. */
397 	clk |= SDHCI_CLOCK_CARD_EN;
398 	WR2(slot, SDHCI_CLOCK_CONTROL, clk);
399 }
400 
401 static void
402 sdhci_set_power(struct sdhci_slot *slot, u_char power)
403 {
404 	uint8_t pwr;
405 
406 	if (slot->power == power)
407 		return;
408 	slot->power = power;
409 
410 	/* Turn off the power. */
411 	pwr = 0;
412 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
413 	/* If power down requested - left it so. */
414 	if (power == 0)
415 		return;
416 	/* Set voltage. */
417 	switch (1 << power) {
418 	case MMC_OCR_LOW_VOLTAGE:
419 		pwr |= SDHCI_POWER_180;
420 		break;
421 	case MMC_OCR_290_300:
422 	case MMC_OCR_300_310:
423 		pwr |= SDHCI_POWER_300;
424 		break;
425 	case MMC_OCR_320_330:
426 	case MMC_OCR_330_340:
427 		pwr |= SDHCI_POWER_330;
428 		break;
429 	}
430 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
431 	/* Turn on the power. */
432 	pwr |= SDHCI_POWER_ON;
433 	WR1(slot, SDHCI_POWER_CONTROL, pwr);
434 }
435 
436 static void
437 sdhci_read_block_pio(struct sdhci_slot *slot)
438 {
439 	uint32_t data;
440 	char *buffer;
441 	size_t left;
442 
443 	buffer = slot->curcmd->data->data;
444 	buffer += slot->offset;
445 	/* Transfer one block at a time. */
446 	left = min(512, slot->curcmd->data->len - slot->offset);
447 	slot->offset += left;
448 
449 	/* If we are too fast, broken controllers return zeroes. */
450 	if (slot->sc->quirks & SDHCI_QUIRK_BROKEN_TIMINGS)
451 		DELAY(10);
452 	/* Handle unalligned and alligned buffer cases. */
453 	if ((intptr_t)buffer & 3) {
454 		while (left > 3) {
455 			data = RD4(slot, SDHCI_BUFFER);
456 			buffer[0] = data;
457 			buffer[1] = (data >> 8);
458 			buffer[2] = (data >> 16);
459 			buffer[3] = (data >> 24);
460 			buffer += 4;
461 			left -= 4;
462 		}
463 	} else {
464 		bus_read_multi_stream_4(slot->mem_res, SDHCI_BUFFER,
465 		    (uint32_t *)buffer, left >> 2);
466 		left &= 3;
467 	}
468 	/* Handle uneven size case. */
469 	if (left > 0) {
470 		data = RD4(slot, SDHCI_BUFFER);
471 		while (left > 0) {
472 			*(buffer++) = data;
473 			data >>= 8;
474 			left--;
475 		}
476 	}
477 }
478 
479 static void
480 sdhci_write_block_pio(struct sdhci_slot *slot)
481 {
482 	uint32_t data = 0;
483 	char *buffer;
484 	size_t left;
485 
486 	buffer = slot->curcmd->data->data;
487 	buffer += slot->offset;
488 	/* Transfer one block at a time. */
489 	left = min(512, slot->curcmd->data->len - slot->offset);
490 	slot->offset += left;
491 
492 	/* Handle unalligned and alligned buffer cases. */
493 	if ((intptr_t)buffer & 3) {
494 		while (left > 3) {
495 			data = buffer[0] +
496 			    (buffer[1] << 8) +
497 			    (buffer[2] << 16) +
498 			    (buffer[3] << 24);
499 			left -= 4;
500 			buffer += 4;
501 			WR4(slot, SDHCI_BUFFER, data);
502 		}
503 	} else {
504 		bus_write_multi_stream_4(slot->mem_res, SDHCI_BUFFER,
505 		    (uint32_t *)buffer, left >> 2);
506 		left &= 3;
507 	}
508 	/* Handle uneven size case. */
509 	if (left > 0) {
510 		while (left > 0) {
511 			data <<= 8;
512 			data += *(buffer++);
513 			left--;
514 		}
515 		WR4(slot, SDHCI_BUFFER, data);
516 	}
517 }
518 
519 static void
520 sdhci_transfer_pio(struct sdhci_slot *slot)
521 {
522 
523 	/* Read as many blocks as possible. */
524 	if (slot->curcmd->data->flags & MMC_DATA_READ) {
525 		while (RD4(slot, SDHCI_PRESENT_STATE) &
526 		    SDHCI_DATA_AVAILABLE) {
527 			sdhci_read_block_pio(slot);
528 			if (slot->offset >= slot->curcmd->data->len)
529 				break;
530 		}
531 	} else {
532 		while (RD4(slot, SDHCI_PRESENT_STATE) &
533 		    SDHCI_SPACE_AVAILABLE) {
534 			sdhci_write_block_pio(slot);
535 			if (slot->offset >= slot->curcmd->data->len)
536 				break;
537 		}
538 	}
539 }
540 
541 static void
542 sdhci_card_delay(void *arg)
543 {
544 	struct sdhci_slot *slot = arg;
545 
546 	taskqueue_enqueue(taskqueue_swi_giant, &slot->card_task);
547 }
548 
549 static void
550 sdhci_card_task(void *arg, int pending)
551 {
552 	struct sdhci_slot *slot = arg;
553 
554 	SDHCI_LOCK(slot);
555 	if (RD4(slot, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT) {
556 		if (slot->dev == NULL) {
557 			/* If card is present - attach mmc bus. */
558 			slot->dev = device_add_child(slot->sc->dev, "mmc", -1);
559 			device_set_ivars(slot->dev, slot);
560 			SDHCI_UNLOCK(slot);
561 			device_probe_and_attach(slot->dev);
562 		} else
563 			SDHCI_UNLOCK(slot);
564 	} else {
565 		if (slot->dev != NULL) {
566 			/* If no card present - detach mmc bus. */
567 			device_t d = slot->dev;
568 			slot->dev = NULL;
569 			SDHCI_UNLOCK(slot);
570 			device_delete_child(slot->sc->dev, d);
571 		} else
572 			SDHCI_UNLOCK(slot);
573 	}
574 }
575 
576 static int
577 sdhci_probe(device_t dev)
578 {
579 	uint32_t model;
580 	uint16_t subvendor;
581 	uint8_t class, subclass;
582 	int i, result;
583 
584 	model = (uint32_t)pci_get_device(dev) << 16;
585 	model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
586 	subvendor = pci_get_subvendor(dev);
587 	class = pci_get_class(dev);
588 	subclass = pci_get_subclass(dev);
589 
590 	result = ENXIO;
591 	for (i = 0; sdhci_devices[i].model != 0; i++) {
592 		if (sdhci_devices[i].model == model &&
593 		    (sdhci_devices[i].subvendor == 0xffff ||
594 		    sdhci_devices[i].subvendor == subvendor)) {
595 			device_set_desc(dev, sdhci_devices[i].desc);
596 			result = BUS_PROBE_DEFAULT;
597 			break;
598 		}
599 	}
600 	if (result == ENXIO && class == PCIC_BASEPERIPH &&
601 	    subclass == PCIS_BASEPERIPH_SDHC) {
602 		device_set_desc(dev, "Generic SD HCI");
603 		result = BUS_PROBE_GENERIC;
604 	}
605 
606 	return (result);
607 }
608 
609 static int
610 sdhci_attach(device_t dev)
611 {
612 	struct sdhci_softc *sc = device_get_softc(dev);
613 	uint32_t model;
614 	uint16_t subvendor;
615 	uint8_t class, subclass, progif;
616 	int err, slots, bar, i;
617 
618 	sc->dev = dev;
619 	model = (uint32_t)pci_get_device(dev) << 16;
620 	model |= (uint32_t)pci_get_vendor(dev) & 0x0000ffff;
621 	subvendor = pci_get_subvendor(dev);
622 	class = pci_get_class(dev);
623 	subclass = pci_get_subclass(dev);
624 	progif = pci_get_progif(dev);
625 	/* Apply chip specific quirks. */
626 	for (i = 0; sdhci_devices[i].model != 0; i++) {
627 		if (sdhci_devices[i].model == model &&
628 		    (sdhci_devices[i].subvendor == 0xffff ||
629 		    sdhci_devices[i].subvendor == subvendor)) {
630 			sc->quirks = sdhci_devices[i].quirks;
631 			break;
632 		}
633 	}
634 	/* Read slots info from PCI registers. */
635 	slots = pci_read_config(dev, PCI_SLOT_INFO, 1);
636 	bar = PCI_SLOT_INFO_FIRST_BAR(slots);
637 	slots = PCI_SLOT_INFO_SLOTS(slots);
638 	if (slots > 6 || bar > 5) {
639 		device_printf(dev, "Incorrect slots information (%d, %d).\n",
640 		    slots, bar);
641 		return (EINVAL);
642 	}
643 	/* Allocate IRQ. */
644 	sc->irq_rid = 0;
645 	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid,
646 	    RF_SHAREABLE | RF_ACTIVE);
647 	if (sc->irq_res == NULL) {
648 		device_printf(dev, "Can't allocate IRQ\n");
649 		return (ENOMEM);
650 	}
651 	/* Scan all slots. */
652 	for (i = 0; i < slots; i++) {
653 		struct sdhci_slot *slot = &sc->slots[sc->num_slots];
654 		uint32_t caps;
655 
656 		SDHCI_LOCK_INIT(slot);
657 		slot->sc = sc;
658 		slot->num = sc->num_slots;
659 		/* Allocate memory. */
660 		slot->mem_rid = PCIR_BAR(bar + i);
661 		slot->mem_res = bus_alloc_resource(dev,
662 		    SYS_RES_MEMORY, &slot->mem_rid, 0ul, ~0ul, 0x100, RF_ACTIVE);
663 		if (slot->mem_res == NULL) {
664 			device_printf(dev, "Can't allocate memory\n");
665 			SDHCI_LOCK_DESTROY(slot);
666 			continue;
667 		}
668 		/* Allocate DMA tag. */
669 		err = bus_dma_tag_create(bus_get_dma_tag(dev),
670 		    DMA_BLOCK_SIZE, 0, BUS_SPACE_MAXADDR_32BIT,
671 		    BUS_SPACE_MAXADDR, NULL, NULL,
672 		    DMA_BLOCK_SIZE, 1, DMA_BLOCK_SIZE,
673 		    BUS_DMA_ALLOCNOW, NULL, NULL,
674 		    &slot->dmatag);
675 		if (err != 0) {
676 			device_printf(dev, "Can't create DMA tag\n");
677 			SDHCI_LOCK_DESTROY(slot);
678 			continue;
679 		}
680 		/* Allocate DMA memory. */
681 		err = bus_dmamem_alloc(slot->dmatag, (void **)&slot->dmamem,
682 		    BUS_DMA_NOWAIT, &slot->dmamap);
683 		if (err != 0) {
684 			device_printf(dev, "Can't alloc DMA memory\n");
685 			SDHCI_LOCK_DESTROY(slot);
686 			continue;
687 		}
688 		/* Map the memory. */
689 		err = bus_dmamap_load(slot->dmatag, slot->dmamap,
690 		    (void *)slot->dmamem, DMA_BLOCK_SIZE,
691 		    sdhci_getaddr, &slot->paddr, 0);
692 		if (err != 0 || slot->paddr == 0) {
693 			device_printf(dev, "Can't load DMA memory\n");
694 			SDHCI_LOCK_DESTROY(slot);
695 			continue;
696 		}
697 		/* Initialize slot. */
698 		sdhci_init(slot);
699 		caps = RD4(slot, SDHCI_CAPABILITIES);
700 		/* Calculate base clock frequency. */
701 		slot->max_clk =
702 			(caps & SDHCI_CLOCK_BASE_MASK) >> SDHCI_CLOCK_BASE_SHIFT;
703 		if (slot->max_clk == 0) {
704 			device_printf(dev, "Hardware doesn't specify base clock "
705 			    "frequency.\n");
706 		}
707 		slot->max_clk *= 1000000;
708 		/* Calculate timeout clock frequency. */
709 		slot->timeout_clk =
710 			(caps & SDHCI_TIMEOUT_CLK_MASK) >> SDHCI_TIMEOUT_CLK_SHIFT;
711 		if (slot->timeout_clk == 0) {
712 			device_printf(dev, "Hardware doesn't specify timeout clock "
713 			    "frequency.\n");
714 		}
715 		if (caps & SDHCI_TIMEOUT_CLK_UNIT)
716 			slot->timeout_clk *= 1000;
717 
718 		slot->host.f_min = slot->max_clk / 256;
719 		slot->host.f_max = slot->max_clk;
720 		slot->host.host_ocr = 0;
721 		if (caps & SDHCI_CAN_VDD_330)
722 		    slot->host.host_ocr |= MMC_OCR_320_330 | MMC_OCR_330_340;
723 		if (caps & SDHCI_CAN_VDD_300)
724 		    slot->host.host_ocr |= MMC_OCR_290_300 | MMC_OCR_300_310;
725 		if (caps & SDHCI_CAN_VDD_180)
726 		    slot->host.host_ocr |= MMC_OCR_LOW_VOLTAGE;
727 		if (slot->host.host_ocr == 0) {
728 			device_printf(dev, "Hardware doesn't report any "
729 			    "support voltages.\n");
730 		}
731 		slot->host.caps = MMC_CAP_4_BIT_DATA;
732 		if (caps & SDHCI_CAN_DO_HISPD)
733 			slot->host.caps |= MMC_CAP_HSPEED;
734 		/* Decide if we have usable DMA. */
735 		if (caps & SDHCI_CAN_DO_DMA)
736 			slot->opt |= SDHCI_HAVE_DMA;
737 		if (class == PCIC_BASEPERIPH &&
738 		    subclass == PCIS_BASEPERIPH_SDHC &&
739 		    progif != PCI_SDHCI_IFDMA)
740 			slot->opt &= ~SDHCI_HAVE_DMA;
741 		if (sc->quirks & SDHCI_QUIRK_BROKEN_DMA)
742 			slot->opt &= ~SDHCI_HAVE_DMA;
743 		if (sc->quirks & SDHCI_QUIRK_FORCE_DMA)
744 			slot->opt |= SDHCI_HAVE_DMA;
745 
746 		if (bootverbose || sdhci_debug) {
747 			slot_printf(slot, "%uMHz%s 4bits%s%s%s %s\n",
748 			    slot->max_clk / 1000000,
749 			    (caps & SDHCI_CAN_DO_HISPD) ? " HS" : "",
750 			    (caps & SDHCI_CAN_VDD_330) ? " 3.3V" : "",
751 			    (caps & SDHCI_CAN_VDD_300) ? " 3.0V" : "",
752 			    (caps & SDHCI_CAN_VDD_180) ? " 1.8V" : "",
753 			    (slot->opt & SDHCI_HAVE_DMA) ? "DMA" : "PIO");
754 			sdhci_dumpregs(slot);
755 		}
756 
757 		TASK_INIT(&slot->card_task, 0, sdhci_card_task, slot);
758 		callout_init(&slot->card_callout, 1);
759 		sc->num_slots++;
760 	}
761 	device_printf(dev, "%d slot(s) allocated\n", sc->num_slots);
762 	/* Activate the interrupt */
763 	err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
764 	    NULL, sdhci_intr, sc, &sc->intrhand);
765 	if (err)
766 		device_printf(dev, "Can't setup IRQ\n");
767 	pci_enable_busmaster(dev);
768 	/* Process cards detection. */
769 	for (i = 0; i < sc->num_slots; i++) {
770 		struct sdhci_slot *slot = &sc->slots[i];
771 
772 		sdhci_card_task(slot, 0);
773 	}
774 
775 	return (0);
776 }
777 
778 static int
779 sdhci_detach(device_t dev)
780 {
781 	struct sdhci_softc *sc = device_get_softc(dev);
782 	int i;
783 
784 	bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
785 	bus_release_resource(dev, SYS_RES_IRQ,
786 	    sc->irq_rid, sc->irq_res);
787 
788 	for (i = 0; i < sc->num_slots; i++) {
789 		struct sdhci_slot *slot = &sc->slots[i];
790 		device_t d;
791 
792 		callout_drain(&slot->card_callout);
793 		taskqueue_drain(taskqueue_swi_giant, &slot->card_task);
794 
795 		SDHCI_LOCK(slot);
796 		d = slot->dev;
797 		slot->dev = NULL;
798 		SDHCI_UNLOCK(slot);
799 		if (d != NULL)
800 			device_delete_child(dev, d);
801 
802 		SDHCI_LOCK(slot);
803 		sdhci_reset(slot, SDHCI_RESET_ALL);
804 		SDHCI_UNLOCK(slot);
805 		bus_dmamap_unload(slot->dmatag, slot->dmamap);
806 		bus_dmamem_free(slot->dmatag, slot->dmamem, slot->dmamap);
807 		bus_dma_tag_destroy(slot->dmatag);
808 		bus_release_resource(dev, SYS_RES_MEMORY,
809 		    slot->mem_rid, slot->mem_res);
810 		SDHCI_LOCK_DESTROY(slot);
811 	}
812 	return (0);
813 }
814 
815 static int
816 sdhci_suspend(device_t dev)
817 {
818 	struct sdhci_softc *sc = device_get_softc(dev);
819 	int i, err;
820 
821 	err = bus_generic_suspend(dev);
822 	if (err)
823 		return (err);
824 	for (i = 0; i < sc->num_slots; i++)
825 		sdhci_reset(&sc->slots[i], SDHCI_RESET_ALL);
826 	return (0);
827 }
828 
829 static int
830 sdhci_resume(device_t dev)
831 {
832 	struct sdhci_softc *sc = device_get_softc(dev);
833 	int i;
834 
835 	for (i = 0; i < sc->num_slots; i++)
836 		sdhci_init(&sc->slots[i]);
837 	return (bus_generic_resume(dev));
838 }
839 
840 static int
841 sdhci_update_ios(device_t brdev, device_t reqdev)
842 {
843 	struct sdhci_slot *slot = device_get_ivars(reqdev);
844 	struct mmc_ios *ios = &slot->host.ios;
845 
846 	SDHCI_LOCK(slot);
847 	/* Do full reset on bus power down to clear from any state. */
848 	if (ios->power_mode == power_off) {
849 		WR4(slot, SDHCI_SIGNAL_ENABLE, 0);
850 		sdhci_init(slot);
851 	}
852 	/* Configure the bus. */
853 	sdhci_set_clock(slot, ios->clock);
854 	sdhci_set_power(slot, (ios->power_mode == power_off)?0:ios->vdd);
855 	if (ios->bus_width == bus_width_4)
856 		slot->hostctrl |= SDHCI_CTRL_4BITBUS;
857 	else
858 		slot->hostctrl &= ~SDHCI_CTRL_4BITBUS;
859 	if (ios->timing == bus_timing_hs)
860 		slot->hostctrl |= SDHCI_CTRL_HISPD;
861 	else
862 		slot->hostctrl &= ~SDHCI_CTRL_HISPD;
863 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl);
864 	/* Some controllers like reset after bus changes. */
865 	if(slot->sc->quirks & SDHCI_QUIRK_RESET_ON_IOS)
866 		sdhci_reset(slot, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
867 
868 	SDHCI_UNLOCK(slot);
869 	return (0);
870 }
871 
872 static void
873 sdhci_set_transfer_mode(struct sdhci_slot *slot,
874 	struct mmc_data *data)
875 {
876 	uint16_t mode;
877 
878 	if (data == NULL)
879 		return;
880 
881 	mode = SDHCI_TRNS_BLK_CNT_EN;
882 	if (data->len > 512)
883 		mode |= SDHCI_TRNS_MULTI;
884 	if (data->flags & MMC_DATA_READ)
885 		mode |= SDHCI_TRNS_READ;
886 	if (slot->req->stop)
887 		mode |= SDHCI_TRNS_ACMD12;
888 	if (slot->flags & SDHCI_USE_DMA)
889 		mode |= SDHCI_TRNS_DMA;
890 
891 	WR2(slot, SDHCI_TRANSFER_MODE, mode);
892 }
893 
894 static void
895 sdhci_start_command(struct sdhci_slot *slot, struct mmc_command *cmd)
896 {
897 	struct mmc_request *req = slot->req;
898 	int flags, timeout;
899 	uint32_t mask, state;
900 
901 	slot->curcmd = cmd;
902 	slot->cmd_done = 0;
903 
904 	cmd->error = MMC_ERR_NONE;
905 
906 	/* This flags combination is not supported by controller. */
907 	if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
908 		slot_printf(slot, "Unsupported response type!\n");
909 		cmd->error = MMC_ERR_FAILED;
910 		slot->req = NULL;
911 		slot->curcmd = NULL;
912 		req->done(req);
913 		return;
914 	}
915 
916 	/* Read controller present state. */
917 	state = RD4(slot, SDHCI_PRESENT_STATE);
918 	/* Do not issue command if there is no card, clock or power.
919 	 * Controller will not detect timeout without clock active. */
920 	if ((state & SDHCI_CARD_PRESENT) == 0 ||
921 	    slot->power == 0 ||
922 	    slot->clock == 0) {
923 		cmd->error = MMC_ERR_FAILED;
924 		slot->req = NULL;
925 		slot->curcmd = NULL;
926 		req->done(req);
927 		return;
928 	}
929 	/* Always wait for free CMD bus. */
930 	mask = SDHCI_CMD_INHIBIT;
931 	/* Wait for free DAT if we have data or busy signal. */
932 	if (cmd->data || (cmd->flags & MMC_RSP_BUSY))
933 		mask |= SDHCI_DAT_INHIBIT;
934 	/* We shouldn't wait for DAT for stop commands. */
935 	if (cmd == slot->req->stop)
936 		mask &= ~SDHCI_DAT_INHIBIT;
937 	/* Wait for bus no more then 10 ms. */
938 	timeout = 10;
939 	while (state & mask) {
940 		if (timeout == 0) {
941 			slot_printf(slot, "Controller never released "
942 			    "inhibit bit(s).\n");
943 			sdhci_dumpregs(slot);
944 			cmd->error = MMC_ERR_FAILED;
945 			slot->req = NULL;
946 			slot->curcmd = NULL;
947 			req->done(req);
948 			return;
949 		}
950 		timeout--;
951 		DELAY(1000);
952 		state = RD4(slot, SDHCI_PRESENT_STATE);
953 	}
954 
955 	/* Prepare command flags. */
956 	if (!(cmd->flags & MMC_RSP_PRESENT))
957 		flags = SDHCI_CMD_RESP_NONE;
958 	else if (cmd->flags & MMC_RSP_136)
959 		flags = SDHCI_CMD_RESP_LONG;
960 	else if (cmd->flags & MMC_RSP_BUSY)
961 		flags = SDHCI_CMD_RESP_SHORT_BUSY;
962 	else
963 		flags = SDHCI_CMD_RESP_SHORT;
964 	if (cmd->flags & MMC_RSP_CRC)
965 		flags |= SDHCI_CMD_CRC;
966 	if (cmd->flags & MMC_RSP_OPCODE)
967 		flags |= SDHCI_CMD_INDEX;
968 	if (cmd->data)
969 		flags |= SDHCI_CMD_DATA;
970 	if (cmd->opcode == MMC_STOP_TRANSMISSION)
971 		flags |= SDHCI_CMD_TYPE_ABORT;
972 	/* Prepare data. */
973 	sdhci_start_data(slot, cmd->data);
974 	/*
975 	 * Interrupt aggregation: To reduce total number of interrupts
976 	 * group response interrupt with data interrupt when possible.
977 	 * If there going to be data interrupt, mask response one.
978 	 */
979 	if (slot->data_done == 0) {
980 		WR4(slot, SDHCI_SIGNAL_ENABLE,
981 		    slot->intmask &= ~SDHCI_INT_RESPONSE);
982 	}
983 	/* Set command argument. */
984 	WR4(slot, SDHCI_ARGUMENT, cmd->arg);
985 	/* Set data transfer mode. */
986 	sdhci_set_transfer_mode(slot, cmd->data);
987 	/* Set command flags. */
988 	WR1(slot, SDHCI_COMMAND_FLAGS, flags);
989 	/* Start command. */
990 	WR1(slot, SDHCI_COMMAND, cmd->opcode);
991 }
992 
993 static void
994 sdhci_finish_command(struct sdhci_slot *slot)
995 {
996 	int i;
997 
998 	slot->cmd_done = 1;
999 	/* Interrupt aggregation: Restore command interrupt.
1000 	 * Main restore point for the case when command interrupt
1001 	 * happened first. */
1002 	WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask |= SDHCI_INT_RESPONSE);
1003 	/* In case of error - reset host and return. */
1004 	if (slot->curcmd->error) {
1005 		sdhci_reset(slot, SDHCI_RESET_CMD);
1006 		sdhci_reset(slot, SDHCI_RESET_DATA);
1007 		sdhci_start(slot);
1008 		return;
1009 	}
1010 	/* If command has response - fetch it. */
1011 	if (slot->curcmd->flags & MMC_RSP_PRESENT) {
1012 		if (slot->curcmd->flags & MMC_RSP_136) {
1013 			/* CRC is stripped so we need one byte shift. */
1014 			uint8_t extra = 0;
1015 			for (i = 0; i < 4; i++) {
1016 				uint32_t val = RD4(slot, SDHCI_RESPONSE + i * 4);
1017 				slot->curcmd->resp[3 - i] = (val << 8) + extra;
1018 				extra = val >> 24;
1019 			}
1020 		} else
1021 			slot->curcmd->resp[0] = RD4(slot, SDHCI_RESPONSE);
1022 	}
1023 	/* If data ready - finish. */
1024 	if (slot->data_done)
1025 		sdhci_start(slot);
1026 }
1027 
1028 static void
1029 sdhci_start_data(struct sdhci_slot *slot, struct mmc_data *data)
1030 {
1031 	uint32_t target_timeout, current_timeout;
1032 	uint8_t div;
1033 
1034 	if (data == NULL && (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
1035 		slot->data_done = 1;
1036 		return;
1037 	}
1038 
1039 	slot->data_done = 0;
1040 
1041 	/* Calculate and set data timeout.*/
1042 	/* XXX: We should have this from mmc layer, now assume 1 sec. */
1043 	target_timeout = 1000000;
1044 	div = 0;
1045 	current_timeout = (1 << 13) * 1000 / slot->timeout_clk;
1046 	while (current_timeout < target_timeout) {
1047 		div++;
1048 		current_timeout <<= 1;
1049 		if (div >= 0xF)
1050 			break;
1051 	}
1052 	/* Compensate for an off-by-one error in the CaFe chip.*/
1053 	if (slot->sc->quirks & SDHCI_QUIRK_INCR_TIMEOUT_CONTROL)
1054 		div++;
1055 	if (div >= 0xF) {
1056 		slot_printf(slot, "Timeout too large!\n");
1057 		div = 0xE;
1058 	}
1059 	WR1(slot, SDHCI_TIMEOUT_CONTROL, div);
1060 
1061 	if (data == NULL)
1062 		return;
1063 
1064 	/* Use DMA if possible. */
1065 	if ((slot->opt & SDHCI_HAVE_DMA))
1066 		slot->flags |= SDHCI_USE_DMA;
1067 	/* If data is small, broken DMA may return zeroes instead of data, */
1068 	if ((slot->sc->quirks & SDHCI_QUIRK_BROKEN_TIMINGS) &&
1069 	    (data->len <= 512))
1070 		slot->flags &= ~SDHCI_USE_DMA;
1071 	/* Some controllers require even block sizes. */
1072 	if ((slot->sc->quirks & SDHCI_QUIRK_32BIT_DMA_SIZE) &&
1073 	    ((data->len) & 0x3))
1074 		slot->flags &= ~SDHCI_USE_DMA;
1075 	/* Load DMA buffer. */
1076 	if (slot->flags & SDHCI_USE_DMA) {
1077 		if (data->flags & MMC_DATA_READ)
1078 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_PREREAD);
1079 		else {
1080 			memcpy(slot->dmamem, data->data,
1081 			    (data->len < DMA_BLOCK_SIZE)?data->len:DMA_BLOCK_SIZE);
1082 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_PREWRITE);
1083 		}
1084 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
1085 		/* Interrupt aggregation: Mask border interrupt
1086 		 * for the last page and unmask else. */
1087 		if (data->len == DMA_BLOCK_SIZE)
1088 			slot->intmask &= ~SDHCI_INT_DMA_END;
1089 		else
1090 			slot->intmask |= SDHCI_INT_DMA_END;
1091 		WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1092 	}
1093 	/* Current data offset for both PIO and DMA. */
1094 	slot->offset = 0;
1095 	/* Set block size and request IRQ on 4K border. */
1096 	WR2(slot, SDHCI_BLOCK_SIZE,
1097 	    SDHCI_MAKE_BLKSZ(DMA_BOUNDARY, (data->len < 512)?data->len:512));
1098 	/* Set block count. */
1099 	WR2(slot, SDHCI_BLOCK_COUNT, (data->len + 511) / 512);
1100 }
1101 
1102 static void
1103 sdhci_finish_data(struct sdhci_slot *slot)
1104 {
1105 	struct mmc_data *data = slot->curcmd->data;
1106 
1107 	slot->data_done = 1;
1108 	/* Interrupt aggregation: Restore command interrupt.
1109 	 * Auxillary restore point for the case when data interrupt
1110 	 * happened first. */
1111 	if (!slot->cmd_done) {
1112 		WR4(slot, SDHCI_SIGNAL_ENABLE,
1113 		    slot->intmask |= SDHCI_INT_RESPONSE);
1114 	}
1115 	/* Unload rest of data from DMA buffer. */
1116 	if (slot->flags & SDHCI_USE_DMA) {
1117 		if (data->flags & MMC_DATA_READ) {
1118 			size_t left = data->len - slot->offset;
1119 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_POSTREAD);
1120 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
1121 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1122 		} else
1123 			bus_dmamap_sync(slot->dmatag, slot->dmamap, BUS_DMASYNC_POSTWRITE);
1124 	}
1125 	/* If there was error - reset the host. */
1126 	if (slot->curcmd->error) {
1127 		sdhci_reset(slot, SDHCI_RESET_CMD);
1128 		sdhci_reset(slot, SDHCI_RESET_DATA);
1129 		sdhci_start(slot);
1130 		return;
1131 	}
1132 	/* If we already have command response - finish. */
1133 	if (slot->cmd_done)
1134 		sdhci_start(slot);
1135 }
1136 
1137 static void
1138 sdhci_start(struct sdhci_slot *slot)
1139 {
1140 	struct mmc_request *req;
1141 
1142 	req = slot->req;
1143 	if (req == NULL)
1144 		return;
1145 
1146 	if (!(slot->flags & CMD_STARTED)) {
1147 		slot->flags |= CMD_STARTED;
1148 		sdhci_start_command(slot, req->cmd);
1149 		return;
1150 	}
1151 /* 	We don't need this until using Auto-CMD12 feature
1152 	if (!(slot->flags & STOP_STARTED) && req->stop) {
1153 		slot->flags |= STOP_STARTED;
1154 		sdhci_start_command(slot, req->stop);
1155 		return;
1156 	}
1157 */
1158 	if (sdhci_debug > 1)
1159 		slot_printf(slot, "result: %d\n", req->cmd->error);
1160 	if (!req->cmd->error &&
1161 	    (slot->sc->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST)) {
1162 		sdhci_reset(slot, SDHCI_RESET_CMD);
1163 		sdhci_reset(slot, SDHCI_RESET_DATA);
1164 	}
1165 
1166 	/* We must be done -- bad idea to do this while locked? */
1167 	slot->req = NULL;
1168 	slot->curcmd = NULL;
1169 	req->done(req);
1170 }
1171 
1172 static int
1173 sdhci_request(device_t brdev, device_t reqdev, struct mmc_request *req)
1174 {
1175 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1176 
1177 	SDHCI_LOCK(slot);
1178 	if (slot->req != NULL) {
1179 		SDHCI_UNLOCK(slot);
1180 		return (EBUSY);
1181 	}
1182 	if (sdhci_debug > 1) {
1183 		slot_printf(slot, "CMD%u arg %#x flags %#x dlen %u dflags %#x\n",
1184     		    req->cmd->opcode, req->cmd->arg, req->cmd->flags,
1185     		    (req->cmd->data)?(u_int)req->cmd->data->len:0,
1186 		    (req->cmd->data)?req->cmd->data->flags:0);
1187 	}
1188 	slot->req = req;
1189 	slot->flags = 0;
1190 	sdhci_start(slot);
1191 	SDHCI_UNLOCK(slot);
1192 	if (dumping) {
1193 		while (slot->req != NULL) {
1194 			sdhci_intr(slot->sc);
1195 			DELAY(10);
1196 		}
1197 	}
1198 	return (0);
1199 }
1200 
1201 static int
1202 sdhci_get_ro(device_t brdev, device_t reqdev)
1203 {
1204 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1205 	uint32_t val;
1206 
1207 	SDHCI_LOCK(slot);
1208 	val = RD4(slot, SDHCI_PRESENT_STATE);
1209 	SDHCI_UNLOCK(slot);
1210 	return (!(val & SDHCI_WRITE_PROTECT));
1211 }
1212 
1213 static int
1214 sdhci_acquire_host(device_t brdev, device_t reqdev)
1215 {
1216 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1217 	int err = 0;
1218 
1219 	SDHCI_LOCK(slot);
1220 	while (slot->bus_busy)
1221 		msleep(slot, &slot->mtx, 0, "sdhciah", 0);
1222 	slot->bus_busy++;
1223 	/* Activate led. */
1224 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl |= SDHCI_CTRL_LED);
1225 	SDHCI_UNLOCK(slot);
1226 	return (err);
1227 }
1228 
1229 static int
1230 sdhci_release_host(device_t brdev, device_t reqdev)
1231 {
1232 	struct sdhci_slot *slot = device_get_ivars(reqdev);
1233 
1234 	SDHCI_LOCK(slot);
1235 	/* Deactivate led. */
1236 	WR1(slot, SDHCI_HOST_CONTROL, slot->hostctrl &= ~SDHCI_CTRL_LED);
1237 	slot->bus_busy--;
1238 	SDHCI_UNLOCK(slot);
1239 	wakeup(slot);
1240 	return (0);
1241 }
1242 
1243 static void
1244 sdhci_cmd_irq(struct sdhci_slot *slot, uint32_t intmask)
1245 {
1246 
1247 	if (!slot->curcmd) {
1248 		slot_printf(slot, "Got command interrupt 0x%08x, but "
1249 		    "there is no active command.\n", intmask);
1250 		sdhci_dumpregs(slot);
1251 		return;
1252 	}
1253 	if (intmask & SDHCI_INT_TIMEOUT)
1254 		slot->curcmd->error = MMC_ERR_TIMEOUT;
1255 	else if (intmask & SDHCI_INT_CRC)
1256 		slot->curcmd->error = MMC_ERR_BADCRC;
1257 	else if (intmask & (SDHCI_INT_END_BIT | SDHCI_INT_INDEX))
1258 		slot->curcmd->error = MMC_ERR_FIFO;
1259 
1260 	sdhci_finish_command(slot);
1261 }
1262 
1263 static void
1264 sdhci_data_irq(struct sdhci_slot *slot, uint32_t intmask)
1265 {
1266 
1267 	if (!slot->curcmd) {
1268 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1269 		    "there is no active command.\n", intmask);
1270 		sdhci_dumpregs(slot);
1271 		return;
1272 	}
1273 	if (slot->curcmd->data == NULL &&
1274 	    (slot->curcmd->flags & MMC_RSP_BUSY) == 0) {
1275 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1276 		    "there is no active data operation.\n",
1277 		    intmask);
1278 		sdhci_dumpregs(slot);
1279 		return;
1280 	}
1281 	if (intmask & SDHCI_INT_DATA_TIMEOUT)
1282 		slot->curcmd->error = MMC_ERR_TIMEOUT;
1283 	else if (intmask & (SDHCI_INT_DATA_CRC | SDHCI_INT_DATA_END_BIT))
1284 		slot->curcmd->error = MMC_ERR_BADCRC;
1285 	if (slot->curcmd->data == NULL &&
1286 	    (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL |
1287 	    SDHCI_INT_DMA_END))) {
1288 		slot_printf(slot, "Got data interrupt 0x%08x, but "
1289 		    "there is busy-only command.\n", intmask);
1290 		sdhci_dumpregs(slot);
1291 		slot->curcmd->error = MMC_ERR_INVALID;
1292 	}
1293 	if (slot->curcmd->error) {
1294 		/* No need to continue after any error. */
1295 		sdhci_finish_data(slot);
1296 		return;
1297 	}
1298 
1299 	/* Handle PIO interrupt. */
1300 	if (intmask & (SDHCI_INT_DATA_AVAIL | SDHCI_INT_SPACE_AVAIL))
1301 		sdhci_transfer_pio(slot);
1302 	/* Handle DMA border. */
1303 	if (intmask & SDHCI_INT_DMA_END) {
1304 		struct mmc_data *data = slot->curcmd->data;
1305 		size_t left;
1306 
1307 		/* Unload DMA buffer... */
1308 		left = data->len - slot->offset;
1309 		if (data->flags & MMC_DATA_READ) {
1310 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1311 			    BUS_DMASYNC_POSTREAD);
1312 			memcpy((u_char*)data->data + slot->offset, slot->dmamem,
1313 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1314 		} else {
1315 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1316 			    BUS_DMASYNC_POSTWRITE);
1317 		}
1318 		/* ... and reload it again. */
1319 		slot->offset += DMA_BLOCK_SIZE;
1320 		left = data->len - slot->offset;
1321 		if (data->flags & MMC_DATA_READ) {
1322 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1323 			    BUS_DMASYNC_PREREAD);
1324 		} else {
1325 			memcpy(slot->dmamem, (u_char*)data->data + slot->offset,
1326 			    (left < DMA_BLOCK_SIZE)?left:DMA_BLOCK_SIZE);
1327 			bus_dmamap_sync(slot->dmatag, slot->dmamap,
1328 			    BUS_DMASYNC_PREWRITE);
1329 		}
1330 		/* Interrupt aggregation: Mask border interrupt
1331 		 * for the last page. */
1332 		if (left == DMA_BLOCK_SIZE) {
1333 			slot->intmask &= ~SDHCI_INT_DMA_END;
1334 			WR4(slot, SDHCI_SIGNAL_ENABLE, slot->intmask);
1335 		}
1336 		/* Restart DMA. */
1337 		WR4(slot, SDHCI_DMA_ADDRESS, slot->paddr);
1338 	}
1339 	/* We have got all data. */
1340 	if (intmask & SDHCI_INT_DATA_END)
1341 		sdhci_finish_data(slot);
1342 }
1343 
1344 static void
1345 sdhci_acmd_irq(struct sdhci_slot *slot)
1346 {
1347 	uint16_t err;
1348 
1349 	err = RD4(slot, SDHCI_ACMD12_ERR);
1350 	if (!slot->curcmd) {
1351 		slot_printf(slot, "Got AutoCMD12 error 0x%04x, but "
1352 		    "there is no active command.\n", err);
1353 		sdhci_dumpregs(slot);
1354 		return;
1355 	}
1356 	slot_printf(slot, "Got AutoCMD12 error 0x%04x\n", err);
1357 	sdhci_reset(slot, SDHCI_RESET_CMD);
1358 }
1359 
1360 static void
1361 sdhci_intr(void *arg)
1362 {
1363 	struct sdhci_softc *sc = (struct sdhci_softc *)arg;
1364 	int i;
1365 
1366 	for (i = 0; i < sc->num_slots; i++) {
1367 		struct sdhci_slot *slot = &sc->slots[i];
1368 		uint32_t intmask;
1369 
1370 		SDHCI_LOCK(slot);
1371 		/* Read slot interrupt status. */
1372 		intmask = RD4(slot, SDHCI_INT_STATUS);
1373 		if (intmask == 0 || intmask == 0xffffffff) {
1374 			SDHCI_UNLOCK(slot);
1375 			continue;
1376 		}
1377 		if (sdhci_debug > 2)
1378 			slot_printf(slot, "Interrupt %#x\n", intmask);
1379 
1380 		/* Handle card presence interrupts. */
1381 		if (intmask & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) {
1382 			WR4(slot, SDHCI_INT_STATUS, intmask &
1383 			    (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE));
1384 
1385 			if (intmask & SDHCI_INT_CARD_REMOVE) {
1386 				if (bootverbose || sdhci_debug)
1387 					slot_printf(slot, "Card removed\n");
1388 				callout_stop(&slot->card_callout);
1389 				taskqueue_enqueue(taskqueue_swi_giant,
1390 				    &slot->card_task);
1391 			}
1392 			if (intmask & SDHCI_INT_CARD_INSERT) {
1393 				if (bootverbose || sdhci_debug)
1394 					slot_printf(slot, "Card inserted\n");
1395 				callout_reset(&slot->card_callout, hz / 2,
1396 				    sdhci_card_delay, slot);
1397 			}
1398 			intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
1399 		}
1400 		/* Handle command interrupts. */
1401 		if (intmask & SDHCI_INT_CMD_MASK) {
1402 			WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_CMD_MASK);
1403 			sdhci_cmd_irq(slot, intmask & SDHCI_INT_CMD_MASK);
1404 		}
1405 		/* Handle data interrupts. */
1406 		if (intmask & SDHCI_INT_DATA_MASK) {
1407 			WR4(slot, SDHCI_INT_STATUS, intmask & SDHCI_INT_DATA_MASK);
1408 			sdhci_data_irq(slot, intmask & SDHCI_INT_DATA_MASK);
1409 		}
1410 		/* Handle AutoCMD12 error interrupt. */
1411 		if (intmask & SDHCI_INT_ACMD12ERR) {
1412 			WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_ACMD12ERR);
1413 			sdhci_acmd_irq(slot);
1414 		}
1415 		intmask &= ~(SDHCI_INT_CMD_MASK | SDHCI_INT_DATA_MASK);
1416 		intmask &= ~SDHCI_INT_ACMD12ERR;
1417 		intmask &= ~SDHCI_INT_ERROR;
1418 		/* Handle bus power interrupt. */
1419 		if (intmask & SDHCI_INT_BUS_POWER) {
1420 			WR4(slot, SDHCI_INT_STATUS, SDHCI_INT_BUS_POWER);
1421 			slot_printf(slot,
1422 			    "Card is consuming too much power!\n");
1423 			intmask &= ~SDHCI_INT_BUS_POWER;
1424 		}
1425 		/* The rest is unknown. */
1426 		if (intmask) {
1427 			WR4(slot, SDHCI_INT_STATUS, intmask);
1428 			slot_printf(slot, "Unexpected interrupt 0x%08x.\n",
1429 			    intmask);
1430 			sdhci_dumpregs(slot);
1431 		}
1432 
1433 		SDHCI_UNLOCK(slot);
1434 	}
1435 }
1436 
1437 static int
1438 sdhci_read_ivar(device_t bus, device_t child, int which, uintptr_t *result)
1439 {
1440 	struct sdhci_slot *slot = device_get_ivars(child);
1441 
1442 	switch (which) {
1443 	default:
1444 		return (EINVAL);
1445 	case MMCBR_IVAR_BUS_MODE:
1446 		*result = slot->host.ios.bus_mode;
1447 		break;
1448 	case MMCBR_IVAR_BUS_WIDTH:
1449 		*result = slot->host.ios.bus_width;
1450 		break;
1451 	case MMCBR_IVAR_CHIP_SELECT:
1452 		*result = slot->host.ios.chip_select;
1453 		break;
1454 	case MMCBR_IVAR_CLOCK:
1455 		*result = slot->host.ios.clock;
1456 		break;
1457 	case MMCBR_IVAR_F_MIN:
1458 		*result = slot->host.f_min;
1459 		break;
1460 	case MMCBR_IVAR_F_MAX:
1461 		*result = slot->host.f_max;
1462 		break;
1463 	case MMCBR_IVAR_HOST_OCR:
1464 		*result = slot->host.host_ocr;
1465 		break;
1466 	case MMCBR_IVAR_MODE:
1467 		*result = slot->host.mode;
1468 		break;
1469 	case MMCBR_IVAR_OCR:
1470 		*result = slot->host.ocr;
1471 		break;
1472 	case MMCBR_IVAR_POWER_MODE:
1473 		*result = slot->host.ios.power_mode;
1474 		break;
1475 	case MMCBR_IVAR_VDD:
1476 		*result = slot->host.ios.vdd;
1477 		break;
1478 	case MMCBR_IVAR_CAPS:
1479 		*result = slot->host.caps;
1480 		break;
1481 	case MMCBR_IVAR_TIMING:
1482 		*result = slot->host.ios.timing;
1483 		break;
1484 	case MMCBR_IVAR_MAX_DATA:
1485 		*result = 65535;
1486 		break;
1487 	}
1488 	return (0);
1489 }
1490 
1491 static int
1492 sdhci_write_ivar(device_t bus, device_t child, int which, uintptr_t value)
1493 {
1494 	struct sdhci_slot *slot = device_get_ivars(child);
1495 
1496 	switch (which) {
1497 	default:
1498 		return (EINVAL);
1499 	case MMCBR_IVAR_BUS_MODE:
1500 		slot->host.ios.bus_mode = value;
1501 		break;
1502 	case MMCBR_IVAR_BUS_WIDTH:
1503 		slot->host.ios.bus_width = value;
1504 		break;
1505 	case MMCBR_IVAR_CHIP_SELECT:
1506 		slot->host.ios.chip_select = value;
1507 		break;
1508 	case MMCBR_IVAR_CLOCK:
1509 		if (value > 0) {
1510 			uint32_t clock = slot->max_clk;
1511 			int i;
1512 
1513 			for (i = 0; i < 8; i++) {
1514 				if (clock <= value)
1515 					break;
1516 				clock >>= 1;
1517 			}
1518 			slot->host.ios.clock = clock;
1519 		} else
1520 			slot->host.ios.clock = 0;
1521 		break;
1522 	case MMCBR_IVAR_MODE:
1523 		slot->host.mode = value;
1524 		break;
1525 	case MMCBR_IVAR_OCR:
1526 		slot->host.ocr = value;
1527 		break;
1528 	case MMCBR_IVAR_POWER_MODE:
1529 		slot->host.ios.power_mode = value;
1530 		break;
1531 	case MMCBR_IVAR_VDD:
1532 		slot->host.ios.vdd = value;
1533 		break;
1534 	case MMCBR_IVAR_TIMING:
1535 		slot->host.ios.timing = value;
1536 		break;
1537 	case MMCBR_IVAR_CAPS:
1538 	case MMCBR_IVAR_HOST_OCR:
1539 	case MMCBR_IVAR_F_MIN:
1540 	case MMCBR_IVAR_F_MAX:
1541 	case MMCBR_IVAR_MAX_DATA:
1542 		return (EINVAL);
1543 	}
1544 	return (0);
1545 }
1546 
1547 static device_method_t sdhci_methods[] = {
1548 	/* device_if */
1549 	DEVMETHOD(device_probe, sdhci_probe),
1550 	DEVMETHOD(device_attach, sdhci_attach),
1551 	DEVMETHOD(device_detach, sdhci_detach),
1552 	DEVMETHOD(device_suspend, sdhci_suspend),
1553 	DEVMETHOD(device_resume, sdhci_resume),
1554 
1555 	/* Bus interface */
1556 	DEVMETHOD(bus_read_ivar,	sdhci_read_ivar),
1557 	DEVMETHOD(bus_write_ivar,	sdhci_write_ivar),
1558 
1559 	/* mmcbr_if */
1560 	DEVMETHOD(mmcbr_update_ios, sdhci_update_ios),
1561 	DEVMETHOD(mmcbr_request, sdhci_request),
1562 	DEVMETHOD(mmcbr_get_ro, sdhci_get_ro),
1563 	DEVMETHOD(mmcbr_acquire_host, sdhci_acquire_host),
1564 	DEVMETHOD(mmcbr_release_host, sdhci_release_host),
1565 
1566 	{0, 0},
1567 };
1568 
1569 static driver_t sdhci_driver = {
1570 	"sdhci",
1571 	sdhci_methods,
1572 	sizeof(struct sdhci_softc),
1573 };
1574 static devclass_t sdhci_devclass;
1575 
1576 
1577 DRIVER_MODULE(sdhci, pci, sdhci_driver, sdhci_devclass, 0, 0);
1578