xref: /freebsd/sys/dev/agp/agp.c (revision b28624fde638caadd4a89f50c9b7e7da0f98c4d2)
1 /*-
2  * Copyright (c) 2000 Doug Rabson
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include "opt_bus.h"
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/malloc.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/ioccom.h>
40 #include <sys/agpio.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 
45 #include <dev/pci/pcivar.h>
46 #include <dev/pci/pcireg.h>
47 #include <pci/agppriv.h>
48 #include <pci/agpvar.h>
49 #include <pci/agpreg.h>
50 
51 #include <vm/vm.h>
52 #include <vm/vm_object.h>
53 #include <vm/vm_page.h>
54 #include <vm/vm_pageout.h>
55 #include <vm/pmap.h>
56 
57 #include <machine/md_var.h>
58 #include <machine/bus.h>
59 #include <machine/resource.h>
60 #include <sys/rman.h>
61 
62 MODULE_VERSION(agp, 1);
63 
64 MALLOC_DEFINE(M_AGP, "agp", "AGP data structures");
65 
66 				/* agp_drv.c */
67 static d_open_t agp_open;
68 static d_close_t agp_close;
69 static d_ioctl_t agp_ioctl;
70 static d_mmap_t agp_mmap;
71 
72 static struct cdevsw agp_cdevsw = {
73 	.d_version =	D_VERSION,
74 	.d_flags =	D_NEEDGIANT,
75 	.d_open =	agp_open,
76 	.d_close =	agp_close,
77 	.d_ioctl =	agp_ioctl,
78 	.d_mmap =	agp_mmap,
79 	.d_name =	"agp",
80 };
81 
82 static devclass_t agp_devclass;
83 #define KDEV2DEV(kdev)	devclass_get_device(agp_devclass, minor(kdev))
84 
85 /* Helper functions for implementing chipset mini drivers. */
86 
87 void
88 agp_flush_cache()
89 {
90 #if defined(__i386__) || defined(__amd64__)
91 	wbinvd();
92 #endif
93 }
94 
95 u_int8_t
96 agp_find_caps(device_t dev)
97 {
98 	int capreg;
99 
100 
101 	if (pci_find_extcap(dev, PCIY_AGP, &capreg) != 0)
102 		capreg = 0;
103 	return (capreg);
104 }
105 
106 /*
107  * Find an AGP display device (if any).
108  */
109 static device_t
110 agp_find_display(void)
111 {
112 	devclass_t pci = devclass_find("pci");
113 	device_t bus, dev = 0;
114 	device_t *kids;
115 	int busnum, numkids, i;
116 
117 	for (busnum = 0; busnum < devclass_get_maxunit(pci); busnum++) {
118 		bus = devclass_get_device(pci, busnum);
119 		if (!bus)
120 			continue;
121 		device_get_children(bus, &kids, &numkids);
122 		for (i = 0; i < numkids; i++) {
123 			dev = kids[i];
124 			if (pci_get_class(dev) == PCIC_DISPLAY
125 			    && pci_get_subclass(dev) == PCIS_DISPLAY_VGA)
126 				if (agp_find_caps(dev)) {
127 					free(kids, M_TEMP);
128 					return dev;
129 				}
130 
131 		}
132 		free(kids, M_TEMP);
133 	}
134 
135 	return 0;
136 }
137 
138 struct agp_gatt *
139 agp_alloc_gatt(device_t dev)
140 {
141 	u_int32_t apsize = AGP_GET_APERTURE(dev);
142 	u_int32_t entries = apsize >> AGP_PAGE_SHIFT;
143 	struct agp_gatt *gatt;
144 
145 	if (bootverbose)
146 		device_printf(dev,
147 			      "allocating GATT for aperture of size %dM\n",
148 			      apsize / (1024*1024));
149 
150 	if (entries == 0) {
151 		device_printf(dev, "bad aperture size\n");
152 		return NULL;
153 	}
154 
155 	gatt = malloc(sizeof(struct agp_gatt), M_AGP, M_NOWAIT);
156 	if (!gatt)
157 		return 0;
158 
159 	gatt->ag_entries = entries;
160 	gatt->ag_virtual = contigmalloc(entries * sizeof(u_int32_t), M_AGP, 0,
161 					0, ~0, PAGE_SIZE, 0);
162 	if (!gatt->ag_virtual) {
163 		if (bootverbose)
164 			device_printf(dev, "contiguous allocation failed\n");
165 		free(gatt, M_AGP);
166 		return 0;
167 	}
168 	bzero(gatt->ag_virtual, entries * sizeof(u_int32_t));
169 	gatt->ag_physical = vtophys((vm_offset_t) gatt->ag_virtual);
170 	agp_flush_cache();
171 
172 	return gatt;
173 }
174 
175 void
176 agp_free_gatt(struct agp_gatt *gatt)
177 {
178 	contigfree(gatt->ag_virtual,
179 		   gatt->ag_entries * sizeof(u_int32_t), M_AGP);
180 	free(gatt, M_AGP);
181 }
182 
183 static u_int agp_max[][2] = {
184 	{0,	0},
185 	{32,	4},
186 	{64,	28},
187 	{128,	96},
188 	{256,	204},
189 	{512,	440},
190 	{1024,	942},
191 	{2048,	1920},
192 	{4096,	3932}
193 };
194 #define agp_max_size	(sizeof(agp_max) / sizeof(agp_max[0]))
195 
196 /**
197  * Sets the PCI resource which represents the AGP aperture.
198  *
199  * If not called, the default AGP aperture resource of AGP_APBASE will
200  * be used.  Must be called before agp_generic_attach().
201  */
202 void
203 agp_set_aperture_resource(device_t dev, int rid)
204 {
205 	struct agp_softc *sc = device_get_softc(dev);
206 
207 	sc->as_aperture_rid = rid;
208 }
209 
210 int
211 agp_generic_attach(device_t dev)
212 {
213 	struct agp_softc *sc = device_get_softc(dev);
214 	int i;
215 	u_int memsize;
216 
217 	/*
218 	 * Find and map the aperture, RF_SHAREABLE for DRM but not RF_ACTIVE
219 	 * because the kernel doesn't need to map it.
220 	 */
221 	if (sc->as_aperture_rid == 0)
222 		sc->as_aperture_rid = AGP_APBASE;
223 
224 	sc->as_aperture = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
225 	    &sc->as_aperture_rid, RF_SHAREABLE);
226 	if (!sc->as_aperture)
227 		return ENOMEM;
228 
229 	/*
230 	 * Work out an upper bound for agp memory allocation. This
231 	 * uses a heurisitc table from the Linux driver.
232 	 */
233 	memsize = ptoa(Maxmem) >> 20;
234 	for (i = 0; i < agp_max_size; i++) {
235 		if (memsize <= agp_max[i][0])
236 			break;
237 	}
238 	if (i == agp_max_size) i = agp_max_size - 1;
239 	sc->as_maxmem = agp_max[i][1] << 20U;
240 
241 	/*
242 	 * The lock is used to prevent re-entry to
243 	 * agp_generic_bind_memory() since that function can sleep.
244 	 */
245 	mtx_init(&sc->as_lock, "agp lock", NULL, MTX_DEF);
246 
247 	/*
248 	 * Initialise stuff for the userland device.
249 	 */
250 	agp_devclass = devclass_find("agp");
251 	TAILQ_INIT(&sc->as_memory);
252 	sc->as_nextid = 1;
253 
254 	sc->as_devnode = make_dev(&agp_cdevsw,
255 				  device_get_unit(dev),
256 				  UID_ROOT,
257 				  GID_WHEEL,
258 				  0600,
259 				  "agpgart");
260 
261 	return 0;
262 }
263 
264 int
265 agp_generic_detach(device_t dev)
266 {
267 	struct agp_softc *sc = device_get_softc(dev);
268 
269 	destroy_dev(sc->as_devnode);
270 	bus_release_resource(dev, SYS_RES_MEMORY, sc->as_aperture_rid,
271 	    sc->as_aperture);
272 	mtx_destroy(&sc->as_lock);
273 	agp_flush_cache();
274 	return 0;
275 }
276 
277 /**
278  * Default AGP aperture size detection which simply returns the size of
279  * the aperture's PCI resource.
280  */
281 int
282 agp_generic_get_aperture(device_t dev)
283 {
284 	struct agp_softc *sc = device_get_softc(dev);
285 
286 	return rman_get_size(sc->as_aperture);
287 }
288 
289 /**
290  * Default AGP aperture size setting function, which simply doesn't allow
291  * changes to resource size.
292  */
293 int
294 agp_generic_set_aperture(device_t dev, u_int32_t aperture)
295 {
296 	u_int32_t current_aperture;
297 
298 	current_aperture = AGP_GET_APERTURE(dev);
299 	if (current_aperture != aperture)
300 		return EINVAL;
301 	else
302 		return 0;
303 }
304 
305 /*
306  * This does the enable logic for v3, with the same topology
307  * restrictions as in place for v2 -- one bus, one device on the bus.
308  */
309 static int
310 agp_v3_enable(device_t dev, device_t mdev, u_int32_t mode)
311 {
312 	u_int32_t tstatus, mstatus;
313 	u_int32_t command;
314 	int rq, sba, fw, rate, arqsz, cal;
315 
316 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
317 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
318 
319 	/* Set RQ to the min of mode, tstatus and mstatus */
320 	rq = AGP_MODE_GET_RQ(mode);
321 	if (AGP_MODE_GET_RQ(tstatus) < rq)
322 		rq = AGP_MODE_GET_RQ(tstatus);
323 	if (AGP_MODE_GET_RQ(mstatus) < rq)
324 		rq = AGP_MODE_GET_RQ(mstatus);
325 
326 	/*
327 	 * ARQSZ - Set the value to the maximum one.
328 	 * Don't allow the mode register to override values.
329 	 */
330 	arqsz = AGP_MODE_GET_ARQSZ(mode);
331 	if (AGP_MODE_GET_ARQSZ(tstatus) > rq)
332 		rq = AGP_MODE_GET_ARQSZ(tstatus);
333 	if (AGP_MODE_GET_ARQSZ(mstatus) > rq)
334 		rq = AGP_MODE_GET_ARQSZ(mstatus);
335 
336 	/* Calibration cycle - don't allow override by mode register */
337 	cal = AGP_MODE_GET_CAL(tstatus);
338 	if (AGP_MODE_GET_CAL(mstatus) < cal)
339 		cal = AGP_MODE_GET_CAL(mstatus);
340 
341 	/* SBA must be supported for AGP v3. */
342 	sba = 1;
343 
344 	/* Set FW if all three support it. */
345 	fw = (AGP_MODE_GET_FW(tstatus)
346 	       & AGP_MODE_GET_FW(mstatus)
347 	       & AGP_MODE_GET_FW(mode));
348 
349 	/* Figure out the max rate */
350 	rate = (AGP_MODE_GET_RATE(tstatus)
351 		& AGP_MODE_GET_RATE(mstatus)
352 		& AGP_MODE_GET_RATE(mode));
353 	if (rate & AGP_MODE_V3_RATE_8x)
354 		rate = AGP_MODE_V3_RATE_8x;
355 	else
356 		rate = AGP_MODE_V3_RATE_4x;
357 	if (bootverbose)
358 		device_printf(dev, "Setting AGP v3 mode %d\n", rate * 4);
359 
360 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, 0, 4);
361 
362 	/* Construct the new mode word and tell the hardware */
363 	command = 0;
364 	command = AGP_MODE_SET_RQ(0, rq);
365 	command = AGP_MODE_SET_ARQSZ(command, arqsz);
366 	command = AGP_MODE_SET_CAL(command, cal);
367 	command = AGP_MODE_SET_SBA(command, sba);
368 	command = AGP_MODE_SET_FW(command, fw);
369 	command = AGP_MODE_SET_RATE(command, rate);
370 	command = AGP_MODE_SET_MODE_3(command, 1);
371 	command = AGP_MODE_SET_AGP(command, 1);
372 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, command, 4);
373 	pci_write_config(mdev, agp_find_caps(mdev) + AGP_COMMAND, command, 4);
374 
375 	return 0;
376 }
377 
378 static int
379 agp_v2_enable(device_t dev, device_t mdev, u_int32_t mode)
380 {
381 	u_int32_t tstatus, mstatus;
382 	u_int32_t command;
383 	int rq, sba, fw, rate;
384 
385 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
386 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
387 
388 	/* Set RQ to the min of mode, tstatus and mstatus */
389 	rq = AGP_MODE_GET_RQ(mode);
390 	if (AGP_MODE_GET_RQ(tstatus) < rq)
391 		rq = AGP_MODE_GET_RQ(tstatus);
392 	if (AGP_MODE_GET_RQ(mstatus) < rq)
393 		rq = AGP_MODE_GET_RQ(mstatus);
394 
395 	/* Set SBA if all three can deal with SBA */
396 	sba = (AGP_MODE_GET_SBA(tstatus)
397 	       & AGP_MODE_GET_SBA(mstatus)
398 	       & AGP_MODE_GET_SBA(mode));
399 
400 	/* Similar for FW */
401 	fw = (AGP_MODE_GET_FW(tstatus)
402 	       & AGP_MODE_GET_FW(mstatus)
403 	       & AGP_MODE_GET_FW(mode));
404 
405 	/* Figure out the max rate */
406 	rate = (AGP_MODE_GET_RATE(tstatus)
407 		& AGP_MODE_GET_RATE(mstatus)
408 		& AGP_MODE_GET_RATE(mode));
409 	if (rate & AGP_MODE_V2_RATE_4x)
410 		rate = AGP_MODE_V2_RATE_4x;
411 	else if (rate & AGP_MODE_V2_RATE_2x)
412 		rate = AGP_MODE_V2_RATE_2x;
413 	else
414 		rate = AGP_MODE_V2_RATE_1x;
415 	if (bootverbose)
416 		device_printf(dev, "Setting AGP v2 mode %d\n", rate);
417 
418 	/* Construct the new mode word and tell the hardware */
419 	command = 0;
420 	command = AGP_MODE_SET_RQ(0, rq);
421 	command = AGP_MODE_SET_SBA(command, sba);
422 	command = AGP_MODE_SET_FW(command, fw);
423 	command = AGP_MODE_SET_RATE(command, rate);
424 	command = AGP_MODE_SET_AGP(command, 1);
425 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, command, 4);
426 	pci_write_config(mdev, agp_find_caps(mdev) + AGP_COMMAND, command, 4);
427 
428 	return 0;
429 }
430 
431 int
432 agp_generic_enable(device_t dev, u_int32_t mode)
433 {
434 	device_t mdev = agp_find_display();
435 	u_int32_t tstatus, mstatus;
436 
437 	if (!mdev) {
438 		AGP_DPF("can't find display\n");
439 		return ENXIO;
440 	}
441 
442 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
443 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
444 
445 	/*
446 	 * Check display and bridge for AGP v3 support.  AGP v3 allows
447 	 * more variety in topology than v2, e.g. multiple AGP devices
448 	 * attached to one bridge, or multiple AGP bridges in one
449 	 * system.  This doesn't attempt to address those situations,
450 	 * but should work fine for a classic single AGP slot system
451 	 * with AGP v3.
452 	 */
453 	if (AGP_MODE_GET_MODE_3(mode) &&
454 	    AGP_MODE_GET_MODE_3(tstatus) &&
455 	    AGP_MODE_GET_MODE_3(mstatus))
456 		return (agp_v3_enable(dev, mdev, mode));
457 	else
458 		return (agp_v2_enable(dev, mdev, mode));
459 }
460 
461 struct agp_memory *
462 agp_generic_alloc_memory(device_t dev, int type, vm_size_t size)
463 {
464 	struct agp_softc *sc = device_get_softc(dev);
465 	struct agp_memory *mem;
466 
467 	if ((size & (AGP_PAGE_SIZE - 1)) != 0)
468 		return 0;
469 
470 	if (sc->as_allocated + size > sc->as_maxmem)
471 		return 0;
472 
473 	if (type != 0) {
474 		printf("agp_generic_alloc_memory: unsupported type %d\n",
475 		       type);
476 		return 0;
477 	}
478 
479 	mem = malloc(sizeof *mem, M_AGP, M_WAITOK);
480 	mem->am_id = sc->as_nextid++;
481 	mem->am_size = size;
482 	mem->am_type = 0;
483 	mem->am_obj = vm_object_allocate(OBJT_DEFAULT, atop(round_page(size)));
484 	mem->am_physical = 0;
485 	mem->am_offset = 0;
486 	mem->am_is_bound = 0;
487 	TAILQ_INSERT_TAIL(&sc->as_memory, mem, am_link);
488 	sc->as_allocated += size;
489 
490 	return mem;
491 }
492 
493 int
494 agp_generic_free_memory(device_t dev, struct agp_memory *mem)
495 {
496 	struct agp_softc *sc = device_get_softc(dev);
497 
498 	if (mem->am_is_bound)
499 		return EBUSY;
500 
501 	sc->as_allocated -= mem->am_size;
502 	TAILQ_REMOVE(&sc->as_memory, mem, am_link);
503 	vm_object_deallocate(mem->am_obj);
504 	free(mem, M_AGP);
505 	return 0;
506 }
507 
508 int
509 agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
510 			vm_offset_t offset)
511 {
512 	struct agp_softc *sc = device_get_softc(dev);
513 	vm_offset_t i, j, k;
514 	vm_page_t m;
515 	int error;
516 
517 	/* Do some sanity checks first. */
518 	if (offset < 0 || (offset & (AGP_PAGE_SIZE - 1)) != 0 ||
519 	    offset + mem->am_size > AGP_GET_APERTURE(dev)) {
520 		device_printf(dev, "binding memory at bad offset %#x\n",
521 		    (int)offset);
522 		return EINVAL;
523 	}
524 
525 	/*
526 	 * Allocate the pages early, before acquiring the lock,
527 	 * because vm_page_grab() used with VM_ALLOC_RETRY may
528 	 * block and we can't hold a mutex while blocking.
529 	 */
530 	VM_OBJECT_LOCK(mem->am_obj);
531 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
532 		/*
533 		 * Find a page from the object and wire it
534 		 * down. This page will be mapped using one or more
535 		 * entries in the GATT (assuming that PAGE_SIZE >=
536 		 * AGP_PAGE_SIZE. If this is the first call to bind,
537 		 * the pages will be allocated and zeroed.
538 		 */
539 		m = vm_page_grab(mem->am_obj, OFF_TO_IDX(i),
540 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
541 		AGP_DPF("found page pa=%#x\n", VM_PAGE_TO_PHYS(m));
542 	}
543 	VM_OBJECT_UNLOCK(mem->am_obj);
544 
545 	mtx_lock(&sc->as_lock);
546 
547 	if (mem->am_is_bound) {
548 		device_printf(dev, "memory already bound\n");
549 		error = EINVAL;
550 		VM_OBJECT_LOCK(mem->am_obj);
551 		goto bad;
552 	}
553 
554 	/*
555 	 * Bind the individual pages and flush the chipset's
556 	 * TLB.
557 	 */
558 	VM_OBJECT_LOCK(mem->am_obj);
559 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
560 		m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(i));
561 
562 		/*
563 		 * Install entries in the GATT, making sure that if
564 		 * AGP_PAGE_SIZE < PAGE_SIZE and mem->am_size is not
565 		 * aligned to PAGE_SIZE, we don't modify too many GATT
566 		 * entries.
567 		 */
568 		for (j = 0; j < PAGE_SIZE && i + j < mem->am_size;
569 		     j += AGP_PAGE_SIZE) {
570 			vm_offset_t pa = VM_PAGE_TO_PHYS(m) + j;
571 			AGP_DPF("binding offset %#x to pa %#x\n",
572 				offset + i + j, pa);
573 			error = AGP_BIND_PAGE(dev, offset + i + j, pa);
574 			if (error) {
575 				/*
576 				 * Bail out. Reverse all the mappings
577 				 * and unwire the pages.
578 				 */
579 				vm_page_wakeup(m);
580 				for (k = 0; k < i + j; k += AGP_PAGE_SIZE)
581 					AGP_UNBIND_PAGE(dev, offset + k);
582 				goto bad;
583 			}
584 		}
585 		vm_page_wakeup(m);
586 	}
587 	VM_OBJECT_UNLOCK(mem->am_obj);
588 
589 	/*
590 	 * Flush the cpu cache since we are providing a new mapping
591 	 * for these pages.
592 	 */
593 	agp_flush_cache();
594 
595 	/*
596 	 * Make sure the chipset gets the new mappings.
597 	 */
598 	AGP_FLUSH_TLB(dev);
599 
600 	mem->am_offset = offset;
601 	mem->am_is_bound = 1;
602 
603 	mtx_unlock(&sc->as_lock);
604 
605 	return 0;
606 bad:
607 	mtx_unlock(&sc->as_lock);
608 	VM_OBJECT_LOCK_ASSERT(mem->am_obj, MA_OWNED);
609 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
610 		m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(i));
611 		vm_page_lock_queues();
612 		vm_page_unwire(m, 0);
613 		vm_page_unlock_queues();
614 	}
615 	VM_OBJECT_UNLOCK(mem->am_obj);
616 
617 	return error;
618 }
619 
620 int
621 agp_generic_unbind_memory(device_t dev, struct agp_memory *mem)
622 {
623 	struct agp_softc *sc = device_get_softc(dev);
624 	vm_page_t m;
625 	int i;
626 
627 	mtx_lock(&sc->as_lock);
628 
629 	if (!mem->am_is_bound) {
630 		device_printf(dev, "memory is not bound\n");
631 		mtx_unlock(&sc->as_lock);
632 		return EINVAL;
633 	}
634 
635 
636 	/*
637 	 * Unbind the individual pages and flush the chipset's
638 	 * TLB. Unwire the pages so they can be swapped.
639 	 */
640 	for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
641 		AGP_UNBIND_PAGE(dev, mem->am_offset + i);
642 	VM_OBJECT_LOCK(mem->am_obj);
643 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
644 		m = vm_page_lookup(mem->am_obj, atop(i));
645 		vm_page_lock_queues();
646 		vm_page_unwire(m, 0);
647 		vm_page_unlock_queues();
648 	}
649 	VM_OBJECT_UNLOCK(mem->am_obj);
650 
651 	agp_flush_cache();
652 	AGP_FLUSH_TLB(dev);
653 
654 	mem->am_offset = 0;
655 	mem->am_is_bound = 0;
656 
657 	mtx_unlock(&sc->as_lock);
658 
659 	return 0;
660 }
661 
662 /* Helper functions for implementing user/kernel api */
663 
664 static int
665 agp_acquire_helper(device_t dev, enum agp_acquire_state state)
666 {
667 	struct agp_softc *sc = device_get_softc(dev);
668 
669 	if (sc->as_state != AGP_ACQUIRE_FREE)
670 		return EBUSY;
671 	sc->as_state = state;
672 
673 	return 0;
674 }
675 
676 static int
677 agp_release_helper(device_t dev, enum agp_acquire_state state)
678 {
679 	struct agp_softc *sc = device_get_softc(dev);
680 
681 	if (sc->as_state == AGP_ACQUIRE_FREE)
682 		return 0;
683 
684 	if (sc->as_state != state)
685 		return EBUSY;
686 
687 	sc->as_state = AGP_ACQUIRE_FREE;
688 	return 0;
689 }
690 
691 static struct agp_memory *
692 agp_find_memory(device_t dev, int id)
693 {
694 	struct agp_softc *sc = device_get_softc(dev);
695 	struct agp_memory *mem;
696 
697 	AGP_DPF("searching for memory block %d\n", id);
698 	TAILQ_FOREACH(mem, &sc->as_memory, am_link) {
699 		AGP_DPF("considering memory block %d\n", mem->am_id);
700 		if (mem->am_id == id)
701 			return mem;
702 	}
703 	return 0;
704 }
705 
706 /* Implementation of the userland ioctl api */
707 
708 static int
709 agp_info_user(device_t dev, agp_info *info)
710 {
711 	struct agp_softc *sc = device_get_softc(dev);
712 
713 	bzero(info, sizeof *info);
714 	info->bridge_id = pci_get_devid(dev);
715 	info->agp_mode =
716 	    pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
717 	info->aper_base = rman_get_start(sc->as_aperture);
718 	info->aper_size = AGP_GET_APERTURE(dev) >> 20;
719 	info->pg_total = info->pg_system = sc->as_maxmem >> AGP_PAGE_SHIFT;
720 	info->pg_used = sc->as_allocated >> AGP_PAGE_SHIFT;
721 
722 	return 0;
723 }
724 
725 static int
726 agp_setup_user(device_t dev, agp_setup *setup)
727 {
728 	return AGP_ENABLE(dev, setup->agp_mode);
729 }
730 
731 static int
732 agp_allocate_user(device_t dev, agp_allocate *alloc)
733 {
734 	struct agp_memory *mem;
735 
736 	mem = AGP_ALLOC_MEMORY(dev,
737 			       alloc->type,
738 			       alloc->pg_count << AGP_PAGE_SHIFT);
739 	if (mem) {
740 		alloc->key = mem->am_id;
741 		alloc->physical = mem->am_physical;
742 		return 0;
743 	} else {
744 		return ENOMEM;
745 	}
746 }
747 
748 static int
749 agp_deallocate_user(device_t dev, int id)
750 {
751 	struct agp_memory *mem = agp_find_memory(dev, id);;
752 
753 	if (mem) {
754 		AGP_FREE_MEMORY(dev, mem);
755 		return 0;
756 	} else {
757 		return ENOENT;
758 	}
759 }
760 
761 static int
762 agp_bind_user(device_t dev, agp_bind *bind)
763 {
764 	struct agp_memory *mem = agp_find_memory(dev, bind->key);
765 
766 	if (!mem)
767 		return ENOENT;
768 
769 	return AGP_BIND_MEMORY(dev, mem, bind->pg_start << AGP_PAGE_SHIFT);
770 }
771 
772 static int
773 agp_unbind_user(device_t dev, agp_unbind *unbind)
774 {
775 	struct agp_memory *mem = agp_find_memory(dev, unbind->key);
776 
777 	if (!mem)
778 		return ENOENT;
779 
780 	return AGP_UNBIND_MEMORY(dev, mem);
781 }
782 
783 static int
784 agp_open(struct cdev *kdev, int oflags, int devtype, struct thread *td)
785 {
786 	device_t dev = KDEV2DEV(kdev);
787 	struct agp_softc *sc = device_get_softc(dev);
788 
789 	if (!sc->as_isopen) {
790 		sc->as_isopen = 1;
791 		device_busy(dev);
792 	}
793 
794 	return 0;
795 }
796 
797 static int
798 agp_close(struct cdev *kdev, int fflag, int devtype, struct thread *td)
799 {
800 	device_t dev = KDEV2DEV(kdev);
801 	struct agp_softc *sc = device_get_softc(dev);
802 	struct agp_memory *mem;
803 
804 	/*
805 	 * Clear the GATT and force release on last close
806 	 */
807 	while ((mem = TAILQ_FIRST(&sc->as_memory)) != 0) {
808 		if (mem->am_is_bound)
809 			AGP_UNBIND_MEMORY(dev, mem);
810 		AGP_FREE_MEMORY(dev, mem);
811 	}
812 	if (sc->as_state == AGP_ACQUIRE_USER)
813 		agp_release_helper(dev, AGP_ACQUIRE_USER);
814 	sc->as_isopen = 0;
815 	device_unbusy(dev);
816 
817 	return 0;
818 }
819 
820 static int
821 agp_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int fflag, struct thread *td)
822 {
823 	device_t dev = KDEV2DEV(kdev);
824 
825 	switch (cmd) {
826 	case AGPIOC_INFO:
827 		return agp_info_user(dev, (agp_info *) data);
828 
829 	case AGPIOC_ACQUIRE:
830 		return agp_acquire_helper(dev, AGP_ACQUIRE_USER);
831 
832 	case AGPIOC_RELEASE:
833 		return agp_release_helper(dev, AGP_ACQUIRE_USER);
834 
835 	case AGPIOC_SETUP:
836 		return agp_setup_user(dev, (agp_setup *)data);
837 
838 	case AGPIOC_ALLOCATE:
839 		return agp_allocate_user(dev, (agp_allocate *)data);
840 
841 	case AGPIOC_DEALLOCATE:
842 		return agp_deallocate_user(dev, *(int *) data);
843 
844 	case AGPIOC_BIND:
845 		return agp_bind_user(dev, (agp_bind *)data);
846 
847 	case AGPIOC_UNBIND:
848 		return agp_unbind_user(dev, (agp_unbind *)data);
849 
850 	}
851 
852 	return EINVAL;
853 }
854 
855 static int
856 agp_mmap(struct cdev *kdev, vm_offset_t offset, vm_paddr_t *paddr, int prot)
857 {
858 	device_t dev = KDEV2DEV(kdev);
859 	struct agp_softc *sc = device_get_softc(dev);
860 
861 	if (offset > AGP_GET_APERTURE(dev))
862 		return -1;
863 	*paddr = rman_get_start(sc->as_aperture) + offset;
864 	return 0;
865 }
866 
867 /* Implementation of the kernel api */
868 
869 device_t
870 agp_find_device()
871 {
872 	device_t *children, child;
873 	int i, count;
874 
875 	if (!agp_devclass)
876 		return NULL;
877 	if (devclass_get_devices(agp_devclass, &children, &count) != 0)
878 		return NULL;
879 	child = NULL;
880 	for (i = 0; i < count; i++) {
881 		if (device_is_attached(children[i])) {
882 			child = children[i];
883 			break;
884 		}
885 	}
886 	free(children, M_TEMP);
887 	return child;
888 }
889 
890 enum agp_acquire_state
891 agp_state(device_t dev)
892 {
893 	struct agp_softc *sc = device_get_softc(dev);
894 	return sc->as_state;
895 }
896 
897 void
898 agp_get_info(device_t dev, struct agp_info *info)
899 {
900 	struct agp_softc *sc = device_get_softc(dev);
901 
902 	info->ai_mode =
903 		pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
904 	info->ai_aperture_base = rman_get_start(sc->as_aperture);
905 	info->ai_aperture_size = rman_get_size(sc->as_aperture);
906 	info->ai_memory_allowed = sc->as_maxmem;
907 	info->ai_memory_used = sc->as_allocated;
908 }
909 
910 int
911 agp_acquire(device_t dev)
912 {
913 	return agp_acquire_helper(dev, AGP_ACQUIRE_KERNEL);
914 }
915 
916 int
917 agp_release(device_t dev)
918 {
919 	return agp_release_helper(dev, AGP_ACQUIRE_KERNEL);
920 }
921 
922 int
923 agp_enable(device_t dev, u_int32_t mode)
924 {
925 	return AGP_ENABLE(dev, mode);
926 }
927 
928 void *agp_alloc_memory(device_t dev, int type, vm_size_t bytes)
929 {
930 	return  (void *) AGP_ALLOC_MEMORY(dev, type, bytes);
931 }
932 
933 void agp_free_memory(device_t dev, void *handle)
934 {
935 	struct agp_memory *mem = (struct agp_memory *) handle;
936 	AGP_FREE_MEMORY(dev, mem);
937 }
938 
939 int agp_bind_memory(device_t dev, void *handle, vm_offset_t offset)
940 {
941 	struct agp_memory *mem = (struct agp_memory *) handle;
942 	return AGP_BIND_MEMORY(dev, mem, offset);
943 }
944 
945 int agp_unbind_memory(device_t dev, void *handle)
946 {
947 	struct agp_memory *mem = (struct agp_memory *) handle;
948 	return AGP_UNBIND_MEMORY(dev, mem);
949 }
950 
951 void agp_memory_info(device_t dev, void *handle, struct
952 		     agp_memory_info *mi)
953 {
954 	struct agp_memory *mem = (struct agp_memory *) handle;
955 
956 	mi->ami_size = mem->am_size;
957 	mi->ami_physical = mem->am_physical;
958 	mi->ami_offset = mem->am_offset;
959 	mi->ami_is_bound = mem->am_is_bound;
960 }
961