xref: /freebsd/sys/dev/agp/agp.c (revision 406a584d7e80c2617dc035ede0d922215a12141c)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2000 Doug Rabson
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include "opt_agp.h"
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/malloc.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/bus.h>
40 #include <sys/conf.h>
41 #include <sys/ioccom.h>
42 #include <sys/agpio.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/proc.h>
46 #include <sys/rwlock.h>
47 
48 #include <dev/agp/agppriv.h>
49 #include <dev/agp/agpvar.h>
50 #include <dev/agp/agpreg.h>
51 #include <dev/pci/pcivar.h>
52 #include <dev/pci/pcireg.h>
53 
54 #include <vm/vm.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_kern.h>
57 #include <vm/vm_param.h>
58 #include <vm/vm_object.h>
59 #include <vm/vm_page.h>
60 #include <vm/vm_pageout.h>
61 #include <vm/pmap.h>
62 
63 #include <machine/bus.h>
64 #include <machine/resource.h>
65 #include <sys/rman.h>
66 
67 MODULE_VERSION(agp, 1);
68 
69 MALLOC_DEFINE(M_AGP, "agp", "AGP data structures");
70 
71 				/* agp_drv.c */
72 static d_open_t agp_open;
73 static d_close_t agp_close;
74 static d_ioctl_t agp_ioctl;
75 static d_mmap_t agp_mmap;
76 
77 static struct cdevsw agp_cdevsw = {
78 	.d_version =	D_VERSION,
79 	.d_flags =	D_NEEDGIANT,
80 	.d_open =	agp_open,
81 	.d_close =	agp_close,
82 	.d_ioctl =	agp_ioctl,
83 	.d_mmap =	agp_mmap,
84 	.d_name =	"agp",
85 };
86 
87 static devclass_t agp_devclass;
88 
89 /* Helper functions for implementing chipset mini drivers. */
90 
91 u_int8_t
92 agp_find_caps(device_t dev)
93 {
94 	int capreg;
95 
96 
97 	if (pci_find_cap(dev, PCIY_AGP, &capreg) != 0)
98 		capreg = 0;
99 	return (capreg);
100 }
101 
102 /*
103  * Find an AGP display device (if any).
104  */
105 static device_t
106 agp_find_display(void)
107 {
108 	devclass_t pci = devclass_find("pci");
109 	device_t bus, dev = 0;
110 	device_t *kids;
111 	int busnum, numkids, i;
112 
113 	for (busnum = 0; busnum < devclass_get_maxunit(pci); busnum++) {
114 		bus = devclass_get_device(pci, busnum);
115 		if (!bus)
116 			continue;
117 		if (device_get_children(bus, &kids, &numkids) != 0)
118 			continue;
119 		for (i = 0; i < numkids; i++) {
120 			dev = kids[i];
121 			if (pci_get_class(dev) == PCIC_DISPLAY
122 			    && pci_get_subclass(dev) == PCIS_DISPLAY_VGA)
123 				if (agp_find_caps(dev)) {
124 					free(kids, M_TEMP);
125 					return dev;
126 				}
127 
128 		}
129 		free(kids, M_TEMP);
130 	}
131 
132 	return 0;
133 }
134 
135 struct agp_gatt *
136 agp_alloc_gatt(device_t dev)
137 {
138 	u_int32_t apsize = AGP_GET_APERTURE(dev);
139 	u_int32_t entries = apsize >> AGP_PAGE_SHIFT;
140 	struct agp_gatt *gatt;
141 
142 	if (bootverbose)
143 		device_printf(dev,
144 			      "allocating GATT for aperture of size %dM\n",
145 			      apsize / (1024*1024));
146 
147 	if (entries == 0) {
148 		device_printf(dev, "bad aperture size\n");
149 		return NULL;
150 	}
151 
152 	gatt = malloc(sizeof(struct agp_gatt), M_AGP, M_NOWAIT);
153 	if (!gatt)
154 		return 0;
155 
156 	gatt->ag_entries = entries;
157 	gatt->ag_virtual = (void *)kmem_alloc_contig(entries *
158 	    sizeof(u_int32_t), M_NOWAIT | M_ZERO, 0, ~0, PAGE_SIZE, 0,
159 	    VM_MEMATTR_WRITE_COMBINING);
160 	if (!gatt->ag_virtual) {
161 		if (bootverbose)
162 			device_printf(dev, "contiguous allocation failed\n");
163 		free(gatt, M_AGP);
164 		return 0;
165 	}
166 	gatt->ag_physical = vtophys((vm_offset_t) gatt->ag_virtual);
167 
168 	return gatt;
169 }
170 
171 void
172 agp_free_gatt(struct agp_gatt *gatt)
173 {
174 	kmem_free((vm_offset_t)gatt->ag_virtual, gatt->ag_entries *
175 	    sizeof(u_int32_t));
176 	free(gatt, M_AGP);
177 }
178 
179 static u_int agp_max[][2] = {
180 	{0,	0},
181 	{32,	4},
182 	{64,	28},
183 	{128,	96},
184 	{256,	204},
185 	{512,	440},
186 	{1024,	942},
187 	{2048,	1920},
188 	{4096,	3932}
189 };
190 #define	AGP_MAX_SIZE	nitems(agp_max)
191 
192 /**
193  * Sets the PCI resource which represents the AGP aperture.
194  *
195  * If not called, the default AGP aperture resource of AGP_APBASE will
196  * be used.  Must be called before agp_generic_attach().
197  */
198 void
199 agp_set_aperture_resource(device_t dev, int rid)
200 {
201 	struct agp_softc *sc = device_get_softc(dev);
202 
203 	sc->as_aperture_rid = rid;
204 }
205 
206 int
207 agp_generic_attach(device_t dev)
208 {
209 	struct agp_softc *sc = device_get_softc(dev);
210 	int i;
211 	u_int memsize;
212 
213 	/*
214 	 * Find and map the aperture, RF_SHAREABLE for DRM but not RF_ACTIVE
215 	 * because the kernel doesn't need to map it.
216 	 */
217 
218 	if (sc->as_aperture_rid != -1) {
219 		if (sc->as_aperture_rid == 0)
220 			sc->as_aperture_rid = AGP_APBASE;
221 
222 		sc->as_aperture = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
223 		    &sc->as_aperture_rid, RF_SHAREABLE);
224 		if (!sc->as_aperture)
225 			return ENOMEM;
226 	}
227 
228 	/*
229 	 * Work out an upper bound for agp memory allocation. This
230 	 * uses a heurisitc table from the Linux driver.
231 	 */
232 	memsize = ptoa(realmem) >> 20;
233 	for (i = 0; i < AGP_MAX_SIZE; i++) {
234 		if (memsize <= agp_max[i][0])
235 			break;
236 	}
237 	if (i == AGP_MAX_SIZE)
238 		i = AGP_MAX_SIZE - 1;
239 	sc->as_maxmem = agp_max[i][1] << 20U;
240 
241 	/*
242 	 * The lock is used to prevent re-entry to
243 	 * agp_generic_bind_memory() since that function can sleep.
244 	 */
245 	mtx_init(&sc->as_lock, "agp lock", NULL, MTX_DEF);
246 
247 	/*
248 	 * Initialise stuff for the userland device.
249 	 */
250 	agp_devclass = devclass_find("agp");
251 	TAILQ_INIT(&sc->as_memory);
252 	sc->as_nextid = 1;
253 
254 	sc->as_devnode = make_dev(&agp_cdevsw,
255 	    0, UID_ROOT, GID_WHEEL, 0600, "agpgart");
256 	sc->as_devnode->si_drv1 = dev;
257 
258 	return 0;
259 }
260 
261 void
262 agp_free_cdev(device_t dev)
263 {
264 	struct agp_softc *sc = device_get_softc(dev);
265 
266 	destroy_dev(sc->as_devnode);
267 }
268 
269 void
270 agp_free_res(device_t dev)
271 {
272 	struct agp_softc *sc = device_get_softc(dev);
273 
274 	if (sc->as_aperture != NULL)
275 		bus_release_resource(dev, SYS_RES_MEMORY, sc->as_aperture_rid,
276 		    sc->as_aperture);
277 	mtx_destroy(&sc->as_lock);
278 }
279 
280 int
281 agp_generic_detach(device_t dev)
282 {
283 
284 	agp_free_cdev(dev);
285 	agp_free_res(dev);
286 	return 0;
287 }
288 
289 /**
290  * Default AGP aperture size detection which simply returns the size of
291  * the aperture's PCI resource.
292  */
293 u_int32_t
294 agp_generic_get_aperture(device_t dev)
295 {
296 	struct agp_softc *sc = device_get_softc(dev);
297 
298 	return rman_get_size(sc->as_aperture);
299 }
300 
301 /**
302  * Default AGP aperture size setting function, which simply doesn't allow
303  * changes to resource size.
304  */
305 int
306 agp_generic_set_aperture(device_t dev, u_int32_t aperture)
307 {
308 	u_int32_t current_aperture;
309 
310 	current_aperture = AGP_GET_APERTURE(dev);
311 	if (current_aperture != aperture)
312 		return EINVAL;
313 	else
314 		return 0;
315 }
316 
317 /*
318  * This does the enable logic for v3, with the same topology
319  * restrictions as in place for v2 -- one bus, one device on the bus.
320  */
321 static int
322 agp_v3_enable(device_t dev, device_t mdev, u_int32_t mode)
323 {
324 	u_int32_t tstatus, mstatus;
325 	u_int32_t command;
326 	int rq, sba, fw, rate, arqsz, cal;
327 
328 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
329 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
330 
331 	/* Set RQ to the min of mode, tstatus and mstatus */
332 	rq = AGP_MODE_GET_RQ(mode);
333 	if (AGP_MODE_GET_RQ(tstatus) < rq)
334 		rq = AGP_MODE_GET_RQ(tstatus);
335 	if (AGP_MODE_GET_RQ(mstatus) < rq)
336 		rq = AGP_MODE_GET_RQ(mstatus);
337 
338 	/*
339 	 * ARQSZ - Set the value to the maximum one.
340 	 * Don't allow the mode register to override values.
341 	 */
342 	arqsz = AGP_MODE_GET_ARQSZ(mode);
343 	if (AGP_MODE_GET_ARQSZ(tstatus) > rq)
344 		rq = AGP_MODE_GET_ARQSZ(tstatus);
345 	if (AGP_MODE_GET_ARQSZ(mstatus) > rq)
346 		rq = AGP_MODE_GET_ARQSZ(mstatus);
347 
348 	/* Calibration cycle - don't allow override by mode register */
349 	cal = AGP_MODE_GET_CAL(tstatus);
350 	if (AGP_MODE_GET_CAL(mstatus) < cal)
351 		cal = AGP_MODE_GET_CAL(mstatus);
352 
353 	/* SBA must be supported for AGP v3. */
354 	sba = 1;
355 
356 	/* Set FW if all three support it. */
357 	fw = (AGP_MODE_GET_FW(tstatus)
358 	       & AGP_MODE_GET_FW(mstatus)
359 	       & AGP_MODE_GET_FW(mode));
360 
361 	/* Figure out the max rate */
362 	rate = (AGP_MODE_GET_RATE(tstatus)
363 		& AGP_MODE_GET_RATE(mstatus)
364 		& AGP_MODE_GET_RATE(mode));
365 	if (rate & AGP_MODE_V3_RATE_8x)
366 		rate = AGP_MODE_V3_RATE_8x;
367 	else
368 		rate = AGP_MODE_V3_RATE_4x;
369 	if (bootverbose)
370 		device_printf(dev, "Setting AGP v3 mode %d\n", rate * 4);
371 
372 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, 0, 4);
373 
374 	/* Construct the new mode word and tell the hardware */
375 	command = 0;
376 	command = AGP_MODE_SET_RQ(0, rq);
377 	command = AGP_MODE_SET_ARQSZ(command, arqsz);
378 	command = AGP_MODE_SET_CAL(command, cal);
379 	command = AGP_MODE_SET_SBA(command, sba);
380 	command = AGP_MODE_SET_FW(command, fw);
381 	command = AGP_MODE_SET_RATE(command, rate);
382 	command = AGP_MODE_SET_MODE_3(command, 1);
383 	command = AGP_MODE_SET_AGP(command, 1);
384 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, command, 4);
385 	pci_write_config(mdev, agp_find_caps(mdev) + AGP_COMMAND, command, 4);
386 
387 	return 0;
388 }
389 
390 static int
391 agp_v2_enable(device_t dev, device_t mdev, u_int32_t mode)
392 {
393 	u_int32_t tstatus, mstatus;
394 	u_int32_t command;
395 	int rq, sba, fw, rate;
396 
397 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
398 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
399 
400 	/* Set RQ to the min of mode, tstatus and mstatus */
401 	rq = AGP_MODE_GET_RQ(mode);
402 	if (AGP_MODE_GET_RQ(tstatus) < rq)
403 		rq = AGP_MODE_GET_RQ(tstatus);
404 	if (AGP_MODE_GET_RQ(mstatus) < rq)
405 		rq = AGP_MODE_GET_RQ(mstatus);
406 
407 	/* Set SBA if all three can deal with SBA */
408 	sba = (AGP_MODE_GET_SBA(tstatus)
409 	       & AGP_MODE_GET_SBA(mstatus)
410 	       & AGP_MODE_GET_SBA(mode));
411 
412 	/* Similar for FW */
413 	fw = (AGP_MODE_GET_FW(tstatus)
414 	       & AGP_MODE_GET_FW(mstatus)
415 	       & AGP_MODE_GET_FW(mode));
416 
417 	/* Figure out the max rate */
418 	rate = (AGP_MODE_GET_RATE(tstatus)
419 		& AGP_MODE_GET_RATE(mstatus)
420 		& AGP_MODE_GET_RATE(mode));
421 	if (rate & AGP_MODE_V2_RATE_4x)
422 		rate = AGP_MODE_V2_RATE_4x;
423 	else if (rate & AGP_MODE_V2_RATE_2x)
424 		rate = AGP_MODE_V2_RATE_2x;
425 	else
426 		rate = AGP_MODE_V2_RATE_1x;
427 	if (bootverbose)
428 		device_printf(dev, "Setting AGP v2 mode %d\n", rate);
429 
430 	/* Construct the new mode word and tell the hardware */
431 	command = 0;
432 	command = AGP_MODE_SET_RQ(0, rq);
433 	command = AGP_MODE_SET_SBA(command, sba);
434 	command = AGP_MODE_SET_FW(command, fw);
435 	command = AGP_MODE_SET_RATE(command, rate);
436 	command = AGP_MODE_SET_AGP(command, 1);
437 	pci_write_config(dev, agp_find_caps(dev) + AGP_COMMAND, command, 4);
438 	pci_write_config(mdev, agp_find_caps(mdev) + AGP_COMMAND, command, 4);
439 
440 	return 0;
441 }
442 
443 int
444 agp_generic_enable(device_t dev, u_int32_t mode)
445 {
446 	device_t mdev = agp_find_display();
447 	u_int32_t tstatus, mstatus;
448 
449 	if (!mdev) {
450 		AGP_DPF("can't find display\n");
451 		return ENXIO;
452 	}
453 
454 	tstatus = pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
455 	mstatus = pci_read_config(mdev, agp_find_caps(mdev) + AGP_STATUS, 4);
456 
457 	/*
458 	 * Check display and bridge for AGP v3 support.  AGP v3 allows
459 	 * more variety in topology than v2, e.g. multiple AGP devices
460 	 * attached to one bridge, or multiple AGP bridges in one
461 	 * system.  This doesn't attempt to address those situations,
462 	 * but should work fine for a classic single AGP slot system
463 	 * with AGP v3.
464 	 */
465 	if (AGP_MODE_GET_MODE_3(mode) &&
466 	    AGP_MODE_GET_MODE_3(tstatus) &&
467 	    AGP_MODE_GET_MODE_3(mstatus))
468 		return (agp_v3_enable(dev, mdev, mode));
469 	else
470 		return (agp_v2_enable(dev, mdev, mode));
471 }
472 
473 struct agp_memory *
474 agp_generic_alloc_memory(device_t dev, int type, vm_size_t size)
475 {
476 	struct agp_softc *sc = device_get_softc(dev);
477 	struct agp_memory *mem;
478 
479 	if ((size & (AGP_PAGE_SIZE - 1)) != 0)
480 		return 0;
481 
482 	if (size > sc->as_maxmem - sc->as_allocated)
483 		return 0;
484 
485 	if (type != 0) {
486 		printf("agp_generic_alloc_memory: unsupported type %d\n",
487 		       type);
488 		return 0;
489 	}
490 
491 	mem = malloc(sizeof *mem, M_AGP, M_WAITOK);
492 	mem->am_id = sc->as_nextid++;
493 	mem->am_size = size;
494 	mem->am_type = 0;
495 	mem->am_obj = vm_object_allocate(OBJT_DEFAULT, atop(round_page(size)));
496 	mem->am_physical = 0;
497 	mem->am_offset = 0;
498 	mem->am_is_bound = 0;
499 	TAILQ_INSERT_TAIL(&sc->as_memory, mem, am_link);
500 	sc->as_allocated += size;
501 
502 	return mem;
503 }
504 
505 int
506 agp_generic_free_memory(device_t dev, struct agp_memory *mem)
507 {
508 	struct agp_softc *sc = device_get_softc(dev);
509 
510 	if (mem->am_is_bound)
511 		return EBUSY;
512 
513 	sc->as_allocated -= mem->am_size;
514 	TAILQ_REMOVE(&sc->as_memory, mem, am_link);
515 	vm_object_deallocate(mem->am_obj);
516 	free(mem, M_AGP);
517 	return 0;
518 }
519 
520 int
521 agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
522 			vm_offset_t offset)
523 {
524 	struct agp_softc *sc = device_get_softc(dev);
525 	vm_offset_t i, j, k;
526 	vm_page_t m;
527 	int error;
528 
529 	/* Do some sanity checks first. */
530 	if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
531 	    offset + mem->am_size > AGP_GET_APERTURE(dev)) {
532 		device_printf(dev, "binding memory at bad offset %#x\n",
533 		    (int)offset);
534 		return EINVAL;
535 	}
536 
537 	/*
538 	 * Allocate the pages early, before acquiring the lock,
539 	 * because vm_page_grab() may sleep and we can't hold a mutex
540 	 * while sleeping.
541 	 */
542 	VM_OBJECT_WLOCK(mem->am_obj);
543 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
544 		/*
545 		 * Find a page from the object and wire it
546 		 * down. This page will be mapped using one or more
547 		 * entries in the GATT (assuming that PAGE_SIZE >=
548 		 * AGP_PAGE_SIZE. If this is the first call to bind,
549 		 * the pages will be allocated and zeroed.
550 		 */
551 		m = vm_page_grab(mem->am_obj, OFF_TO_IDX(i),
552 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO);
553 		AGP_DPF("found page pa=%#jx\n", (uintmax_t)VM_PAGE_TO_PHYS(m));
554 	}
555 	VM_OBJECT_WUNLOCK(mem->am_obj);
556 
557 	mtx_lock(&sc->as_lock);
558 
559 	if (mem->am_is_bound) {
560 		device_printf(dev, "memory already bound\n");
561 		error = EINVAL;
562 		VM_OBJECT_WLOCK(mem->am_obj);
563 		i = 0;
564 		goto bad;
565 	}
566 
567 	/*
568 	 * Bind the individual pages and flush the chipset's
569 	 * TLB.
570 	 */
571 	VM_OBJECT_WLOCK(mem->am_obj);
572 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
573 		m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(i));
574 
575 		/*
576 		 * Install entries in the GATT, making sure that if
577 		 * AGP_PAGE_SIZE < PAGE_SIZE and mem->am_size is not
578 		 * aligned to PAGE_SIZE, we don't modify too many GATT
579 		 * entries.
580 		 */
581 		for (j = 0; j < PAGE_SIZE && i + j < mem->am_size;
582 		     j += AGP_PAGE_SIZE) {
583 			vm_offset_t pa = VM_PAGE_TO_PHYS(m) + j;
584 			AGP_DPF("binding offset %#jx to pa %#jx\n",
585 				(uintmax_t)offset + i + j, (uintmax_t)pa);
586 			error = AGP_BIND_PAGE(dev, offset + i + j, pa);
587 			if (error) {
588 				/*
589 				 * Bail out. Reverse all the mappings
590 				 * and unwire the pages.
591 				 */
592 				for (k = 0; k < i + j; k += AGP_PAGE_SIZE)
593 					AGP_UNBIND_PAGE(dev, offset + k);
594 				goto bad;
595 			}
596 		}
597 		vm_page_xunbusy(m);
598 	}
599 	VM_OBJECT_WUNLOCK(mem->am_obj);
600 
601 	/*
602 	 * Make sure the chipset gets the new mappings.
603 	 */
604 	AGP_FLUSH_TLB(dev);
605 
606 	mem->am_offset = offset;
607 	mem->am_is_bound = 1;
608 
609 	mtx_unlock(&sc->as_lock);
610 
611 	return 0;
612 bad:
613 	mtx_unlock(&sc->as_lock);
614 	VM_OBJECT_ASSERT_WLOCKED(mem->am_obj);
615 	for (k = 0; k < mem->am_size; k += PAGE_SIZE) {
616 		m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(k));
617 		if (k >= i)
618 			vm_page_xunbusy(m);
619 		vm_page_lock(m);
620 		vm_page_unwire(m, PQ_INACTIVE);
621 		vm_page_unlock(m);
622 	}
623 	VM_OBJECT_WUNLOCK(mem->am_obj);
624 
625 	return error;
626 }
627 
628 int
629 agp_generic_unbind_memory(device_t dev, struct agp_memory *mem)
630 {
631 	struct agp_softc *sc = device_get_softc(dev);
632 	vm_page_t m;
633 	int i;
634 
635 	mtx_lock(&sc->as_lock);
636 
637 	if (!mem->am_is_bound) {
638 		device_printf(dev, "memory is not bound\n");
639 		mtx_unlock(&sc->as_lock);
640 		return EINVAL;
641 	}
642 
643 
644 	/*
645 	 * Unbind the individual pages and flush the chipset's
646 	 * TLB. Unwire the pages so they can be swapped.
647 	 */
648 	for (i = 0; i < mem->am_size; i += AGP_PAGE_SIZE)
649 		AGP_UNBIND_PAGE(dev, mem->am_offset + i);
650 
651 	AGP_FLUSH_TLB(dev);
652 
653 	VM_OBJECT_WLOCK(mem->am_obj);
654 	for (i = 0; i < mem->am_size; i += PAGE_SIZE) {
655 		m = vm_page_lookup(mem->am_obj, atop(i));
656 		vm_page_lock(m);
657 		vm_page_unwire(m, PQ_INACTIVE);
658 		vm_page_unlock(m);
659 	}
660 	VM_OBJECT_WUNLOCK(mem->am_obj);
661 
662 	mem->am_offset = 0;
663 	mem->am_is_bound = 0;
664 
665 	mtx_unlock(&sc->as_lock);
666 
667 	return 0;
668 }
669 
670 /* Helper functions for implementing user/kernel api */
671 
672 static int
673 agp_acquire_helper(device_t dev, enum agp_acquire_state state)
674 {
675 	struct agp_softc *sc = device_get_softc(dev);
676 
677 	if (sc->as_state != AGP_ACQUIRE_FREE)
678 		return EBUSY;
679 	sc->as_state = state;
680 
681 	return 0;
682 }
683 
684 static int
685 agp_release_helper(device_t dev, enum agp_acquire_state state)
686 {
687 	struct agp_softc *sc = device_get_softc(dev);
688 
689 	if (sc->as_state == AGP_ACQUIRE_FREE)
690 		return 0;
691 
692 	if (sc->as_state != state)
693 		return EBUSY;
694 
695 	sc->as_state = AGP_ACQUIRE_FREE;
696 	return 0;
697 }
698 
699 static struct agp_memory *
700 agp_find_memory(device_t dev, int id)
701 {
702 	struct agp_softc *sc = device_get_softc(dev);
703 	struct agp_memory *mem;
704 
705 	AGP_DPF("searching for memory block %d\n", id);
706 	TAILQ_FOREACH(mem, &sc->as_memory, am_link) {
707 		AGP_DPF("considering memory block %d\n", mem->am_id);
708 		if (mem->am_id == id)
709 			return mem;
710 	}
711 	return 0;
712 }
713 
714 /* Implementation of the userland ioctl api */
715 
716 static int
717 agp_info_user(device_t dev, agp_info *info)
718 {
719 	struct agp_softc *sc = device_get_softc(dev);
720 
721 	bzero(info, sizeof *info);
722 	info->bridge_id = pci_get_devid(dev);
723 	info->agp_mode =
724 	    pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
725 	if (sc->as_aperture)
726 		info->aper_base = rman_get_start(sc->as_aperture);
727 	else
728 		info->aper_base = 0;
729 	info->aper_size = AGP_GET_APERTURE(dev) >> 20;
730 	info->pg_total = info->pg_system = sc->as_maxmem >> AGP_PAGE_SHIFT;
731 	info->pg_used = sc->as_allocated >> AGP_PAGE_SHIFT;
732 
733 	return 0;
734 }
735 
736 static int
737 agp_setup_user(device_t dev, agp_setup *setup)
738 {
739 	return AGP_ENABLE(dev, setup->agp_mode);
740 }
741 
742 static int
743 agp_allocate_user(device_t dev, agp_allocate *alloc)
744 {
745 	struct agp_memory *mem;
746 
747 	mem = AGP_ALLOC_MEMORY(dev,
748 			       alloc->type,
749 			       alloc->pg_count << AGP_PAGE_SHIFT);
750 	if (mem) {
751 		alloc->key = mem->am_id;
752 		alloc->physical = mem->am_physical;
753 		return 0;
754 	} else {
755 		return ENOMEM;
756 	}
757 }
758 
759 static int
760 agp_deallocate_user(device_t dev, int id)
761 {
762 	struct agp_memory *mem = agp_find_memory(dev, id);
763 
764 	if (mem) {
765 		AGP_FREE_MEMORY(dev, mem);
766 		return 0;
767 	} else {
768 		return ENOENT;
769 	}
770 }
771 
772 static int
773 agp_bind_user(device_t dev, agp_bind *bind)
774 {
775 	struct agp_memory *mem = agp_find_memory(dev, bind->key);
776 
777 	if (!mem)
778 		return ENOENT;
779 
780 	return AGP_BIND_MEMORY(dev, mem, bind->pg_start << AGP_PAGE_SHIFT);
781 }
782 
783 static int
784 agp_unbind_user(device_t dev, agp_unbind *unbind)
785 {
786 	struct agp_memory *mem = agp_find_memory(dev, unbind->key);
787 
788 	if (!mem)
789 		return ENOENT;
790 
791 	return AGP_UNBIND_MEMORY(dev, mem);
792 }
793 
794 static int
795 agp_chipset_flush(device_t dev)
796 {
797 
798 	return (AGP_CHIPSET_FLUSH(dev));
799 }
800 
801 static int
802 agp_open(struct cdev *kdev, int oflags, int devtype, struct thread *td)
803 {
804 	device_t dev = kdev->si_drv1;
805 	struct agp_softc *sc = device_get_softc(dev);
806 
807 	if (!sc->as_isopen) {
808 		sc->as_isopen = 1;
809 		device_busy(dev);
810 	}
811 
812 	return 0;
813 }
814 
815 static int
816 agp_close(struct cdev *kdev, int fflag, int devtype, struct thread *td)
817 {
818 	device_t dev = kdev->si_drv1;
819 	struct agp_softc *sc = device_get_softc(dev);
820 	struct agp_memory *mem;
821 
822 	/*
823 	 * Clear the GATT and force release on last close
824 	 */
825 	while ((mem = TAILQ_FIRST(&sc->as_memory)) != NULL) {
826 		if (mem->am_is_bound)
827 			AGP_UNBIND_MEMORY(dev, mem);
828 		AGP_FREE_MEMORY(dev, mem);
829 	}
830 	if (sc->as_state == AGP_ACQUIRE_USER)
831 		agp_release_helper(dev, AGP_ACQUIRE_USER);
832 	sc->as_isopen = 0;
833 	device_unbusy(dev);
834 
835 	return 0;
836 }
837 
838 static int
839 agp_ioctl(struct cdev *kdev, u_long cmd, caddr_t data, int fflag, struct thread *td)
840 {
841 	device_t dev = kdev->si_drv1;
842 
843 	switch (cmd) {
844 	case AGPIOC_INFO:
845 		return agp_info_user(dev, (agp_info *) data);
846 
847 	case AGPIOC_ACQUIRE:
848 		return agp_acquire_helper(dev, AGP_ACQUIRE_USER);
849 
850 	case AGPIOC_RELEASE:
851 		return agp_release_helper(dev, AGP_ACQUIRE_USER);
852 
853 	case AGPIOC_SETUP:
854 		return agp_setup_user(dev, (agp_setup *)data);
855 
856 	case AGPIOC_ALLOCATE:
857 		return agp_allocate_user(dev, (agp_allocate *)data);
858 
859 	case AGPIOC_DEALLOCATE:
860 		return agp_deallocate_user(dev, *(int *) data);
861 
862 	case AGPIOC_BIND:
863 		return agp_bind_user(dev, (agp_bind *)data);
864 
865 	case AGPIOC_UNBIND:
866 		return agp_unbind_user(dev, (agp_unbind *)data);
867 
868 	case AGPIOC_CHIPSET_FLUSH:
869 		return agp_chipset_flush(dev);
870 	}
871 
872 	return EINVAL;
873 }
874 
875 static int
876 agp_mmap(struct cdev *kdev, vm_ooffset_t offset, vm_paddr_t *paddr,
877     int prot, vm_memattr_t *memattr)
878 {
879 	device_t dev = kdev->si_drv1;
880 	struct agp_softc *sc = device_get_softc(dev);
881 
882 	if (offset > AGP_GET_APERTURE(dev))
883 		return -1;
884 	if (sc->as_aperture == NULL)
885 		return -1;
886 	*paddr = rman_get_start(sc->as_aperture) + offset;
887 	return 0;
888 }
889 
890 /* Implementation of the kernel api */
891 
892 device_t
893 agp_find_device()
894 {
895 	device_t *children, child;
896 	int i, count;
897 
898 	if (!agp_devclass)
899 		return NULL;
900 	if (devclass_get_devices(agp_devclass, &children, &count) != 0)
901 		return NULL;
902 	child = NULL;
903 	for (i = 0; i < count; i++) {
904 		if (device_is_attached(children[i])) {
905 			child = children[i];
906 			break;
907 		}
908 	}
909 	free(children, M_TEMP);
910 	return child;
911 }
912 
913 enum agp_acquire_state
914 agp_state(device_t dev)
915 {
916 	struct agp_softc *sc = device_get_softc(dev);
917 	return sc->as_state;
918 }
919 
920 void
921 agp_get_info(device_t dev, struct agp_info *info)
922 {
923 	struct agp_softc *sc = device_get_softc(dev);
924 
925 	info->ai_mode =
926 		pci_read_config(dev, agp_find_caps(dev) + AGP_STATUS, 4);
927 	if (sc->as_aperture != NULL)
928 		info->ai_aperture_base = rman_get_start(sc->as_aperture);
929 	else
930 		info->ai_aperture_base = 0;
931 	info->ai_aperture_size = AGP_GET_APERTURE(dev);
932 	info->ai_memory_allowed = sc->as_maxmem;
933 	info->ai_memory_used = sc->as_allocated;
934 }
935 
936 int
937 agp_acquire(device_t dev)
938 {
939 	return agp_acquire_helper(dev, AGP_ACQUIRE_KERNEL);
940 }
941 
942 int
943 agp_release(device_t dev)
944 {
945 	return agp_release_helper(dev, AGP_ACQUIRE_KERNEL);
946 }
947 
948 int
949 agp_enable(device_t dev, u_int32_t mode)
950 {
951 	return AGP_ENABLE(dev, mode);
952 }
953 
954 void *agp_alloc_memory(device_t dev, int type, vm_size_t bytes)
955 {
956 	return  (void *) AGP_ALLOC_MEMORY(dev, type, bytes);
957 }
958 
959 void agp_free_memory(device_t dev, void *handle)
960 {
961 	struct agp_memory *mem = (struct agp_memory *) handle;
962 	AGP_FREE_MEMORY(dev, mem);
963 }
964 
965 int agp_bind_memory(device_t dev, void *handle, vm_offset_t offset)
966 {
967 	struct agp_memory *mem = (struct agp_memory *) handle;
968 	return AGP_BIND_MEMORY(dev, mem, offset);
969 }
970 
971 int agp_unbind_memory(device_t dev, void *handle)
972 {
973 	struct agp_memory *mem = (struct agp_memory *) handle;
974 	return AGP_UNBIND_MEMORY(dev, mem);
975 }
976 
977 void agp_memory_info(device_t dev, void *handle, struct
978 		     agp_memory_info *mi)
979 {
980 	struct agp_memory *mem = (struct agp_memory *) handle;
981 
982 	mi->ami_size = mem->am_size;
983 	mi->ami_physical = mem->am_physical;
984 	mi->ami_offset = mem->am_offset;
985 	mi->ami_is_bound = mem->am_is_bound;
986 }
987 
988 int
989 agp_bind_pages(device_t dev, vm_page_t *pages, vm_size_t size,
990     vm_offset_t offset)
991 {
992 	struct agp_softc *sc;
993 	vm_offset_t i, j, k, pa;
994 	vm_page_t m;
995 	int error;
996 
997 	if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
998 	    (offset & (AGP_PAGE_SIZE - 1)) != 0)
999 		return (EINVAL);
1000 
1001 	sc = device_get_softc(dev);
1002 
1003 	mtx_lock(&sc->as_lock);
1004 	for (i = 0; i < size; i += PAGE_SIZE) {
1005 		m = pages[OFF_TO_IDX(i)];
1006 		KASSERT(m->wire_count > 0,
1007 		    ("agp_bind_pages: page %p hasn't been wired", m));
1008 
1009 		/*
1010 		 * Install entries in the GATT, making sure that if
1011 		 * AGP_PAGE_SIZE < PAGE_SIZE and size is not
1012 		 * aligned to PAGE_SIZE, we don't modify too many GATT
1013 		 * entries.
1014 		 */
1015 		for (j = 0; j < PAGE_SIZE && i + j < size; j += AGP_PAGE_SIZE) {
1016 			pa = VM_PAGE_TO_PHYS(m) + j;
1017 			AGP_DPF("binding offset %#jx to pa %#jx\n",
1018 				(uintmax_t)offset + i + j, (uintmax_t)pa);
1019 			error = AGP_BIND_PAGE(dev, offset + i + j, pa);
1020 			if (error) {
1021 				/*
1022 				 * Bail out. Reverse all the mappings.
1023 				 */
1024 				for (k = 0; k < i + j; k += AGP_PAGE_SIZE)
1025 					AGP_UNBIND_PAGE(dev, offset + k);
1026 
1027 				mtx_unlock(&sc->as_lock);
1028 				return (error);
1029 			}
1030 		}
1031 	}
1032 
1033 	AGP_FLUSH_TLB(dev);
1034 
1035 	mtx_unlock(&sc->as_lock);
1036 	return (0);
1037 }
1038 
1039 int
1040 agp_unbind_pages(device_t dev, vm_size_t size, vm_offset_t offset)
1041 {
1042 	struct agp_softc *sc;
1043 	vm_offset_t i;
1044 
1045 	if ((size & (AGP_PAGE_SIZE - 1)) != 0 ||
1046 	    (offset & (AGP_PAGE_SIZE - 1)) != 0)
1047 		return (EINVAL);
1048 
1049 	sc = device_get_softc(dev);
1050 
1051 	mtx_lock(&sc->as_lock);
1052 	for (i = 0; i < size; i += AGP_PAGE_SIZE)
1053 		AGP_UNBIND_PAGE(dev, offset + i);
1054 
1055 	AGP_FLUSH_TLB(dev);
1056 
1057 	mtx_unlock(&sc->as_lock);
1058 	return (0);
1059 }
1060