1 /* 2 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 3 * Use is subject to license terms. 4 */ 5 /* 6 * Portions Philip Brown phil@bolthole.com Dec 2001 7 */ 8 9 #pragma ident "%Z%%M% %I% %E% SMI" 10 11 /* 12 * agpgart driver 13 * 14 * This driver is primary targeted at providing memory support for INTEL 15 * AGP device, INTEL memory less video card, and AMD64 cpu GART devices. 16 * So there are four main architectures, ARC_IGD810, ARC_IGD830, ARC_INTELAGP, 17 * ARC_AMD64AGP to agpgart driver. However, the memory 18 * interfaces are the same for these architectures. The difference is how to 19 * manage the hardware GART table for them. 20 * 21 * For large memory allocation, this driver use direct mapping to userland 22 * application interface to save kernel virtual memory . 23 */ 24 25 #include <sys/types.h> 26 #include <sys/pci.h> 27 #include <sys/systm.h> 28 #include <sys/conf.h> 29 #include <sys/file.h> 30 #include <sys/kstat.h> 31 #include <sys/stat.h> 32 #include <sys/modctl.h> 33 #include <sys/ddi.h> 34 #include <sys/sunddi.h> 35 #include <sys/sunldi.h> 36 #include <sys/policy.h> 37 #include <sys/ddidevmap.h> 38 #include <vm/seg_dev.h> 39 #include <sys/pmem.h> 40 #include <sys/agpgart.h> 41 #include <sys/agp/agpdefs.h> 42 #include <sys/agp/agpgart_impl.h> 43 #include <sys/agp/agpamd64gart_io.h> 44 #include <sys/agp/agpmaster_io.h> 45 #include <sys/agp/agptarget_io.h> 46 47 /* Dynamic debug support */ 48 int agp_debug_var = 0; 49 #define AGPDB_PRINT1(fmt) if (agp_debug_var == 1) cmn_err fmt 50 #define AGPDB_PRINT2(fmt) if (agp_debug_var >= 1) cmn_err fmt 51 52 /* Driver global softstate handle */ 53 static void *agpgart_glob_soft_handle; 54 55 #define MAX_INSTNUM 16 56 57 #define AGP_DEV2INST(devt) (getminor((devt)) >> 4) 58 #define AGP_INST2MINOR(instance) ((instance) << 4) 59 #define IS_INTEL_830(type) ((type) == ARC_IGD830) 60 #define IS_TRUE_AGP(type) (((type) == ARC_INTELAGP) || \ 61 ((type) == ARC_AMD64AGP)) 62 63 #define agpinfo_default_to_32(v, v32) \ 64 { \ 65 (v32).agpi32_version = (v).agpi_version; \ 66 (v32).agpi32_devid = (v).agpi_devid; \ 67 (v32).agpi32_mode = (v).agpi_mode; \ 68 (v32).agpi32_aperbase = (v).agpi_aperbase; \ 69 (v32).agpi32_apersize = (v).agpi_apersize; \ 70 (v32).agpi32_pgtotal = (v).agpi_pgtotal; \ 71 (v32).agpi32_pgsystem = (v).agpi_pgsystem; \ 72 (v32).agpi32_pgused = (v).agpi_pgused; \ 73 } 74 75 static ddi_dma_attr_t agpgart_dma_attr = { 76 DMA_ATTR_V0, 77 0U, /* dma_attr_addr_lo */ 78 0xffffffffU, /* dma_attr_addr_hi */ 79 0xffffffffU, /* dma_attr_count_max */ 80 (uint64_t)AGP_PAGE_SIZE, /* dma_attr_align */ 81 1, /* dma_attr_burstsizes */ 82 1, /* dma_attr_minxfer */ 83 0xffffffffU, /* dma_attr_maxxfer */ 84 0xffffffffU, /* dma_attr_seg */ 85 1, /* dma_attr_sgllen, variable */ 86 4, /* dma_attr_granular */ 87 0 /* dma_attr_flags */ 88 }; 89 90 /* 91 * AMD64 supports gart table above 4G. See alloc_gart_table. 92 */ 93 static ddi_dma_attr_t garttable_dma_attr = { 94 DMA_ATTR_V0, 95 0U, /* dma_attr_addr_lo */ 96 0xffffffffU, /* dma_attr_addr_hi */ 97 0xffffffffU, /* dma_attr_count_max */ 98 (uint64_t)AGP_PAGE_SIZE, /* dma_attr_align */ 99 1, /* dma_attr_burstsizes */ 100 1, /* dma_attr_minxfer */ 101 0xffffffffU, /* dma_attr_maxxfer */ 102 0xffffffffU, /* dma_attr_seg */ 103 1, /* dma_attr_sgllen, variable */ 104 4, /* dma_attr_granular */ 105 0 /* dma_attr_flags */ 106 }; 107 108 /* 109 * AGPGART table need a physical contiguous memory. To assure that 110 * each access to gart table is strongly ordered and uncachable, 111 * we use DDI_STRICTORDER_ACC. 112 */ 113 static ddi_device_acc_attr_t gart_dev_acc_attr = { 114 DDI_DEVICE_ATTR_V0, 115 DDI_NEVERSWAP_ACC, 116 DDI_STRICTORDER_ACC /* must be DDI_STRICTORDER_ACC */ 117 }; 118 119 /* 120 * AGP memory is usually used as texture memory or for a framebuffer, so we 121 * can set the memory attribute to write combining. Video drivers will 122 * determine the frame buffer attributes, for example the memory is write 123 * combinging or non-cachable. However, the interface between Xorg and agpgart 124 * driver to support attribute selcetion doesn't exist yet. So we set agp memory 125 * to non-cachable by default now. This attribute might be overridden 126 * by MTTR in X86. 127 */ 128 static ddi_device_acc_attr_t mem_dev_acc_attr = { 129 DDI_DEVICE_ATTR_V0, 130 DDI_NEVERSWAP_ACC, 131 DDI_STRICTORDER_ACC /* Can be DDI_MERGING_OK_ACC */ 132 }; 133 134 static keytable_ent_t * 135 agp_find_bound_keyent(agpgart_softstate_t *softstate, uint32_t pg_offset); 136 static void 137 amd64_gart_unregister(amd64_garts_dev_t *cpu_garts); 138 139 140 static void 141 agp_devmap_unmap(devmap_cookie_t handle, void *devprivate, 142 offset_t off, size_t len, devmap_cookie_t new_handle1, 143 void **new_devprivate1, devmap_cookie_t new_handle2, 144 void **new_devprivate2) 145 { 146 147 struct keytable_ent *mementry; 148 agpgart_softstate_t *softstate; 149 agpgart_ctx_t *ctxp, *newctxp1, *newctxp2; 150 151 ASSERT(AGP_ALIGNED(len) && AGP_ALIGNED(off)); 152 ASSERT(devprivate); 153 ASSERT(handle); 154 155 ctxp = (agpgart_ctx_t *)devprivate; 156 softstate = ctxp->actx_sc; 157 ASSERT(softstate); 158 159 if (new_handle1 != NULL) { 160 newctxp1 = kmem_zalloc(sizeof (agpgart_ctx_t), KM_SLEEP); 161 newctxp1->actx_sc = softstate; 162 newctxp1->actx_off = ctxp->actx_off; 163 *new_devprivate1 = newctxp1; 164 } 165 166 if (new_handle2 != NULL) { 167 newctxp2 = kmem_zalloc(sizeof (agpgart_ctx_t), KM_SLEEP); 168 newctxp2->actx_sc = softstate; 169 newctxp2->actx_off = off + len; 170 *new_devprivate2 = newctxp2; 171 } 172 173 mutex_enter(&softstate->asoft_instmutex); 174 if ((new_handle1 == NULL) && (new_handle2 == NULL)) { 175 mementry = 176 agp_find_bound_keyent(softstate, AGP_BYTES2PAGES(off)); 177 ASSERT(mementry); 178 mementry->kte_refcnt--; 179 } else if ((new_handle1 != NULL) && (new_handle2 != NULL)) { 180 mementry = 181 agp_find_bound_keyent(softstate, AGP_BYTES2PAGES(off)); 182 ASSERT(mementry); 183 mementry->kte_refcnt++; 184 } 185 ASSERT(mementry->kte_refcnt >= 0); 186 mutex_exit(&softstate->asoft_instmutex); 187 kmem_free(ctxp, sizeof (struct agpgart_ctx)); 188 } 189 190 /*ARGSUSED*/ 191 static int 192 agp_devmap_map(devmap_cookie_t handle, dev_t dev, 193 uint_t flags, offset_t offset, size_t len, void **new_devprivate) 194 { 195 agpgart_softstate_t *softstate; 196 int instance; 197 struct keytable_ent *mementry; 198 agpgart_ctx_t *newctxp; 199 200 ASSERT(handle); 201 instance = AGP_DEV2INST(dev); 202 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 203 if (softstate == NULL) { 204 AGPDB_PRINT2((CE_WARN, "agp_devmap_map: get soft state err")); 205 return (ENXIO); 206 } 207 208 ASSERT(softstate); 209 ASSERT(mutex_owned(&softstate->asoft_instmutex)); 210 ASSERT(len); 211 ASSERT(AGP_ALIGNED(offset) && AGP_ALIGNED(len)); 212 213 mementry = 214 agp_find_bound_keyent(softstate, AGP_BYTES2PAGES(offset)); 215 ASSERT(mementry); 216 mementry->kte_refcnt++; 217 ASSERT(mementry->kte_refcnt >= 0); 218 newctxp = kmem_zalloc(sizeof (agpgart_ctx_t), KM_SLEEP); 219 newctxp->actx_off = offset; 220 newctxp->actx_sc = softstate; 221 *new_devprivate = newctxp; 222 223 return (0); 224 } 225 226 /*ARGSUSED*/ 227 static int agp_devmap_dup(devmap_cookie_t handle, void *devprivate, 228 devmap_cookie_t new_handle, void **new_devprivate) 229 { 230 struct keytable_ent *mementry; 231 agpgart_ctx_t *newctxp, *ctxp; 232 agpgart_softstate_t *softstate; 233 234 ASSERT(devprivate); 235 ASSERT(handle && new_handle); 236 237 ctxp = (agpgart_ctx_t *)devprivate; 238 ASSERT(AGP_ALIGNED(ctxp->actx_off)); 239 240 newctxp = kmem_zalloc(sizeof (agpgart_ctx_t), KM_SLEEP); 241 newctxp->actx_off = ctxp->actx_off; 242 newctxp->actx_sc = ctxp->actx_sc; 243 softstate = (agpgart_softstate_t *)newctxp->actx_sc; 244 245 mutex_enter(&softstate->asoft_instmutex); 246 mementry = agp_find_bound_keyent(softstate, 247 AGP_BYTES2PAGES(newctxp->actx_off)); 248 mementry->kte_refcnt++; 249 ASSERT(mementry->kte_refcnt >= 0); 250 mutex_exit(&softstate->asoft_instmutex); 251 *new_devprivate = newctxp; 252 253 return (0); 254 } 255 256 struct devmap_callback_ctl agp_devmap_cb = { 257 DEVMAP_OPS_REV, /* rev */ 258 agp_devmap_map, /* map */ 259 NULL, /* access */ 260 agp_devmap_dup, /* dup */ 261 agp_devmap_unmap, /* unmap */ 262 }; 263 264 /* 265 * agp_master_regis_byname() 266 * 267 * Description: 268 * Open the AGP master device node by device path name and 269 * register the device handle for later operations. 270 * We check all possible driver instance from 0 271 * to MAX_INSTNUM because the master device could be 272 * at any instance number. Only one AGP master is supported. 273 * 274 * Arguments: 275 * master_hdlp AGP master device LDI handle pointer 276 * agpgart_l AGPGART driver LDI identifier 277 * 278 * Returns: 279 * -1 failed 280 * 0 success 281 */ 282 static int 283 agp_master_regis_byname(ldi_handle_t *master_hdlp, ldi_ident_t agpgart_li) 284 { 285 int i; 286 char buf[MAXPATHLEN]; 287 288 ASSERT(master_hdlp); 289 ASSERT(agpgart_li); 290 291 /* 292 * Search all possible instance numbers for the agp master device. 293 * Only one master device is supported now, so the search ends 294 * when one master device is found. 295 */ 296 for (i = 0; i < MAX_INSTNUM; i++) { 297 (void) snprintf(buf, MAXPATHLEN, "%s%d", AGPMASTER_DEVLINK, i); 298 if ((ldi_open_by_name(buf, 0, kcred, 299 master_hdlp, agpgart_li))) 300 continue; 301 AGPDB_PRINT1((CE_NOTE, 302 "master device found: instance number=%d", i)); 303 break; 304 305 } 306 307 /* AGP master device not found */ 308 if (i == MAX_INSTNUM) 309 return (-1); 310 311 return (0); 312 } 313 314 /* 315 * agp_target_regis_byname() 316 * 317 * Description: 318 * This function opens agp bridge device node by 319 * device path name and registers the device handle 320 * for later operations. 321 * We check driver instance from 0 to MAX_INSTNUM 322 * because the master device could be at any instance 323 * number. Only one agp target is supported. 324 * 325 * 326 * Arguments: 327 * target_hdlp AGP target device LDI handle pointer 328 * agpgart_l AGPGART driver LDI identifier 329 * 330 * Returns: 331 * -1 failed 332 * 0 success 333 */ 334 static int 335 agp_target_regis_byname(ldi_handle_t *target_hdlp, ldi_ident_t agpgart_li) 336 { 337 int i; 338 char buf[MAXPATHLEN]; 339 340 ASSERT(target_hdlp); 341 ASSERT(agpgart_li); 342 343 for (i = 0; i < MAX_INSTNUM; i++) { 344 (void) snprintf(buf, MAXPATHLEN, "%s%d", AGPTARGET_DEVLINK, i); 345 if ((ldi_open_by_name(buf, 0, kcred, 346 target_hdlp, agpgart_li))) 347 continue; 348 349 AGPDB_PRINT1((CE_NOTE, 350 "bridge device found: instance number=%d", i)); 351 break; 352 353 } 354 355 /* AGP bridge device not found */ 356 if (i == MAX_INSTNUM) { 357 AGPDB_PRINT2((CE_WARN, "bridge device not found")); 358 return (-1); 359 } 360 361 return (0); 362 } 363 364 /* 365 * amd64_gart_regis_byname() 366 * 367 * Description: 368 * Open all amd64 gart device nodes by deice path name and 369 * register the device handles for later operations. Each cpu 370 * has its own amd64 gart device. 371 * 372 * Arguments: 373 * cpu_garts cpu garts device list header 374 * agpgart_l AGPGART driver LDI identifier 375 * 376 * Returns: 377 * -1 failed 378 * 0 success 379 */ 380 static int 381 amd64_gart_regis_byname(amd64_garts_dev_t *cpu_garts, ldi_ident_t agpgart_li) 382 { 383 amd64_gart_dev_list_t *gart_list; 384 int i; 385 char buf[MAXPATHLEN]; 386 ldi_handle_t gart_hdl; 387 int ret; 388 389 ASSERT(cpu_garts); 390 ASSERT(agpgart_li); 391 392 /* 393 * Search all possible instance numbers for the gart devices. 394 * There can be multiple on-cpu gart devices for Opteron server. 395 */ 396 for (i = 0; i < MAX_INSTNUM; i++) { 397 (void) snprintf(buf, MAXPATHLEN, "%s%d", CPUGART_DEVLINK, i); 398 ret = ldi_open_by_name(buf, 0, kcred, 399 &gart_hdl, agpgart_li); 400 401 if (ret == ENODEV) 402 continue; 403 else if (ret != 0) { /* There was an error opening the device */ 404 amd64_gart_unregister(cpu_garts); 405 return (ret); 406 } 407 408 AGPDB_PRINT1((CE_NOTE, 409 "amd64 gart device found: instance number=%d", i)); 410 411 gart_list = (amd64_gart_dev_list_t *) 412 kmem_zalloc(sizeof (amd64_gart_dev_list_t), KM_SLEEP); 413 414 /* Add new item to the head of the gart device list */ 415 gart_list->gart_devhdl = gart_hdl; 416 gart_list->next = cpu_garts->gart_dev_list_head; 417 cpu_garts->gart_dev_list_head = gart_list; 418 cpu_garts->gart_device_num++; 419 } 420 421 if (cpu_garts->gart_device_num == 0) 422 return (ENODEV); 423 return (0); 424 } 425 426 /* 427 * Unregister agp master device handle 428 */ 429 static void 430 agp_master_unregister(ldi_handle_t *master_hdlp) 431 { 432 ASSERT(master_hdlp); 433 434 if (master_hdlp) { 435 (void) ldi_close(*master_hdlp, 0, kcred); 436 *master_hdlp = NULL; 437 } 438 } 439 440 /* 441 * Unregister agp bridge device handle 442 */ 443 static void 444 agp_target_unregister(ldi_handle_t *target_hdlp) 445 { 446 if (target_hdlp) { 447 (void) ldi_close(*target_hdlp, 0, kcred); 448 *target_hdlp = NULL; 449 } 450 } 451 452 /* 453 * Unregister all amd64 gart device handles 454 */ 455 static void 456 amd64_gart_unregister(amd64_garts_dev_t *cpu_garts) 457 { 458 amd64_gart_dev_list_t *gart_list; 459 amd64_gart_dev_list_t *next; 460 461 ASSERT(cpu_garts); 462 463 for (gart_list = cpu_garts->gart_dev_list_head; 464 gart_list; gart_list = next) { 465 466 ASSERT(gart_list->gart_devhdl); 467 (void) ldi_close(gart_list->gart_devhdl, 0, kcred); 468 next = gart_list->next; 469 /* Free allocated memory */ 470 kmem_free(gart_list, sizeof (amd64_gart_dev_list_t)); 471 } 472 cpu_garts->gart_dev_list_head = NULL; 473 cpu_garts->gart_device_num = 0; 474 } 475 476 /* 477 * lyr_detect_master_type() 478 * 479 * Description: 480 * This function gets agp master type by querying agp master device. 481 * 482 * Arguments: 483 * master_hdlp agp master device ldi handle pointer 484 * 485 * Returns: 486 * -1 unsupported device 487 * DEVICE_IS_I810 i810 series 488 * DEVICE_IS_I810 i830 series 489 * DEVICE_IS_AGP true agp master 490 */ 491 static int 492 lyr_detect_master_type(ldi_handle_t *master_hdlp) 493 { 494 int vtype; 495 int err; 496 497 ASSERT(master_hdlp); 498 499 /* ldi_ioctl(agpmaster) */ 500 err = ldi_ioctl(*master_hdlp, DEVICE_DETECT, 501 (intptr_t)&vtype, FKIOCTL, kcred, 0); 502 if (err) /* Unsupported graphics device */ 503 return (-1); 504 return (vtype); 505 } 506 507 /* 508 * devtect_target_type() 509 * 510 * Description: 511 * This function gets the host bridge chipset type by querying the agp 512 * target device. 513 * 514 * Arguments: 515 * target_hdlp agp target device LDI handle pointer 516 * 517 * Returns: 518 * CHIP_IS_INTEL Intel agp chipsets 519 * CHIP_IS_AMD AMD agp chipset 520 * -1 unsupported chipset 521 */ 522 static int 523 lyr_detect_target_type(ldi_handle_t *target_hdlp) 524 { 525 int btype; 526 int err; 527 528 ASSERT(target_hdlp); 529 530 err = ldi_ioctl(*target_hdlp, CHIP_DETECT, (intptr_t)&btype, 531 FKIOCTL, kcred, 0); 532 if (err) /* Unsupported bridge device */ 533 return (-1); 534 return (btype); 535 } 536 537 /* 538 * lyr_init() 539 * 540 * Description: 541 * This function detects the graphics system architecture and 542 * registers all relative device handles in a global structure 543 * "agp_regdev". Then it stores the system arc type in driver 544 * soft state. 545 * 546 * Arguments: 547 * agp_regdev AGP devices registration struct pointer 548 * agpgart_l AGPGART driver LDI identifier 549 * 550 * Returns: 551 * 0 System arc supported and agp devices registration successed. 552 * -1 System arc not supported or device registration failed. 553 */ 554 int 555 lyr_init(agp_registered_dev_t *agp_regdev, ldi_ident_t agpgart_li) 556 { 557 ldi_handle_t *master_hdlp; 558 ldi_handle_t *target_hdlp; 559 amd64_garts_dev_t *garts_dev; 560 int card_type, chip_type; 561 int ret; 562 563 ASSERT(agp_regdev); 564 565 bzero(agp_regdev, sizeof (agp_registered_dev_t)); 566 agp_regdev->agprd_arctype = ARC_UNKNOWN; 567 /* 568 * Register agp devices, assuming all instances attached, and 569 * detect which agp architucture this server belongs to. This 570 * must be done before the agpgart driver starts to use layered 571 * driver interfaces. 572 */ 573 master_hdlp = &agp_regdev->agprd_masterhdl; 574 target_hdlp = &agp_regdev->agprd_targethdl; 575 garts_dev = &agp_regdev->agprd_cpugarts; 576 577 /* Check whether the system is amd64 arc */ 578 if ((ret = amd64_gart_regis_byname(garts_dev, agpgart_li)) == ENODEV) { 579 /* No amd64 gart devices */ 580 AGPDB_PRINT1((CE_NOTE, 581 "lyr_init: this is not an amd64 system")); 582 if (agp_master_regis_byname(master_hdlp, agpgart_li)) { 583 AGPDB_PRINT2((CE_WARN, 584 "lyr_init: register master device unsuccessful")); 585 goto err1; 586 } 587 if (agp_target_regis_byname(target_hdlp, agpgart_li)) { 588 AGPDB_PRINT2((CE_WARN, 589 "lyr_init: register target device unsuccessful")); 590 goto err2; 591 } 592 card_type = lyr_detect_master_type(master_hdlp); 593 /* 594 * Detect system arc by master device. If it is a intel 595 * integrated device, finish the detection successfully. 596 */ 597 switch (card_type) { 598 case DEVICE_IS_I810: /* I810 likewise graphics */ 599 AGPDB_PRINT1((CE_NOTE, 600 "lyr_init: the system is Intel 810 arch")); 601 agp_regdev->agprd_arctype = ARC_IGD810; 602 return (0); 603 case DEVICE_IS_I830: /* I830 likewise graphics */ 604 AGPDB_PRINT1((CE_NOTE, 605 "lyr_init: the system is Intel 830 arch")); 606 agp_regdev->agprd_arctype = ARC_IGD830; 607 return (0); 608 case DEVICE_IS_AGP: /* AGP graphics */ 609 break; 610 default: /* Non IGD/AGP graphics */ 611 AGPDB_PRINT2((CE_WARN, 612 "lyr_init: non-supported master device")); 613 goto err3; 614 } 615 616 chip_type = lyr_detect_target_type(target_hdlp); 617 618 /* Continue to detect AGP arc by target device */ 619 switch (chip_type) { 620 case CHIP_IS_INTEL: /* Intel chipset */ 621 AGPDB_PRINT1((CE_NOTE, 622 "lyr_init: Intel AGP arch detected")); 623 agp_regdev->agprd_arctype = ARC_INTELAGP; 624 return (0); 625 case CHIP_IS_AMD: /* AMD chipset */ 626 AGPDB_PRINT2((CE_WARN, 627 "lyr_init: no cpu gart, but have AMD64 chipsets")); 628 goto err3; 629 default: /* Non supported chipset */ 630 AGPDB_PRINT2((CE_WARN, 631 "lyr_init: detection can not continue")); 632 goto err3; 633 } 634 635 } 636 637 if (ret) 638 return (-1); /* Errors in open amd64 cpu gart devices */ 639 640 /* 641 * AMD64 cpu gart device exsits, continue detection 642 */ 643 if (agp_master_regis_byname(master_hdlp, agpgart_li)) { 644 AGPDB_PRINT1((CE_NOTE, "lyr_init: no AGP master in amd64")); 645 goto err1; 646 } 647 648 if (agp_target_regis_byname(target_hdlp, agpgart_li)) { 649 AGPDB_PRINT1((CE_NOTE, 650 "lyr_init: no AGP bridge")); 651 goto err2; 652 } 653 654 AGPDB_PRINT1((CE_NOTE, 655 "lyr_init: the system is AMD64 AGP architecture")); 656 657 agp_regdev->agprd_arctype = ARC_AMD64AGP; 658 659 return (0); /* Finished successfully */ 660 661 err3: 662 agp_target_unregister(&agp_regdev->agprd_targethdl); 663 err2: 664 agp_master_unregister(&agp_regdev->agprd_masterhdl); 665 err1: 666 /* AMD64 CPU gart registered ? */ 667 if (ret == 0) { 668 amd64_gart_unregister(garts_dev); 669 } 670 agp_regdev->agprd_arctype = ARC_UNKNOWN; 671 return (-1); 672 } 673 674 void 675 lyr_end(agp_registered_dev_t *agp_regdev) 676 { 677 ASSERT(agp_regdev); 678 679 switch (agp_regdev->agprd_arctype) { 680 case ARC_IGD810: 681 case ARC_IGD830: 682 case ARC_INTELAGP: 683 agp_master_unregister(&agp_regdev->agprd_masterhdl); 684 agp_target_unregister(&agp_regdev->agprd_targethdl); 685 686 return; 687 case ARC_AMD64AGP: 688 agp_master_unregister(&agp_regdev->agprd_masterhdl); 689 agp_target_unregister(&agp_regdev->agprd_targethdl); 690 amd64_gart_unregister(&agp_regdev->agprd_cpugarts); 691 692 return; 693 default: 694 ASSERT(0); 695 return; 696 } 697 } 698 699 int 700 lyr_get_info(agp_kern_info_t *info, agp_registered_dev_t *agp_regdev) 701 { 702 ldi_handle_t hdl; 703 igd_info_t value1; 704 i_agp_info_t value2; 705 size_t prealloc_size; 706 int err; 707 708 ASSERT(info); 709 ASSERT(agp_regdev); 710 711 switch (agp_regdev->agprd_arctype) { 712 case ARC_IGD810: 713 hdl = agp_regdev->agprd_masterhdl; 714 err = ldi_ioctl(hdl, I8XX_GET_INFO, (intptr_t)&value1, 715 FKIOCTL, kcred, 0); 716 if (err) 717 return (-1); 718 info->agpki_mdevid = value1.igd_devid; 719 info->agpki_aperbase = value1.igd_aperbase; 720 info->agpki_apersize = value1.igd_apersize; 721 722 hdl = agp_regdev->agprd_targethdl; 723 err = ldi_ioctl(hdl, I8XX_GET_PREALLOC_SIZE, 724 (intptr_t)&prealloc_size, FKIOCTL, kcred, 0); 725 if (err) 726 return (-1); 727 info->agpki_presize = prealloc_size; 728 729 break; 730 731 case ARC_IGD830: 732 hdl = agp_regdev->agprd_masterhdl; 733 err = ldi_ioctl(hdl, I8XX_GET_INFO, (intptr_t)&value1, 734 FKIOCTL, kcred, 0); 735 if (err) 736 return (-1); 737 info->agpki_mdevid = value1.igd_devid; 738 info->agpki_aperbase = value1.igd_aperbase; 739 info->agpki_apersize = value1.igd_apersize; 740 741 hdl = agp_regdev->agprd_targethdl; 742 err = ldi_ioctl(hdl, I8XX_GET_PREALLOC_SIZE, 743 (intptr_t)&prealloc_size, FKIOCTL, kcred, 0); 744 if (err) 745 return (-1); 746 747 /* 748 * Assume all units are kilobytes unless explicitly 749 * stated below: 750 * preallocated GTT memory = preallocated memory - GTT size 751 * - scratch page size 752 * 753 * scratch page size = 4 754 * GTT size (KB) = aperture size (MB) 755 * this algorithm came from Xorg source code 756 */ 757 if (prealloc_size > (info->agpki_apersize + 4)) 758 prealloc_size = 759 prealloc_size - info->agpki_apersize - 4; 760 else { 761 AGPDB_PRINT2((CE_WARN, "lyr_get_info: " 762 "pre-allocated memory too small, setting to zero")); 763 prealloc_size = 0; 764 } 765 info->agpki_presize = prealloc_size; 766 AGPDB_PRINT2((CE_NOTE, 767 "lyr_get_info: prealloc_size = %ldKB, apersize = %dMB", 768 prealloc_size, info->agpki_apersize)); 769 break; 770 case ARC_INTELAGP: 771 case ARC_AMD64AGP: 772 /* AGP devices */ 773 hdl = agp_regdev->agprd_masterhdl; 774 err = ldi_ioctl(hdl, AGP_MASTER_GETINFO, 775 (intptr_t)&value2, FKIOCTL, kcred, 0); 776 if (err) 777 return (-1); 778 info->agpki_mdevid = value2.iagp_devid; 779 info->agpki_mver = value2.iagp_ver; 780 info->agpki_mstatus = value2.iagp_mode; 781 hdl = agp_regdev->agprd_targethdl; 782 err = ldi_ioctl(hdl, AGP_TARGET_GETINFO, 783 (intptr_t)&value2, FKIOCTL, kcred, 0); 784 if (err) 785 return (-1); 786 info->agpki_tdevid = value2.iagp_devid; 787 info->agpki_tver = value2.iagp_ver; 788 info->agpki_tstatus = value2.iagp_mode; 789 info->agpki_aperbase = value2.iagp_aperbase; 790 info->agpki_apersize = value2.iagp_apersize; 791 break; 792 default: 793 AGPDB_PRINT2((CE_WARN, 794 "lyr_get_info: function doesn't work for unknown arc")); 795 return (-1); 796 } 797 if ((info->agpki_apersize >= MAXAPERMEGAS) || 798 (info->agpki_apersize == 0) || 799 (info->agpki_aperbase == 0)) { 800 AGPDB_PRINT2((CE_WARN, 801 "lyr_get_info: aperture is not programmed correctly!")); 802 return (-1); 803 } 804 805 return (0); 806 } 807 808 /* 809 * lyr_i8xx_add_to_gtt() 810 * 811 * Description: 812 * This function sets up the integrated video device gtt table 813 * via an ioclt to the AGP master driver. 814 * 815 * Arguments: 816 * pg_offset The start entry to be setup 817 * keyent Keytable entity pointer 818 * agp_regdev AGP devices registration struct pointer 819 * 820 * Returns: 821 * 0 success 822 * -1 invalid operations 823 */ 824 int 825 lyr_i8xx_add_to_gtt(uint32_t pg_offset, keytable_ent_t *keyent, 826 agp_registered_dev_t *agp_regdev) 827 { 828 int err = 0; 829 int rval; 830 ldi_handle_t hdl; 831 igd_gtt_seg_t gttseg; 832 uint32_t *addrp, i; 833 uint32_t npages; 834 835 ASSERT(keyent); 836 ASSERT(agp_regdev); 837 gttseg.igs_pgstart = pg_offset; 838 npages = keyent->kte_pages; 839 gttseg.igs_npage = npages; 840 gttseg.igs_type = keyent->kte_type; 841 gttseg.igs_phyaddr = (uint32_t *)kmem_zalloc 842 (sizeof (uint32_t) * gttseg.igs_npage, KM_SLEEP); 843 844 addrp = gttseg.igs_phyaddr; 845 for (i = 0; i < npages; i++, addrp++) { 846 *addrp = 847 (uint32_t)((keyent->kte_pfnarray[i]) << GTT_PAGE_SHIFT); 848 } 849 850 hdl = agp_regdev->agprd_masterhdl; 851 if (ldi_ioctl(hdl, I8XX_ADD2GTT, (intptr_t)>tseg, FKIOCTL, 852 kcred, &rval)) { 853 AGPDB_PRINT2((CE_WARN, "lyr_i8xx_add_to_gtt: ldi_ioctl error")); 854 AGPDB_PRINT2((CE_WARN, "lyr_i8xx_add_to_gtt: pg_start=0x%x", 855 gttseg.igs_pgstart)); 856 AGPDB_PRINT2((CE_WARN, "lyr_i8xx_add_to_gtt: pages=0x%x", 857 gttseg.igs_npage)); 858 AGPDB_PRINT2((CE_WARN, "lyr_i8xx_add_to_gtt: type=0x%x", 859 gttseg.igs_type)); 860 err = -1; 861 } 862 kmem_free(gttseg.igs_phyaddr, sizeof (uint32_t) * gttseg.igs_npage); 863 return (err); 864 } 865 866 /* 867 * lyr_i8xx_remove_from_gtt() 868 * 869 * Description: 870 * This function clears the integrated video device gtt table via 871 * an ioctl to the agp master device. 872 * 873 * Arguments: 874 * pg_offset The starting entry to be cleared 875 * npage The number of entries to be cleared 876 * agp_regdev AGP devices struct pointer 877 * 878 * Returns: 879 * 0 success 880 * -1 invalid operations 881 */ 882 int 883 lyr_i8xx_remove_from_gtt(uint32_t pg_offset, uint32_t npage, 884 agp_registered_dev_t *agp_regdev) 885 { 886 int rval; 887 ldi_handle_t hdl; 888 igd_gtt_seg_t gttseg; 889 890 gttseg.igs_pgstart = pg_offset; 891 gttseg.igs_npage = npage; 892 893 hdl = agp_regdev->agprd_masterhdl; 894 if (ldi_ioctl(hdl, I8XX_REM_GTT, (intptr_t)>tseg, FKIOCTL, 895 kcred, &rval)) 896 return (-1); 897 898 return (0); 899 } 900 901 /* 902 * lyr_set_gart_addr() 903 * 904 * Description: 905 * This function puts the gart table physical address in the 906 * gart base register. 907 * Please refer to gart and gtt table base register format for 908 * gart base register format in agpdefs.h. 909 * 910 * Arguments: 911 * phy_base The base physical address of gart table 912 * agp_regdev AGP devices registration struct pointer 913 * 914 * Returns: 915 * 0 success 916 * -1 failed 917 * 918 */ 919 920 int 921 lyr_set_gart_addr(uint64_t phy_base, agp_registered_dev_t *agp_regdev) 922 { 923 amd64_gart_dev_list_t *gart_list; 924 ldi_handle_t hdl; 925 int err = 0; 926 927 ASSERT(agp_regdev); 928 switch (agp_regdev->agprd_arctype) { 929 case ARC_IGD810: 930 { 931 uint32_t base; 932 933 ASSERT((phy_base & ~I810_POINTER_MASK) == 0); 934 base = (uint32_t)phy_base; 935 936 hdl = agp_regdev->agprd_masterhdl; 937 err = ldi_ioctl(hdl, I810_SET_GTT_BASE, 938 (intptr_t)&base, FKIOCTL, kcred, 0); 939 break; 940 } 941 case ARC_INTELAGP: 942 { 943 uint32_t addr; 944 addr = (uint32_t)phy_base; 945 946 ASSERT((phy_base & ~GTT_POINTER_MASK) == 0); 947 hdl = agp_regdev->agprd_targethdl; 948 err = ldi_ioctl(hdl, AGP_TARGET_SET_GATTADDR, 949 (intptr_t)&addr, FKIOCTL, kcred, 0); 950 break; 951 } 952 case ARC_AMD64AGP: 953 { 954 uint32_t addr; 955 956 ASSERT((phy_base & ~AMD64_POINTER_MASK) == 0); 957 addr = (uint32_t)((phy_base >> AMD64_GARTBASE_SHIFT) 958 & AMD64_GARTBASE_MASK); 959 960 for (gart_list = agp_regdev->agprd_cpugarts.gart_dev_list_head; 961 gart_list; 962 gart_list = gart_list->next) { 963 hdl = gart_list->gart_devhdl; 964 if (ldi_ioctl(hdl, AMD64_SET_GART_ADDR, 965 (intptr_t)&addr, FKIOCTL, kcred, 0)) { 966 err = -1; 967 break; 968 } 969 } 970 break; 971 } 972 default: 973 err = -1; 974 } 975 976 if (err) 977 return (-1); 978 979 return (0); 980 } 981 982 int 983 lyr_set_agp_cmd(uint32_t cmd, agp_registered_dev_t *agp_regdev) 984 { 985 ldi_handle_t hdl; 986 uint32_t command; 987 988 ASSERT(agp_regdev); 989 command = cmd; 990 hdl = agp_regdev->agprd_targethdl; 991 if (ldi_ioctl(hdl, AGP_TARGET_SETCMD, 992 (intptr_t)&command, FKIOCTL, kcred, 0)) 993 return (-1); 994 hdl = agp_regdev->agprd_masterhdl; 995 if (ldi_ioctl(hdl, AGP_MASTER_SETCMD, 996 (intptr_t)&command, FKIOCTL, kcred, 0)) 997 return (-1); 998 999 return (0); 1000 } 1001 1002 int 1003 lyr_config_devices(agp_registered_dev_t *agp_regdev) 1004 { 1005 amd64_gart_dev_list_t *gart_list; 1006 ldi_handle_t hdl; 1007 int rc = 0; 1008 1009 ASSERT(agp_regdev); 1010 switch (agp_regdev->agprd_arctype) { 1011 case ARC_IGD830: 1012 case ARC_IGD810: 1013 break; 1014 case ARC_INTELAGP: 1015 { 1016 hdl = agp_regdev->agprd_targethdl; 1017 rc = ldi_ioctl(hdl, AGP_TARGET_CONFIGURE, 1018 0, FKIOCTL, kcred, 0); 1019 break; 1020 } 1021 case ARC_AMD64AGP: 1022 { 1023 /* 1024 * BIOS always shadow registers such like Aperture Base 1025 * register, Aperture Size Register from the AGP bridge 1026 * to the AMD64 CPU host bridge. If future BIOSes are broken 1027 * in this regard, we may need to shadow these registers 1028 * in driver. 1029 */ 1030 1031 for (gart_list = agp_regdev->agprd_cpugarts.gart_dev_list_head; 1032 gart_list; 1033 gart_list = gart_list->next) { 1034 hdl = gart_list->gart_devhdl; 1035 if (ldi_ioctl(hdl, AMD64_CONFIGURE, 1036 0, FKIOCTL, kcred, 0)) { 1037 rc = -1; 1038 break; 1039 } 1040 } 1041 break; 1042 } 1043 default: 1044 rc = -1; 1045 } 1046 1047 if (rc) 1048 return (-1); 1049 1050 return (0); 1051 } 1052 1053 int 1054 lyr_unconfig_devices(agp_registered_dev_t *agp_regdev) 1055 { 1056 amd64_gart_dev_list_t *gart_list; 1057 ldi_handle_t hdl; 1058 int rc = 0; 1059 1060 ASSERT(agp_regdev); 1061 switch (agp_regdev->agprd_arctype) { 1062 case ARC_IGD830: 1063 case ARC_IGD810: 1064 { 1065 hdl = agp_regdev->agprd_masterhdl; 1066 rc = ldi_ioctl(hdl, I8XX_UNCONFIG, 0, FKIOCTL, kcred, 0); 1067 break; 1068 } 1069 case ARC_INTELAGP: 1070 { 1071 hdl = agp_regdev->agprd_targethdl; 1072 rc = ldi_ioctl(hdl, AGP_TARGET_UNCONFIG, 1073 0, FKIOCTL, kcred, 0); 1074 break; 1075 } 1076 case ARC_AMD64AGP: 1077 { 1078 for (gart_list = agp_regdev->agprd_cpugarts.gart_dev_list_head; 1079 gart_list; gart_list = gart_list->next) { 1080 hdl = gart_list->gart_devhdl; 1081 if (ldi_ioctl(hdl, AMD64_UNCONFIG, 1082 0, FKIOCTL, kcred, 0)) { 1083 rc = -1; 1084 break; 1085 } 1086 } 1087 break; 1088 } 1089 default: 1090 rc = -1; 1091 } 1092 1093 if (rc) 1094 return (-1); 1095 1096 return (0); 1097 } 1098 1099 /* 1100 * lyr_flush_gart_cache() 1101 * 1102 * Description: 1103 * This function flushes the GART translation look-aside buffer. All 1104 * GART translation caches will be flushed after this operation. 1105 * 1106 * Arguments: 1107 * agp_regdev AGP devices struct pointer 1108 */ 1109 void 1110 lyr_flush_gart_cache(agp_registered_dev_t *agp_regdev) 1111 { 1112 amd64_gart_dev_list_t *gart_list; 1113 ldi_handle_t hdl; 1114 1115 ASSERT(agp_regdev); 1116 if (agp_regdev->agprd_arctype == ARC_AMD64AGP) { 1117 for (gart_list = agp_regdev->agprd_cpugarts.gart_dev_list_head; 1118 gart_list; gart_list = gart_list->next) { 1119 hdl = gart_list->gart_devhdl; 1120 (void) ldi_ioctl(hdl, AMD64_FLUSH_GTLB, 1121 0, FKIOCTL, kcred, 0); 1122 } 1123 } else if (agp_regdev->agprd_arctype == ARC_INTELAGP) { 1124 hdl = agp_regdev->agprd_targethdl; 1125 (void) ldi_ioctl(hdl, AGP_TARGET_FLUSH_GTLB, 0, 1126 FKIOCTL, kcred, 0); 1127 } 1128 } 1129 1130 /* 1131 * get_max_pages() 1132 * 1133 * Description: 1134 * This function compute the total pages allowed for agp aperture 1135 * based on the ammount of physical pages. 1136 * The algorithm is: compare the aperture size with 1/4 of total 1137 * physical pages, and use the smaller one to for the max available 1138 * pages. 1139 * 1140 * Arguments: 1141 * aper_size system agp aperture size (in MB) 1142 * 1143 * Returns: 1144 * The max possible number of agp memory pages available to users 1145 */ 1146 static uint32_t 1147 get_max_pages(uint32_t aper_size) 1148 { 1149 uint32_t i, j; 1150 1151 ASSERT(aper_size <= MAXAPERMEGAS); 1152 1153 i = AGP_MB2PAGES(aper_size); 1154 j = (physmem >> 2); 1155 1156 return ((i < j) ? i : j); 1157 } 1158 1159 /* 1160 * agp_fill_empty_keyent() 1161 * 1162 * Description: 1163 * This function finds a empty key table slot and 1164 * fills it with a new entity. 1165 * 1166 * Arguments: 1167 * softsate driver soft state pointer 1168 * entryp new entity data pointer 1169 * 1170 * Returns: 1171 * NULL no key table slot available 1172 * entryp the new entity slot pointer 1173 */ 1174 static keytable_ent_t * 1175 agp_fill_empty_keyent(agpgart_softstate_t *softstate, keytable_ent_t *entryp) 1176 { 1177 int key; 1178 keytable_ent_t *newentryp; 1179 1180 ASSERT(softstate); 1181 ASSERT(entryp); 1182 ASSERT(entryp->kte_memhdl); 1183 ASSERT(entryp->kte_pfnarray); 1184 ASSERT(mutex_owned(&softstate->asoft_instmutex)); 1185 1186 for (key = 0; key < AGP_MAXKEYS; key++) { 1187 newentryp = &softstate->asoft_table[key]; 1188 if (newentryp->kte_memhdl == NULL) { 1189 break; 1190 } 1191 } 1192 1193 if (key >= AGP_MAXKEYS) { 1194 AGPDB_PRINT2((CE_WARN, 1195 "agp_fill_empty_keyent: key table exhausted")); 1196 return (NULL); 1197 } 1198 1199 ASSERT(newentryp->kte_pfnarray == NULL); 1200 bcopy(entryp, newentryp, sizeof (keytable_ent_t)); 1201 newentryp->kte_key = key; 1202 1203 return (newentryp); 1204 } 1205 1206 /* 1207 * agp_find_bound_keyent() 1208 * 1209 * Description: 1210 * This function finds the key table entity by agp aperture page offset. 1211 * Every keytable entity will have an agp aperture range after the binding 1212 * operation. 1213 * 1214 * Arguments: 1215 * softsate driver soft state pointer 1216 * pg_offset agp aperture page offset 1217 * 1218 * Returns: 1219 * NULL no such keytable entity 1220 * pointer key table entity pointer found 1221 */ 1222 static keytable_ent_t * 1223 agp_find_bound_keyent(agpgart_softstate_t *softstate, uint32_t pg_offset) 1224 { 1225 int keycount; 1226 keytable_ent_t *entryp; 1227 1228 ASSERT(softstate); 1229 ASSERT(mutex_owned(&softstate->asoft_instmutex)); 1230 1231 for (keycount = 0; keycount < AGP_MAXKEYS; keycount++) { 1232 entryp = &softstate->asoft_table[keycount]; 1233 if (entryp->kte_bound == 0) { 1234 continue; 1235 } 1236 1237 if (pg_offset < entryp->kte_pgoff) 1238 continue; 1239 if (pg_offset >= (entryp->kte_pgoff + entryp->kte_pages)) 1240 continue; 1241 1242 ASSERT(entryp->kte_memhdl); 1243 ASSERT(entryp->kte_pfnarray); 1244 1245 return (entryp); 1246 } 1247 1248 return (NULL); 1249 } 1250 1251 /* 1252 * agp_check_off() 1253 * 1254 * Description: 1255 * This function checks whether an AGP aperture range to be bound 1256 * overlaps with AGP offset already bound. 1257 * 1258 * Arguments: 1259 * entryp key table start entry pointer 1260 * pg_start AGP range start page offset 1261 * pg_num pages number to be bound 1262 * 1263 * Returns: 1264 * 0 Does not overlap 1265 * -1 Overlaps 1266 */ 1267 1268 static int 1269 agp_check_off(keytable_ent_t *entryp, uint32_t pg_start, uint32_t pg_num) 1270 { 1271 int key; 1272 uint64_t pg_end; 1273 uint64_t kpg_end; 1274 1275 ASSERT(entryp); 1276 1277 pg_end = pg_start + pg_num; 1278 for (key = 0; key < AGP_MAXKEYS; key++) { 1279 if (!entryp[key].kte_bound) 1280 continue; 1281 1282 kpg_end = entryp[key].kte_pgoff + entryp[key].kte_pages; 1283 if (!((pg_end <= entryp[key].kte_pgoff) || 1284 (pg_start >= kpg_end))) 1285 break; 1286 } 1287 1288 if (key == AGP_MAXKEYS) 1289 return (0); 1290 else 1291 return (-1); 1292 } 1293 1294 static int 1295 is_controlling_proc(agpgart_softstate_t *st) 1296 { 1297 ASSERT(st); 1298 1299 if (!st->asoft_acquired) { 1300 AGPDB_PRINT2((CE_WARN, 1301 "ioctl_agpgart_setup: gart not acquired")); 1302 return (-1); 1303 } 1304 if (st->asoft_curpid != ddi_get_pid()) { 1305 AGPDB_PRINT2((CE_WARN, 1306 "ioctl_agpgart_release: not controlling process")); 1307 return (-1); 1308 } 1309 1310 return (0); 1311 } 1312 1313 static void release_control(agpgart_softstate_t *st) 1314 { 1315 st->asoft_curpid = 0; 1316 st->asoft_acquired = 0; 1317 } 1318 1319 static void acquire_control(agpgart_softstate_t *st) 1320 { 1321 st->asoft_curpid = ddi_get_pid(); 1322 st->asoft_acquired = 1; 1323 } 1324 1325 /* 1326 * agp_remove_from_gart() 1327 * 1328 * Description: 1329 * This function fills the gart table entries by a given page 1330 * frame number array and setup the agp aperture page to physical 1331 * memory page translation. 1332 * Arguments: 1333 * pg_offset Starting aperture page to be bound 1334 * entries the number of pages to be bound 1335 * acc_hdl GART table dma memory acc handle 1336 * tablep GART table kernel virtual address 1337 */ 1338 static void 1339 agp_remove_from_gart( 1340 uint32_t pg_offset, 1341 uint32_t entries, 1342 ddi_dma_handle_t dma_hdl, 1343 uint32_t *tablep) 1344 { 1345 uint32_t items = 0; 1346 uint32_t *entryp; 1347 1348 entryp = tablep + pg_offset; 1349 while (items < entries) { 1350 *(entryp + items) = 0; 1351 items++; 1352 } 1353 (void) ddi_dma_sync(dma_hdl, pg_offset * sizeof (uint32_t), 1354 entries * sizeof (uint32_t), DDI_DMA_SYNC_FORDEV); 1355 } 1356 1357 /* 1358 * agp_unbind_key() 1359 * 1360 * Description: 1361 * This function unbinds AGP memory from the gart table. It will clear 1362 * all the gart entries related to this agp memory. 1363 * 1364 * Arguments: 1365 * softstate driver soft state pointer 1366 * entryp key table entity pointer 1367 * 1368 * Returns: 1369 * EINVAL invalid key table entity pointer 1370 * 0 success 1371 * 1372 */ 1373 static int 1374 agp_unbind_key(agpgart_softstate_t *softstate, keytable_ent_t *entryp) 1375 { 1376 int retval = 0; 1377 1378 ASSERT(entryp); 1379 ASSERT((entryp->kte_key >= 0) && (entryp->kte_key < AGP_MAXKEYS)); 1380 1381 if (!entryp->kte_bound) { 1382 AGPDB_PRINT2((CE_WARN, 1383 "agp_unbind_key: key = 0x%x, not bound", 1384 entryp->kte_key)); 1385 return (EINVAL); 1386 } 1387 if (entryp->kte_refcnt) { 1388 AGPDB_PRINT2((CE_WARN, 1389 "agp_unbind_key: memory is exported to users")); 1390 return (EINVAL); 1391 } 1392 1393 ASSERT((entryp->kte_pgoff + entryp->kte_pages) <= 1394 AGP_MB2PAGES(softstate->asoft_info.agpki_apersize)); 1395 ASSERT((softstate->asoft_devreg.agprd_arctype != ARC_UNKNOWN)); 1396 1397 switch (softstate->asoft_devreg.agprd_arctype) { 1398 case ARC_IGD810: 1399 case ARC_IGD830: 1400 retval = lyr_i8xx_remove_from_gtt( 1401 entryp->kte_pgoff, entryp->kte_pages, 1402 &softstate->asoft_devreg); 1403 if (retval) { 1404 AGPDB_PRINT2((CE_WARN, 1405 "agp_unbind_key: Key = 0x%x, clear table error", 1406 entryp->kte_key)); 1407 return (EIO); 1408 } 1409 break; 1410 case ARC_INTELAGP: 1411 case ARC_AMD64AGP: 1412 agp_remove_from_gart(entryp->kte_pgoff, 1413 entryp->kte_pages, 1414 softstate->gart_dma_handle, 1415 (uint32_t *)softstate->gart_vbase); 1416 /* Flush GTLB table */ 1417 lyr_flush_gart_cache(&softstate->asoft_devreg); 1418 1419 break; 1420 } 1421 1422 entryp->kte_bound = 0; 1423 1424 return (0); 1425 } 1426 1427 /* 1428 * agp_dealloc_kmem() 1429 * 1430 * Description: 1431 * This function deallocates dma memory resources for userland 1432 * applications. 1433 * 1434 * Arguments: 1435 * entryp keytable entity pointer 1436 */ 1437 static void 1438 agp_dealloc_kmem(keytable_ent_t *entryp) 1439 { 1440 kmem_free(entryp->kte_pfnarray, sizeof (pfn_t) * entryp->kte_pages); 1441 entryp->kte_pfnarray = NULL; 1442 1443 (void) ddi_dma_unbind_handle(KMEMP(entryp->kte_memhdl)->kmem_handle); 1444 KMEMP(entryp->kte_memhdl)->kmem_cookies_num = 0; 1445 ddi_dma_mem_free(&KMEMP(entryp->kte_memhdl)->kmem_acchdl); 1446 KMEMP(entryp->kte_memhdl)->kmem_acchdl = NULL; 1447 KMEMP(entryp->kte_memhdl)->kmem_reallen = 0; 1448 KMEMP(entryp->kte_memhdl)->kmem_kvaddr = NULL; 1449 1450 ddi_dma_free_handle(&(KMEMP(entryp->kte_memhdl)->kmem_handle)); 1451 KMEMP(entryp->kte_memhdl)->kmem_handle = NULL; 1452 1453 kmem_free(entryp->kte_memhdl, sizeof (agp_kmem_handle_t)); 1454 entryp->kte_memhdl = NULL; 1455 } 1456 1457 /* 1458 * agp_dealloc_pmem() 1459 * 1460 * Description: 1461 * This function deallocates memory resource for direct mapping to 1462 * userland applications. 1463 * 1464 * Arguments: 1465 * entryp key table entity pointer 1466 * 1467 */ 1468 static void 1469 agp_dealloc_pmem(keytable_ent_t *entryp) 1470 { 1471 devmap_pmem_free(PMEMP(entryp->kte_memhdl)->pmem_cookie); 1472 PMEMP(entryp->kte_memhdl)->pmem_cookie = NULL; 1473 kmem_free(entryp->kte_memhdl, sizeof (agp_pmem_handle_t)); 1474 entryp->kte_memhdl = NULL; 1475 1476 /* free the page frame number array */ 1477 kmem_free(entryp->kte_pfnarray, sizeof (pfn_t) * entryp->kte_pages); 1478 entryp->kte_pfnarray = NULL; 1479 } 1480 1481 /* 1482 * agp_dealloc_mem() 1483 * 1484 * Description: 1485 * This function deallocates physical memory resources allocated for 1486 * userland applications. 1487 * 1488 * Arguments: 1489 * st driver soft state pointer 1490 * entryp key table entity pointer 1491 * 1492 * Returns: 1493 * -1 not a valid memory type or the memory is mapped by 1494 * user area applications 1495 * 0 success 1496 */ 1497 static int 1498 agp_dealloc_mem(agpgart_softstate_t *st, keytable_ent_t *entryp) 1499 { 1500 1501 ASSERT(entryp); 1502 ASSERT(st); 1503 ASSERT(entryp->kte_memhdl); 1504 ASSERT(mutex_owned(&st->asoft_instmutex)); 1505 1506 /* auto unbind here */ 1507 if (entryp->kte_bound && !entryp->kte_refcnt) { 1508 AGPDB_PRINT2((CE_WARN, 1509 "agp_dealloc_mem: key=0x%x, auto unbind", 1510 entryp->kte_key)); 1511 1512 /* 1513 * agp_dealloc_mem may be called indirectly by agp_detach. 1514 * In the agp_detach function, agpgart_close is already 1515 * called which will free the gart table. agp_unbind_key 1516 * will panic if no valid gart table exists. So test if 1517 * gart table exsits here. 1518 */ 1519 if (st->asoft_opened) 1520 (void) agp_unbind_key(st, entryp); 1521 } 1522 if (entryp->kte_refcnt) { 1523 AGPDB_PRINT2((CE_WARN, 1524 "agp_dealloc_pmem: memory is exported to users")); 1525 return (-1); 1526 } 1527 1528 switch (entryp->kte_type) { 1529 case AGP_NORMAL: 1530 agp_dealloc_pmem(entryp); 1531 break; 1532 case AGP_PHYSICAL: 1533 agp_dealloc_kmem(entryp); 1534 break; 1535 default: 1536 return (-1); 1537 } 1538 1539 return (0); 1540 } 1541 1542 /* 1543 * agp_del_allkeys() 1544 * 1545 * Description: 1546 * This function calls agp_dealloc_mem to release all the agp memory 1547 * resource allocated. 1548 * 1549 * Arguments: 1550 * softsate driver soft state pointer 1551 * Returns: 1552 * -1 can not free all agp memory 1553 * 0 success 1554 * 1555 */ 1556 static int 1557 agp_del_allkeys(agpgart_softstate_t *softstate) 1558 { 1559 int key; 1560 int ret = 0; 1561 1562 ASSERT(softstate); 1563 for (key = 0; key < AGP_MAXKEYS; key++) { 1564 if (softstate->asoft_table[key].kte_memhdl != NULL) { 1565 /* 1566 * Check if we can free agp memory now. 1567 * If agp memory is exported to user 1568 * applications, agp_dealloc_mem will fail. 1569 */ 1570 if (agp_dealloc_mem(softstate, 1571 &softstate->asoft_table[key])) 1572 ret = -1; 1573 } 1574 } 1575 1576 return (ret); 1577 } 1578 1579 /* 1580 * pfn2gartentry() 1581 * 1582 * Description: 1583 * This function converts a physical address to GART entry. 1584 * For AMD64, hardware only support addresses below 40bits, 1585 * about 1024G physical address, so the largest pfn 1586 * number is below 28 bits. Please refer to GART and GTT entry 1587 * format table in agpdefs.h for entry format. Intel IGD only 1588 * only supports GTT entry below 1G. Intel AGP only supports 1589 * GART entry below 4G. 1590 * 1591 * Arguments: 1592 * arc_type system agp arc type 1593 * pfn page frame number 1594 * itemv the entry item to be returned 1595 * Returns: 1596 * -1 not a invalid page frame 1597 * 0 conversion success 1598 */ 1599 static int 1600 pfn2gartentry(agp_arc_type_t arc_type, pfn_t pfn, uint32_t *itemv) 1601 { 1602 uint64_t paddr; 1603 1604 paddr = pfn<<AGP_PAGE_SHIFT; 1605 1606 switch (arc_type) { 1607 case ARC_INTELAGP: 1608 { 1609 /* Only support 32-bit hardware address */ 1610 if ((paddr & ~AGP_INTEL_POINTER_MASK) != 0) { 1611 AGPDB_PRINT2((CE_WARN, 1612 "INTEL AGP Hardware only support 32 bits")); 1613 return (-1); 1614 } 1615 *itemv = (pfn << AGP_PAGE_SHIFT) | AGP_ENTRY_VALID; 1616 1617 break; 1618 } 1619 case ARC_AMD64AGP: 1620 { 1621 uint32_t value1, value2; 1622 /* Physaddr should not exceed 40-bit */ 1623 if ((paddr & ~AMD64_POINTER_MASK) != 0) { 1624 AGPDB_PRINT2((CE_WARN, 1625 "AMD64 GART hardware only supoort 40 bits")); 1626 return (-1); 1627 } 1628 value1 = (uint32_t)pfn >> 20; 1629 value1 <<= 4; 1630 value2 = (uint32_t)pfn << 12; 1631 1632 *itemv = value1 | value2 | AMD64_ENTRY_VALID; 1633 break; 1634 } 1635 case ARC_IGD810: 1636 if ((paddr & ~I810_POINTER_MASK) != 0) { 1637 AGPDB_PRINT2((CE_WARN, 1638 "Intel i810 only support 30 bits")); 1639 return (-1); 1640 } 1641 break; 1642 1643 case ARC_IGD830: 1644 if ((paddr & ~GTT_POINTER_MASK) != 0) { 1645 AGPDB_PRINT2((CE_WARN, 1646 "Intel IGD only support 32 bits")); 1647 return (-1); 1648 } 1649 break; 1650 default: 1651 AGPDB_PRINT2((CE_WARN, 1652 "pfn2gartentry: arc type = %d, not support", arc_type)); 1653 return (-1); 1654 } 1655 return (0); 1656 } 1657 1658 /* 1659 * Check allocated physical pages validity, only called in DEBUG 1660 * mode. 1661 */ 1662 static int 1663 agp_check_pfns(agp_arc_type_t arc_type, pfn_t *pfnarray, int items) 1664 { 1665 int count; 1666 uint32_t ret; 1667 1668 for (count = 0; count < items; count++) { 1669 if (pfn2gartentry(arc_type, pfnarray[count], &ret)) 1670 break; 1671 } 1672 if (count < items) 1673 return (-1); 1674 else 1675 return (0); 1676 } 1677 1678 /* 1679 * kmem_getpfns() 1680 * 1681 * Description: 1682 * This function gets page frame numbers from dma handle. 1683 * 1684 * Arguments: 1685 * dma_handle dma hanle allocated by ddi_dma_alloc_handle 1686 * dma_cookip dma cookie pointer 1687 * cookies_num cookies number 1688 * pfnarray array to store page frames 1689 * 1690 * Returns: 1691 * 0 success 1692 */ 1693 static int 1694 kmem_getpfns( 1695 ddi_dma_handle_t dma_handle, 1696 ddi_dma_cookie_t *dma_cookiep, 1697 int cookies_num, 1698 pfn_t *pfnarray) 1699 { 1700 int num_cookies; 1701 int index = 0; 1702 1703 num_cookies = cookies_num; 1704 1705 while (num_cookies > 0) { 1706 uint64_t ck_startaddr, ck_length, ck_end; 1707 ck_startaddr = dma_cookiep->dmac_address; 1708 ck_length = dma_cookiep->dmac_size; 1709 1710 ck_end = ck_startaddr + ck_length; 1711 while (ck_startaddr < ck_end) { 1712 pfnarray[index] = (pfn_t)ck_startaddr >> AGP_PAGE_SHIFT; 1713 ck_startaddr += AGP_PAGE_SIZE; 1714 index++; 1715 } 1716 1717 num_cookies--; 1718 if (num_cookies > 0) { 1719 ddi_dma_nextcookie(dma_handle, dma_cookiep); 1720 } 1721 } 1722 1723 return (0); 1724 } 1725 1726 static int 1727 copyinfo(agpgart_softstate_t *softstate, agp_info_t *info) 1728 { 1729 switch (softstate->asoft_devreg.agprd_arctype) { 1730 case ARC_IGD810: 1731 case ARC_IGD830: 1732 info->agpi_version.agpv_major = 0; 1733 info->agpi_version.agpv_minor = 0; 1734 info->agpi_devid = softstate->asoft_info.agpki_mdevid; 1735 info->agpi_mode = 0; 1736 break; 1737 case ARC_INTELAGP: 1738 case ARC_AMD64AGP: 1739 info->agpi_version = softstate->asoft_info.agpki_tver; 1740 info->agpi_devid = softstate->asoft_info.agpki_tdevid; 1741 info->agpi_mode = softstate->asoft_info.agpki_tstatus; 1742 break; 1743 default: 1744 AGPDB_PRINT2((CE_WARN, "copyinfo: UNKNOW ARC")); 1745 return (-1); 1746 } 1747 /* 1748 * 64bit->32bit conversion possible 1749 */ 1750 info->agpi_aperbase = softstate->asoft_info.agpki_aperbase; 1751 info->agpi_apersize = softstate->asoft_info.agpki_apersize; 1752 info->agpi_pgtotal = softstate->asoft_pgtotal; 1753 info->agpi_pgsystem = info->agpi_pgtotal; 1754 info->agpi_pgused = softstate->asoft_pgused; 1755 1756 return (0); 1757 } 1758 1759 static uint32_t 1760 agp_v2_setup(uint32_t tstatus, uint32_t mstatus, uint32_t mode) 1761 { 1762 uint32_t cmd; 1763 int rq, sba, over4g, fw, rate; 1764 1765 /* 1766 * tstatus: target device status 1767 * mstatus: master device status 1768 * mode: the agp mode to be sent 1769 */ 1770 1771 /* 1772 * RQ - Request Queue size 1773 * set RQ to the min of mode and tstatus 1774 * if mode set a RQ larger than hardware can support, 1775 * use the max RQ which hardware can support. 1776 * tstatus & AGPSTAT_RQ_MASK is the max RQ hardware can support 1777 * Corelogic will enqueue agp transaction 1778 */ 1779 rq = mode & AGPSTAT_RQ_MASK; 1780 if ((tstatus & AGPSTAT_RQ_MASK) < rq) 1781 rq = tstatus & AGPSTAT_RQ_MASK; 1782 1783 /* 1784 * SBA - Sideband Addressing 1785 * 1786 * Sideband Addressing provides an additional bus to pass requests 1787 * (address and command) to the target from the master. 1788 * 1789 * set SBA if all three support it 1790 */ 1791 sba = (tstatus & AGPSTAT_SBA) & (mstatus & AGPSTAT_SBA) 1792 & (mode & AGPSTAT_SBA); 1793 1794 /* set OVER4G if all three support it */ 1795 over4g = (tstatus & AGPSTAT_OVER4G) & (mstatus & AGPSTAT_OVER4G) 1796 & (mode & AGPSTAT_OVER4G); 1797 1798 /* 1799 * FW - fast write 1800 * 1801 * acceleration of memory write transactions from the corelogic to the 1802 * A.G.P. master device acting like a PCI target. 1803 * 1804 * set FW if all three support it 1805 */ 1806 fw = (tstatus & AGPSTAT_FW) & (mstatus & AGPSTAT_FW) 1807 & (mode & AGPSTAT_FW); 1808 1809 /* 1810 * figure out the max rate 1811 * AGP v2 support: 4X, 2X, 1X speed 1812 * status bit meaning 1813 * --------------------------------------------- 1814 * 7:3 others 1815 * 3 0 stand for V2 support 1816 * 0:2 001:1X, 010:2X, 100:4X 1817 * ---------------------------------------------- 1818 */ 1819 rate = (tstatus & AGPSTAT_RATE_MASK) & (mstatus & AGPSTAT_RATE_MASK) 1820 & (mode & AGPSTAT_RATE_MASK); 1821 if (rate & AGP2_RATE_4X) 1822 rate = AGP2_RATE_4X; 1823 else if (rate & AGP2_RATE_2X) 1824 rate = AGP2_RATE_2X; 1825 else 1826 rate = AGP2_RATE_1X; 1827 1828 cmd = rq | sba | over4g | fw | rate; 1829 /* enable agp mode */ 1830 cmd |= AGPCMD_AGPEN; 1831 1832 return (cmd); 1833 } 1834 1835 static uint32_t 1836 agp_v3_setup(uint32_t tstatus, uint32_t mstatus, uint32_t mode) 1837 { 1838 uint32_t cmd = 0; 1839 uint32_t rq, arqsz, cal, sba, over4g, fw, rate; 1840 1841 /* 1842 * tstatus: target device status 1843 * mstatus: master device status 1844 * mode: the agp mode to be set 1845 */ 1846 1847 /* 1848 * RQ - Request Queue size 1849 * Set RQ to the min of mode and tstatus 1850 * If mode set a RQ larger than hardware can support, 1851 * use the max RQ which hardware can support. 1852 * tstatus & AGPSTAT_RQ_MASK is the max RQ hardware can support 1853 * Corelogic will enqueue agp transaction; 1854 */ 1855 rq = mode & AGPSTAT_RQ_MASK; 1856 if ((tstatus & AGPSTAT_RQ_MASK) < rq) 1857 rq = tstatus & AGPSTAT_RQ_MASK; 1858 1859 /* 1860 * ARQSZ - Asynchronous Request Queue size 1861 * Set the value equal to tstatus. 1862 * Don't allow the mode register to override values 1863 */ 1864 arqsz = tstatus & AGPSTAT_ARQSZ_MASK; 1865 1866 /* 1867 * CAL - Calibration cycle 1868 * Set to the min of tstatus and mstatus 1869 * Don't allow override by mode register 1870 */ 1871 cal = tstatus & AGPSTAT_CAL_MASK; 1872 if ((mstatus & AGPSTAT_CAL_MASK) < cal) 1873 cal = mstatus & AGPSTAT_CAL_MASK; 1874 1875 /* 1876 * SBA - Sideband Addressing 1877 * 1878 * Sideband Addressing provides an additional bus to pass requests 1879 * (address and command) to the target from the master. 1880 * 1881 * SBA in agp v3.0 must be set 1882 */ 1883 sba = AGPCMD_SBAEN; 1884 1885 /* GART64B is not set since no hardware supports it now */ 1886 1887 /* Set OVER4G if all three support it */ 1888 over4g = (tstatus & AGPSTAT_OVER4G) & (mstatus & AGPSTAT_OVER4G) 1889 & (mode & AGPSTAT_OVER4G); 1890 1891 /* 1892 * FW - fast write 1893 * 1894 * Acceleration of memory write transactions from the corelogic to the 1895 * A.G.P. master device acting like a PCI target. 1896 * 1897 * Always set FW in AGP 3.0 1898 */ 1899 fw = (tstatus & AGPSTAT_FW) & (mstatus & AGPSTAT_FW) 1900 & (mode & AGPSTAT_FW); 1901 1902 /* 1903 * Figure out the max rate 1904 * 1905 * AGP v3 support: 8X, 4X speed 1906 * 1907 * status bit meaning 1908 * --------------------------------------------- 1909 * 7:3 others 1910 * 3 1 stand for V3 support 1911 * 0:2 001:4X, 010:8X, 011:4X,8X 1912 * ---------------------------------------------- 1913 */ 1914 rate = (tstatus & AGPSTAT_RATE_MASK) & (mstatus & AGPSTAT_RATE_MASK) 1915 & (mode & AGPSTAT_RATE_MASK); 1916 if (rate & AGP3_RATE_8X) 1917 rate = AGP3_RATE_8X; 1918 else 1919 rate = AGP3_RATE_4X; 1920 1921 cmd = rq | arqsz | cal | sba | over4g | fw | rate; 1922 /* Enable AGP mode */ 1923 cmd |= AGPCMD_AGPEN; 1924 1925 return (cmd); 1926 } 1927 1928 static int 1929 agp_setup(agpgart_softstate_t *softstate, uint32_t mode) 1930 { 1931 uint32_t tstatus, mstatus; 1932 uint32_t agp_mode; 1933 1934 tstatus = softstate->asoft_info.agpki_tstatus; 1935 mstatus = softstate->asoft_info.agpki_mstatus; 1936 1937 /* 1938 * There are three kinds of AGP mode. AGP mode 1.0, 2.0, 3.0 1939 * AGP mode 2.0 is fully compatible with AGP mode 1.0, so we 1940 * only check 2.0 and 3.0 mode. AGP 3.0 device can work in 1941 * two AGP 2.0 or AGP 3.0 mode. By checking AGP status register, 1942 * we can get which mode it is working at. The working mode of 1943 * AGP master and AGP target must be consistent. That is, both 1944 * of them must work on AGP 3.0 mode or AGP 2.0 mode. 1945 */ 1946 if ((softstate->asoft_info.agpki_tver.agpv_major == 3) && 1947 (tstatus & AGPSTAT_MODE3)) { 1948 /* Master device should be 3.0 mode, too */ 1949 if ((softstate->asoft_info.agpki_mver.agpv_major != 3) || 1950 ((mstatus & AGPSTAT_MODE3) == 0)) 1951 return (EIO); 1952 1953 agp_mode = agp_v3_setup(tstatus, mstatus, mode); 1954 /* Write to the AGPCMD register of target and master devices */ 1955 if (lyr_set_agp_cmd(agp_mode, 1956 &softstate->asoft_devreg)) 1957 return (EIO); 1958 1959 softstate->asoft_mode = agp_mode; 1960 1961 return (0); 1962 } 1963 1964 /* 1965 * If agp taget device doesn't work in AGP 3.0 mode, 1966 * it must work in AGP 2.0 mode. And make sure 1967 * master device work in AGP 2.0 mode too 1968 */ 1969 if ((softstate->asoft_info.agpki_mver.agpv_major == 3) && 1970 (mstatus & AGPSTAT_MODE3)) 1971 return (EIO); 1972 1973 agp_mode = agp_v2_setup(tstatus, mstatus, mode); 1974 if (lyr_set_agp_cmd(agp_mode, &softstate->asoft_devreg)) 1975 return (EIO); 1976 softstate->asoft_mode = agp_mode; 1977 1978 return (0); 1979 } 1980 1981 /* 1982 * agp_alloc_pmem() 1983 * 1984 * Description: 1985 * This function allocates physical memory for direct mapping to userland 1986 * applications. 1987 * 1988 * Arguments: 1989 * softsate driver soft state pointer 1990 * length memory size 1991 * type AGP_NORMAL: normal agp memory, AGP_PHISYCAL: specical 1992 * memory type for intel i810 IGD 1993 * 1994 * Returns: 1995 * entryp new key table entity pointer 1996 * NULL no key table slot available 1997 */ 1998 static keytable_ent_t * 1999 agp_alloc_pmem(agpgart_softstate_t *softstate, size_t length, int type) 2000 { 2001 keytable_ent_t keyentry; 2002 keytable_ent_t *entryp; 2003 2004 ASSERT(AGP_ALIGNED(length)); 2005 bzero(&keyentry, sizeof (keytable_ent_t)); 2006 2007 keyentry.kte_pages = AGP_BYTES2PAGES(length); 2008 keyentry.kte_type = type; 2009 2010 keyentry.kte_memhdl = 2011 (agp_pmem_handle_t *)kmem_zalloc(sizeof (agp_pmem_handle_t), 2012 KM_SLEEP); 2013 2014 if (devmap_pmem_alloc(length, 2015 PMEM_SLEEP, 2016 &PMEMP(keyentry.kte_memhdl)->pmem_cookie) != DDI_SUCCESS) 2017 goto err1; 2018 2019 keyentry.kte_pfnarray = (pfn_t *)kmem_zalloc(sizeof (pfn_t) * 2020 keyentry.kte_pages, KM_SLEEP); 2021 2022 if (devmap_pmem_getpfns( 2023 PMEMP(keyentry.kte_memhdl)->pmem_cookie, 2024 0, keyentry.kte_pages, keyentry.kte_pfnarray) != DDI_SUCCESS) { 2025 AGPDB_PRINT2((CE_WARN, 2026 "agp_alloc_pmem: devmap_map_getpfns failed")); 2027 goto err2; 2028 } 2029 ASSERT(!agp_check_pfns(softstate->asoft_devreg.agprd_arctype, 2030 keyentry.kte_pfnarray, keyentry.kte_pages)); 2031 entryp = agp_fill_empty_keyent(softstate, &keyentry); 2032 2033 if (!entryp) { 2034 AGPDB_PRINT2((CE_WARN, 2035 "agp_alloc_pmem: agp_fill_empty_keyent error")); 2036 goto err2; 2037 } 2038 ASSERT((entryp->kte_key >= 0) && (entryp->kte_key < AGP_MAXKEYS)); 2039 2040 return (entryp); 2041 2042 err2: 2043 kmem_free(keyentry.kte_pfnarray, sizeof (pfn_t) * keyentry.kte_pages); 2044 keyentry.kte_pfnarray = NULL; 2045 devmap_pmem_free(PMEMP(keyentry.kte_memhdl)->pmem_cookie); 2046 PMEMP(keyentry.kte_memhdl)->pmem_cookie = NULL; 2047 err1: 2048 kmem_free(keyentry.kte_memhdl, sizeof (agp_pmem_handle_t)); 2049 keyentry.kte_memhdl = NULL; 2050 2051 return (NULL); 2052 2053 } 2054 2055 /* 2056 * agp_alloc_kmem() 2057 * 2058 * Description: 2059 * This function allocates physical memory for userland applications 2060 * by ddi interfaces. This function can only be called to allocate 2061 * small phsyical contiguous pages, usually tens of kilobytes. 2062 * 2063 * Arguments: 2064 * softsate driver soft state pointer 2065 * length memory size 2066 * 2067 * Returns: 2068 * entryp new keytable entity pointer 2069 * NULL no keytable slot available or no physical 2070 * memory available 2071 */ 2072 static keytable_ent_t * 2073 agp_alloc_kmem(agpgart_softstate_t *softstate, size_t length) 2074 { 2075 keytable_ent_t keyentry; 2076 keytable_ent_t *entryp; 2077 int ret; 2078 2079 ASSERT(AGP_ALIGNED(length)); 2080 2081 bzero(&keyentry, sizeof (keytable_ent_t)); 2082 2083 keyentry.kte_pages = AGP_BYTES2PAGES(length); 2084 keyentry.kte_type = AGP_PHYSICAL; 2085 2086 /* 2087 * Set dma_attr_sgllen to assure contiguous physical pages 2088 */ 2089 agpgart_dma_attr.dma_attr_sgllen = 1; 2090 2091 /* 4k size pages */ 2092 keyentry.kte_memhdl = kmem_zalloc(sizeof (agp_kmem_handle_t), KM_SLEEP); 2093 2094 if (ddi_dma_alloc_handle(softstate->asoft_dip, 2095 &agpgart_dma_attr, 2096 DDI_DMA_SLEEP, NULL, 2097 &(KMEMP(keyentry.kte_memhdl)->kmem_handle))) { 2098 AGPDB_PRINT2((CE_WARN, 2099 "agp_alloc_kmem: ddi_dma_allco_hanlde error")); 2100 goto err4; 2101 } 2102 2103 if ((ret = ddi_dma_mem_alloc( 2104 KMEMP(keyentry.kte_memhdl)->kmem_handle, 2105 length, 2106 &gart_dev_acc_attr, 2107 DDI_DMA_CONSISTENT, 2108 DDI_DMA_SLEEP, NULL, 2109 &KMEMP(keyentry.kte_memhdl)->kmem_kvaddr, 2110 &KMEMP(keyentry.kte_memhdl)->kmem_reallen, 2111 &KMEMP(keyentry.kte_memhdl)->kmem_acchdl)) != 0) { 2112 AGPDB_PRINT2((CE_WARN, 2113 "agp_alloc_kmem: ddi_dma_mem_alloc error")); 2114 2115 goto err3; 2116 } 2117 2118 ret = ddi_dma_addr_bind_handle( 2119 KMEMP(keyentry.kte_memhdl)->kmem_handle, 2120 NULL, 2121 KMEMP(keyentry.kte_memhdl)->kmem_kvaddr, 2122 length, 2123 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 2124 DDI_DMA_SLEEP, 2125 NULL, 2126 &KMEMP(keyentry.kte_memhdl)->kmem_dcookie, 2127 &KMEMP(keyentry.kte_memhdl)->kmem_cookies_num); 2128 2129 /* 2130 * Even dma_attr_sgllen = 1, ddi_dma_addr_bind_handle may return more 2131 * than one cookie, we check this in the if statement. 2132 */ 2133 2134 if ((ret != DDI_DMA_MAPPED) || 2135 (KMEMP(keyentry.kte_memhdl)->kmem_cookies_num != 1)) { 2136 AGPDB_PRINT2((CE_WARN, 2137 "agp_alloc_kmem: can not alloc physical memory properly")); 2138 goto err2; 2139 } 2140 2141 keyentry.kte_pfnarray = (pfn_t *)kmem_zalloc(sizeof (pfn_t) * 2142 keyentry.kte_pages, KM_SLEEP); 2143 2144 if (kmem_getpfns( 2145 KMEMP(keyentry.kte_memhdl)->kmem_handle, 2146 &KMEMP(keyentry.kte_memhdl)->kmem_dcookie, 2147 KMEMP(keyentry.kte_memhdl)->kmem_cookies_num, 2148 keyentry.kte_pfnarray)) { 2149 AGPDB_PRINT2((CE_WARN, "agp_alloc_kmem: get pfn array error")); 2150 goto err1; 2151 } 2152 2153 ASSERT(!agp_check_pfns(softstate->asoft_devreg.agprd_arctype, 2154 keyentry.kte_pfnarray, keyentry.kte_pages)); 2155 entryp = agp_fill_empty_keyent(softstate, &keyentry); 2156 if (!entryp) { 2157 AGPDB_PRINT2((CE_WARN, 2158 "agp_alloc_kmem: agp_fill_empty_keyent error")); 2159 2160 goto err1; 2161 } 2162 ASSERT((entryp->kte_key >= 0) && (entryp->kte_key < AGP_MAXKEYS)); 2163 2164 return (entryp); 2165 2166 err1: 2167 kmem_free(keyentry.kte_pfnarray, sizeof (pfn_t) * keyentry.kte_pages); 2168 keyentry.kte_pfnarray = NULL; 2169 (void) ddi_dma_unbind_handle(KMEMP(keyentry.kte_memhdl)->kmem_handle); 2170 KMEMP(keyentry.kte_memhdl)->kmem_cookies_num = 0; 2171 err2: 2172 ddi_dma_mem_free(&KMEMP(keyentry.kte_memhdl)->kmem_acchdl); 2173 KMEMP(keyentry.kte_memhdl)->kmem_acchdl = NULL; 2174 KMEMP(keyentry.kte_memhdl)->kmem_reallen = 0; 2175 KMEMP(keyentry.kte_memhdl)->kmem_kvaddr = NULL; 2176 err3: 2177 ddi_dma_free_handle(&(KMEMP(keyentry.kte_memhdl)->kmem_handle)); 2178 KMEMP(keyentry.kte_memhdl)->kmem_handle = NULL; 2179 err4: 2180 kmem_free(keyentry.kte_memhdl, sizeof (agp_kmem_handle_t)); 2181 keyentry.kte_memhdl = NULL; 2182 return (NULL); 2183 2184 } 2185 2186 /* 2187 * agp_alloc_mem() 2188 * 2189 * Description: 2190 * This function allocate physical memory for userland applications, 2191 * in order to save kernel virtual space, we use the direct mapping 2192 * memory interface if it is available. 2193 * 2194 * Arguments: 2195 * st driver soft state pointer 2196 * length memory size 2197 * type AGP_NORMAL: normal agp memory, AGP_PHISYCAL: specical 2198 * memory type for intel i810 IGD 2199 * 2200 * Returns: 2201 * NULL Invalid memory type or can not allocate memory 2202 * Keytable entry pointer returned by agp_alloc_kmem or agp_alloc_pmem 2203 */ 2204 static keytable_ent_t * 2205 agp_alloc_mem(agpgart_softstate_t *st, size_t length, int type) 2206 { 2207 2208 /* 2209 * AGP_PHYSICAL type require contiguous physical pages exported 2210 * to X drivers, like i810 HW cursor, ARGB cursor. the number of 2211 * pages needed is usuallysmall and contiguous, 4K, 16K. So we 2212 * use DDI interface to allocated such memory. And X use xsvc 2213 * drivers to map this memory into its own address space. 2214 */ 2215 ASSERT(st); 2216 2217 switch (type) { 2218 case AGP_NORMAL: 2219 return (agp_alloc_pmem(st, length, type)); 2220 case AGP_PHYSICAL: 2221 return (agp_alloc_kmem(st, length)); 2222 default: 2223 return (NULL); 2224 } 2225 } 2226 2227 /* 2228 * free_gart_table() 2229 * 2230 * Description: 2231 * This function frees the gart table memory allocated by driver. 2232 * Must disable gart table before calling this function. 2233 * 2234 * Arguments: 2235 * softstate driver soft state pointer 2236 * 2237 */ 2238 static void 2239 free_gart_table(agpgart_softstate_t *st) 2240 { 2241 2242 if (st->gart_dma_handle == NULL) 2243 return; 2244 2245 (void) ddi_dma_unbind_handle(st->gart_dma_handle); 2246 ddi_dma_mem_free(&st->gart_dma_acc_handle); 2247 st->gart_dma_acc_handle = NULL; 2248 ddi_dma_free_handle(&st->gart_dma_handle); 2249 st->gart_dma_handle = NULL; 2250 st->gart_vbase = 0; 2251 st->gart_size = 0; 2252 } 2253 2254 /* 2255 * alloc_gart_table() 2256 * 2257 * Description: 2258 * This function allocates one physical continuous gart table. 2259 * INTEL integrated video device except i810 have their special 2260 * video bios; No need to allocate gart table for them. 2261 * 2262 * Arguments: 2263 * st driver soft state pointer 2264 * 2265 * Returns: 2266 * 0 success 2267 * -1 can not allocate gart tabl 2268 */ 2269 static int 2270 alloc_gart_table(agpgart_softstate_t *st) 2271 { 2272 int num_pages; 2273 size_t table_size; 2274 int ret = DDI_SUCCESS; 2275 ddi_dma_cookie_t cookie; 2276 uint32_t num_cookies; 2277 2278 num_pages = AGP_MB2PAGES(st->asoft_info.agpki_apersize); 2279 2280 /* 2281 * Only 40-bit maximum physical memory is supported by today's 2282 * AGP hardware (32-bit gart tables can hold 40-bit memory addresses). 2283 * No one supports 64-bit gart entries now, so the size of gart 2284 * entries defaults to 32-bit though AGP3.0 specifies the possibility 2285 * of 64-bit gart entries. 2286 */ 2287 2288 table_size = num_pages * (sizeof (uint32_t)); 2289 2290 /* 2291 * Only AMD64 can put gart table above 4G, 40 bits at maximum 2292 */ 2293 if (st->asoft_devreg.agprd_arctype == ARC_AMD64AGP) 2294 garttable_dma_attr.dma_attr_addr_hi = 0xffffffffffLL; 2295 else 2296 garttable_dma_attr.dma_attr_addr_hi = 0xffffffffU; 2297 /* Allocate physical continuous page frame for gart table */ 2298 if (ret = ddi_dma_alloc_handle(st->asoft_dip, 2299 &garttable_dma_attr, 2300 DDI_DMA_SLEEP, 2301 NULL, &st->gart_dma_handle)) { 2302 AGPDB_PRINT2((CE_WARN, 2303 "alloc_gart_table: ddi_dma_alloc_handle failed")); 2304 goto err3; 2305 } 2306 2307 if (ret = ddi_dma_mem_alloc(st->gart_dma_handle, 2308 table_size, 2309 &gart_dev_acc_attr, 2310 DDI_DMA_CONSISTENT, 2311 DDI_DMA_SLEEP, NULL, 2312 &st->gart_vbase, 2313 &st->gart_size, 2314 &st->gart_dma_acc_handle)) { 2315 AGPDB_PRINT2((CE_WARN, 2316 "alloc_gart_table: ddi_dma_mem_alloc failed")); 2317 goto err2; 2318 2319 } 2320 2321 ret = ddi_dma_addr_bind_handle(st->gart_dma_handle, 2322 NULL, st->gart_vbase, 2323 table_size, 2324 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 2325 DDI_DMA_SLEEP, NULL, 2326 &cookie, &num_cookies); 2327 2328 st->gart_pbase = cookie.dmac_address; 2329 2330 if ((ret != DDI_DMA_MAPPED) || (num_cookies != 1)) { 2331 if (num_cookies > 1) 2332 (void) ddi_dma_unbind_handle(st->gart_dma_handle); 2333 AGPDB_PRINT2((CE_WARN, 2334 "alloc_gart_table: alloc contiguous phys memory failed")); 2335 goto err1; 2336 } 2337 2338 return (0); 2339 err1: 2340 ddi_dma_mem_free(&st->gart_dma_acc_handle); 2341 st->gart_dma_acc_handle = NULL; 2342 err2: 2343 ddi_dma_free_handle(&st->gart_dma_handle); 2344 st->gart_dma_handle = NULL; 2345 err3: 2346 st->gart_pbase = 0; 2347 st->gart_size = 0; 2348 st->gart_vbase = 0; 2349 2350 return (-1); 2351 } 2352 2353 /* 2354 * agp_add_to_gart() 2355 * 2356 * Description: 2357 * This function fills the gart table entries by a given page frame number 2358 * array and set up the agp aperture page to physical memory page 2359 * translation. 2360 * Arguments: 2361 * type valid sytem arc types ARC_AMD64AGP, ARC_INTELAGP, 2362 * ARC_AMD64AGP 2363 * pfnarray allocated physical page frame number array 2364 * pg_offset agp aperture start page to be bound 2365 * entries the number of pages to be bound 2366 * dma_hdl gart table dma memory handle 2367 * tablep gart table kernel virtual address 2368 * Returns: 2369 * -1 failed 2370 * 0 success 2371 */ 2372 static int 2373 agp_add_to_gart( 2374 agp_arc_type_t type, 2375 pfn_t *pfnarray, 2376 uint32_t pg_offset, 2377 uint32_t entries, 2378 ddi_dma_handle_t dma_hdl, 2379 uint32_t *tablep) 2380 { 2381 int items = 0; 2382 uint32_t *entryp; 2383 uint32_t itemv; 2384 2385 entryp = tablep + pg_offset; 2386 while (items < entries) { 2387 if (pfn2gartentry(type, pfnarray[items], &itemv)) 2388 break; 2389 *(entryp + items) = itemv; 2390 items++; 2391 } 2392 if (items < entries) 2393 return (-1); 2394 2395 (void) ddi_dma_sync(dma_hdl, pg_offset * sizeof (uint32_t), 2396 entries * sizeof (uint32_t), DDI_DMA_SYNC_FORDEV); 2397 2398 return (0); 2399 } 2400 2401 /* 2402 * agp_bind_key() 2403 * 2404 * Description: 2405 * This function will call low level gart table access functions to 2406 * set up gart table translation. Also it will do some sanity 2407 * checking on key table entry. 2408 * 2409 * Arguments: 2410 * softstate driver soft state pointer 2411 * keyent key table entity pointer to be bound 2412 * pg_offset aperture start page to be bound 2413 * Returns: 2414 * EINVAL not a valid operation 2415 */ 2416 static int 2417 agp_bind_key(agpgart_softstate_t *softstate, 2418 keytable_ent_t *keyent, uint32_t pg_offset) 2419 { 2420 uint64_t pg_end; 2421 int ret = 0; 2422 2423 ASSERT(keyent); 2424 ASSERT((keyent->kte_key >= 0) && (keyent->kte_key < AGP_MAXKEYS)); 2425 ASSERT(mutex_owned(&softstate->asoft_instmutex)); 2426 2427 pg_end = pg_offset + keyent->kte_pages; 2428 2429 if (pg_end > AGP_MB2PAGES(softstate->asoft_info.agpki_apersize)) { 2430 AGPDB_PRINT2((CE_WARN, 2431 "agp_bind_key: key=0x%x,exceed aper range", 2432 keyent->kte_key)); 2433 2434 return (EINVAL); 2435 } 2436 2437 if (agp_check_off(softstate->asoft_table, 2438 pg_offset, keyent->kte_pages)) { 2439 AGPDB_PRINT2((CE_WARN, 2440 "agp_bind_key: pg_offset=0x%x, pages=0x%lx overlaped", 2441 pg_offset, keyent->kte_pages)); 2442 return (EINVAL); 2443 } 2444 2445 ASSERT(keyent->kte_pfnarray != NULL); 2446 2447 switch (softstate->asoft_devreg.agprd_arctype) { 2448 case ARC_IGD810: 2449 case ARC_IGD830: 2450 ret = lyr_i8xx_add_to_gtt(pg_offset, keyent, 2451 &softstate->asoft_devreg); 2452 if (ret) 2453 return (EIO); 2454 break; 2455 case ARC_INTELAGP: 2456 case ARC_AMD64AGP: 2457 ret = agp_add_to_gart( 2458 softstate->asoft_devreg.agprd_arctype, 2459 keyent->kte_pfnarray, 2460 pg_offset, 2461 keyent->kte_pages, 2462 softstate->gart_dma_handle, 2463 (uint32_t *)softstate->gart_vbase); 2464 if (ret) 2465 return (EINVAL); 2466 /* Flush GTLB table */ 2467 lyr_flush_gart_cache(&softstate->asoft_devreg); 2468 break; 2469 default: 2470 AGPDB_PRINT2((CE_WARN, 2471 "agp_bind_key: arc type = 0x%x unsupported", 2472 softstate->asoft_devreg.agprd_arctype)); 2473 return (EINVAL); 2474 } 2475 return (0); 2476 } 2477 2478 static int 2479 agpgart_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 2480 { 2481 int instance; 2482 agpgart_softstate_t *softstate; 2483 2484 if (cmd != DDI_ATTACH) { 2485 AGPDB_PRINT2((CE_WARN, 2486 "agpgart_attach: only attach op supported")); 2487 return (DDI_FAILURE); 2488 } 2489 instance = ddi_get_instance(dip); 2490 2491 if (ddi_soft_state_zalloc(agpgart_glob_soft_handle, instance) 2492 != DDI_SUCCESS) { 2493 AGPDB_PRINT2((CE_WARN, 2494 "agpgart_attach: soft state zalloc failed")); 2495 goto err1; 2496 2497 } 2498 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 2499 mutex_init(&softstate->asoft_instmutex, NULL, MUTEX_DRIVER, NULL); 2500 softstate->asoft_dip = dip; 2501 /* 2502 * Allocate LDI identifier for agpgart driver 2503 * Agpgart driver is the kernel consumer 2504 */ 2505 if (ldi_ident_from_dip(dip, &softstate->asoft_li)) { 2506 AGPDB_PRINT2((CE_WARN, 2507 "agpgart_attach: LDI indentifier allcation failed")); 2508 goto err2; 2509 } 2510 2511 softstate->asoft_devreg.agprd_arctype = ARC_UNKNOWN; 2512 /* Install agp kstat */ 2513 if (agp_init_kstats(softstate)) { 2514 AGPDB_PRINT2((CE_WARN, "agpgart_attach: init kstats error")); 2515 goto err3; 2516 } 2517 /* 2518 * devfs will create /dev/agpgart 2519 * and /devices/agpgart:agpgart 2520 */ 2521 2522 if (ddi_create_minor_node(dip, AGPGART_DEVNODE, S_IFCHR, 2523 AGP_INST2MINOR(instance), 2524 DDI_NT_AGP_PSEUDO, 0)) { 2525 AGPDB_PRINT2((CE_WARN, 2526 "agpgart_attach: Can not create minor node")); 2527 goto err4; 2528 } 2529 2530 softstate->asoft_table = kmem_zalloc( 2531 AGP_MAXKEYS * (sizeof (keytable_ent_t)), 2532 KM_SLEEP); 2533 2534 return (DDI_SUCCESS); 2535 err4: 2536 agp_fini_kstats(softstate); 2537 err3: 2538 ldi_ident_release(softstate->asoft_li); 2539 err2: 2540 ddi_soft_state_free(agpgart_glob_soft_handle, instance); 2541 err1: 2542 return (DDI_FAILURE); 2543 } 2544 2545 static int 2546 agpgart_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2547 { 2548 int instance; 2549 agpgart_softstate_t *st; 2550 2551 instance = ddi_get_instance(dip); 2552 2553 st = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 2554 2555 if (cmd != DDI_DETACH) 2556 return (DDI_FAILURE); 2557 2558 /* 2559 * Caller should free all the memory allocated explicitly. 2560 * We release the memory allocated by caller which is not 2561 * properly freed. mutex_enter here make sure assertion on 2562 * softstate mutex success in agp_dealloc_mem. 2563 */ 2564 mutex_enter(&st->asoft_instmutex); 2565 if (agp_del_allkeys(st)) { 2566 AGPDB_PRINT2((CE_WARN, "agpgart_detach: agp_del_allkeys err")); 2567 AGPDB_PRINT2((CE_WARN, 2568 "you might free agp memory exported to your applications")); 2569 2570 mutex_exit(&st->asoft_instmutex); 2571 return (DDI_FAILURE); 2572 } 2573 mutex_exit(&st->asoft_instmutex); 2574 if (st->asoft_table) { 2575 kmem_free(st->asoft_table, 2576 AGP_MAXKEYS * (sizeof (keytable_ent_t))); 2577 st->asoft_table = 0; 2578 } 2579 2580 ddi_remove_minor_node(dip, AGPGART_DEVNODE); 2581 agp_fini_kstats(st); 2582 ldi_ident_release(st->asoft_li); 2583 mutex_destroy(&st->asoft_instmutex); 2584 ddi_soft_state_free(agpgart_glob_soft_handle, instance); 2585 2586 return (DDI_SUCCESS); 2587 } 2588 2589 /*ARGSUSED*/ 2590 static int 2591 agpgart_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 2592 void **resultp) 2593 { 2594 agpgart_softstate_t *st; 2595 int instance, rval = DDI_FAILURE; 2596 dev_t dev; 2597 2598 switch (cmd) { 2599 case DDI_INFO_DEVT2DEVINFO: 2600 dev = (dev_t)arg; 2601 instance = AGP_DEV2INST(dev); 2602 st = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 2603 if (st != NULL) { 2604 mutex_enter(&st->asoft_instmutex); 2605 *resultp = st->asoft_dip; 2606 mutex_exit(&st->asoft_instmutex); 2607 rval = DDI_SUCCESS; 2608 } else 2609 *resultp = NULL; 2610 2611 break; 2612 case DDI_INFO_DEVT2INSTANCE: 2613 dev = (dev_t)arg; 2614 instance = AGP_DEV2INST(dev); 2615 *resultp = (void *)(uintptr_t)instance; 2616 rval = DDI_SUCCESS; 2617 2618 break; 2619 default: 2620 break; 2621 } 2622 2623 return (rval); 2624 } 2625 2626 /* 2627 * agpgart_open() 2628 * 2629 * Description: 2630 * This function is the driver open entry point. If it is the 2631 * first time the agpgart driver is opened, the driver will 2632 * open other agp related layered drivers and set up the agpgart 2633 * table properly. 2634 * 2635 * Arguments: 2636 * dev device number pointer 2637 * openflags open flags 2638 * otyp OTYP_BLK, OTYP_CHR 2639 * credp user's credential's struct pointer 2640 * 2641 * Returns: 2642 * ENXIO operation error 2643 * EAGAIN resoure temporarily unvailable 2644 * 0 success 2645 */ 2646 /*ARGSUSED*/ 2647 static int 2648 agpgart_open(dev_t *dev, int openflags, int otyp, cred_t *credp) 2649 { 2650 int instance = AGP_DEV2INST(*dev); 2651 agpgart_softstate_t *softstate; 2652 int rc = 0; 2653 2654 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 2655 if (softstate == NULL) { 2656 AGPDB_PRINT2((CE_WARN, "agpgart_open: get soft state err")); 2657 return (ENXIO); 2658 } 2659 mutex_enter(&softstate->asoft_instmutex); 2660 2661 if (softstate->asoft_opened) { 2662 softstate->asoft_opened++; 2663 mutex_exit(&softstate->asoft_instmutex); 2664 return (0); 2665 } 2666 2667 /* 2668 * The driver is opened first time, so we initialize layered 2669 * driver interface and softstate member here. 2670 */ 2671 softstate->asoft_pgused = 0; 2672 if (lyr_init(&softstate->asoft_devreg, softstate->asoft_li)) { 2673 AGPDB_PRINT2((CE_WARN, "agpgart_open: lyr_init failed")); 2674 mutex_exit(&softstate->asoft_instmutex); 2675 return (EAGAIN); 2676 } 2677 2678 /* Call into layered driver */ 2679 if (lyr_get_info(&softstate->asoft_info, &softstate->asoft_devreg)) { 2680 AGPDB_PRINT2((CE_WARN, "agpgart_open: lyr_get_info error")); 2681 lyr_end(&softstate->asoft_devreg); 2682 mutex_exit(&softstate->asoft_instmutex); 2683 return (EIO); 2684 } 2685 2686 /* 2687 * BIOS already set up gtt table for ARC_IGD830 2688 */ 2689 if (IS_INTEL_830(softstate->asoft_devreg.agprd_arctype)) { 2690 softstate->asoft_opened++; 2691 2692 softstate->asoft_pgtotal = 2693 get_max_pages(softstate->asoft_info.agpki_apersize); 2694 2695 if (lyr_config_devices(&softstate->asoft_devreg)) { 2696 AGPDB_PRINT2((CE_WARN, 2697 "agpgart_open: lyr_config_devices error")); 2698 lyr_end(&softstate->asoft_devreg); 2699 mutex_exit(&softstate->asoft_instmutex); 2700 2701 return (EIO); 2702 } 2703 mutex_exit(&softstate->asoft_instmutex); 2704 return (0); 2705 } 2706 2707 rc = alloc_gart_table(softstate); 2708 2709 /* 2710 * Allocate physically contiguous pages for AGP arc or 2711 * i810 arc. If failed, divide aper_size by 2 to 2712 * reduce gart table size until 4 megabytes. This 2713 * is just a workaround for systems with very few 2714 * physically contiguous memory. 2715 */ 2716 if (rc) { 2717 while ((softstate->asoft_info.agpki_apersize >= 4) && 2718 (alloc_gart_table(softstate))) { 2719 softstate->asoft_info.agpki_apersize >>= 1; 2720 } 2721 if (softstate->asoft_info.agpki_apersize >= 4) 2722 rc = 0; 2723 } 2724 2725 if (rc != 0) { 2726 AGPDB_PRINT2((CE_WARN, 2727 "agpgart_open: alloc gart table failed")); 2728 lyr_end(&softstate->asoft_devreg); 2729 mutex_exit(&softstate->asoft_instmutex); 2730 return (EAGAIN); 2731 } 2732 2733 softstate->asoft_pgtotal = 2734 get_max_pages(softstate->asoft_info.agpki_apersize); 2735 /* 2736 * BIOS doesn't initialize GTT for i810, 2737 * So i810 GTT must be created by driver. 2738 * 2739 * Set up gart table and enable it. 2740 */ 2741 if (lyr_set_gart_addr(softstate->gart_pbase, 2742 &softstate->asoft_devreg)) { 2743 AGPDB_PRINT2((CE_WARN, 2744 "agpgart_open: set gart table addr failed")); 2745 free_gart_table(softstate); 2746 lyr_end(&softstate->asoft_devreg); 2747 mutex_exit(&softstate->asoft_instmutex); 2748 return (EIO); 2749 } 2750 if (lyr_config_devices(&softstate->asoft_devreg)) { 2751 AGPDB_PRINT2((CE_WARN, 2752 "agpgart_open: lyr_config_devices failed")); 2753 free_gart_table(softstate); 2754 lyr_end(&softstate->asoft_devreg); 2755 mutex_exit(&softstate->asoft_instmutex); 2756 return (EIO); 2757 } 2758 2759 softstate->asoft_opened++; 2760 mutex_exit(&softstate->asoft_instmutex); 2761 2762 return (0); 2763 } 2764 2765 /* 2766 * agpgart_close() 2767 * 2768 * Description: 2769 * agpgart_close will release resources allocated in the first open 2770 * and close other open layered drivers. Also it frees the memory 2771 * allocated by ioctls. 2772 * 2773 * Arguments: 2774 * dev device number 2775 * flag file status flag 2776 * otyp OTYP_BLK, OTYP_CHR 2777 * credp user's credential's struct pointer 2778 * 2779 * Returns: 2780 * ENXIO not an error, to support "deferred attach" 2781 * 0 success 2782 */ 2783 /*ARGSUSED*/ 2784 static int 2785 agpgart_close(dev_t dev, int flag, int otyp, cred_t *credp) 2786 { 2787 int instance = AGP_DEV2INST(dev); 2788 agpgart_softstate_t *softstate; 2789 2790 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 2791 if (softstate == NULL) { 2792 AGPDB_PRINT2((CE_WARN, "agpgart_close: get soft state err")); 2793 return (ENXIO); 2794 } 2795 2796 mutex_enter(&softstate->asoft_instmutex); 2797 ASSERT(softstate->asoft_opened); 2798 2799 2800 /* 2801 * If the last process close this device is not the controlling 2802 * process, also release the control over agpgart driver here if the 2803 * the controlling process fails to release the control before it 2804 * close the driver. 2805 */ 2806 if (softstate->asoft_acquired == 1) { 2807 AGPDB_PRINT2((CE_WARN, 2808 "agpgart_close: auto release control over driver")); 2809 release_control(softstate); 2810 } 2811 2812 if (lyr_unconfig_devices(&softstate->asoft_devreg)) { 2813 AGPDB_PRINT2((CE_WARN, 2814 "agpgart_close: lyr_unconfig_device error")); 2815 mutex_exit(&softstate->asoft_instmutex); 2816 return (EIO); 2817 } 2818 softstate->asoft_agpen = 0; 2819 2820 if (!IS_INTEL_830(softstate->asoft_devreg.agprd_arctype)) { 2821 free_gart_table(softstate); 2822 } 2823 2824 lyr_end(&softstate->asoft_devreg); 2825 2826 /* 2827 * This statement must be positioned before agp_del_allkeys 2828 * agp_dealloc_mem indirectly called by agp_del_allkeys 2829 * will test this variable. 2830 */ 2831 softstate->asoft_opened = 0; 2832 2833 /* 2834 * Free the memory allocated by user applications which 2835 * was never deallocated. 2836 */ 2837 (void) agp_del_allkeys(softstate); 2838 2839 mutex_exit(&softstate->asoft_instmutex); 2840 2841 return (0); 2842 } 2843 2844 static int 2845 ioctl_agpgart_info(agpgart_softstate_t *softstate, void *arg, int flags) 2846 { 2847 agp_info_t infostruct; 2848 #ifdef _MULTI_DATAMODEL 2849 agp_info32_t infostruct32; 2850 #endif 2851 2852 bzero(&infostruct, sizeof (agp_info_t)); 2853 2854 #ifdef _MULTI_DATAMODEL 2855 bzero(&infostruct32, sizeof (agp_info32_t)); 2856 if (ddi_model_convert_from(flags & FMODELS) == DDI_MODEL_ILP32) { 2857 if (copyinfo(softstate, &infostruct)) 2858 return (EINVAL); 2859 2860 agpinfo_default_to_32(infostruct, infostruct32); 2861 if (ddi_copyout(&infostruct32, arg, 2862 sizeof (agp_info32_t), flags) != 0) 2863 return (EFAULT); 2864 2865 return (0); 2866 } 2867 #endif /* _MULTI_DATAMODEL */ 2868 if (copyinfo(softstate, &infostruct)) 2869 return (EINVAL); 2870 2871 if (ddi_copyout(&infostruct, arg, sizeof (agp_info_t), flags) != 0) { 2872 return (EFAULT); 2873 } 2874 2875 return (0); 2876 } 2877 2878 static int 2879 ioctl_agpgart_acquire(agpgart_softstate_t *st) 2880 { 2881 if (st->asoft_acquired) { 2882 AGPDB_PRINT2((CE_WARN, "ioctl_acquire: already acquired")); 2883 return (EBUSY); 2884 } 2885 acquire_control(st); 2886 return (0); 2887 } 2888 2889 static int 2890 ioctl_agpgart_release(agpgart_softstate_t *st) 2891 { 2892 if (is_controlling_proc(st) < 0) { 2893 AGPDB_PRINT2((CE_WARN, 2894 "ioctl_agpgart_release: not a controlling process")); 2895 return (EPERM); 2896 } 2897 release_control(st); 2898 return (0); 2899 } 2900 2901 static int 2902 ioctl_agpgart_setup(agpgart_softstate_t *st, void *arg, int flags) 2903 { 2904 agp_setup_t data; 2905 int rc = 0; 2906 2907 if (is_controlling_proc(st) < 0) { 2908 AGPDB_PRINT2((CE_WARN, 2909 "ioctl_agpgart_setup: not a controlling process")); 2910 return (EPERM); 2911 } 2912 2913 if (!IS_TRUE_AGP(st->asoft_devreg.agprd_arctype)) { 2914 AGPDB_PRINT2((CE_WARN, 2915 "ioctl_agpgart_setup: no true agp bridge")); 2916 return (EINVAL); 2917 } 2918 2919 if (ddi_copyin(arg, &data, sizeof (agp_setup_t), flags) != 0) 2920 return (EFAULT); 2921 2922 if (rc = agp_setup(st, data.agps_mode)) 2923 return (rc); 2924 /* Store agp mode status for kstat */ 2925 st->asoft_agpen = 1; 2926 return (0); 2927 } 2928 2929 static int 2930 ioctl_agpgart_alloc(agpgart_softstate_t *st, void *arg, int flags) 2931 { 2932 agp_allocate_t alloc_info; 2933 keytable_ent_t *entryp; 2934 size_t length; 2935 uint64_t pg_num; 2936 2937 if (is_controlling_proc(st) < 0) { 2938 AGPDB_PRINT2((CE_WARN, 2939 "ioctl_agpgart_alloc: not a controlling process")); 2940 return (EPERM); 2941 } 2942 2943 if (ddi_copyin(arg, &alloc_info, 2944 sizeof (agp_allocate_t), flags) != 0) { 2945 return (EFAULT); 2946 } 2947 pg_num = st->asoft_pgused + alloc_info.agpa_pgcount; 2948 if (pg_num > st->asoft_pgtotal) { 2949 AGPDB_PRINT2((CE_WARN, 2950 "ioctl_agpgart_alloc: exceeding the memory pages limit")); 2951 AGPDB_PRINT2((CE_WARN, 2952 "ioctl_agpgart_alloc: request %x pages failed", 2953 alloc_info.agpa_pgcount)); 2954 AGPDB_PRINT2((CE_WARN, 2955 "ioctl_agpgart_alloc: pages used %x total is %x", 2956 st->asoft_pgused, st->asoft_pgtotal)); 2957 2958 return (EINVAL); 2959 } 2960 2961 length = AGP_PAGES2BYTES(alloc_info.agpa_pgcount); 2962 entryp = agp_alloc_mem(st, length, alloc_info.agpa_type); 2963 if (!entryp) { 2964 AGPDB_PRINT2((CE_WARN, 2965 "ioctl_agpgart_alloc: allocate 0x%lx bytes failed", 2966 length)); 2967 return (ENOMEM); 2968 } 2969 ASSERT((entryp->kte_key >= 0) && (entryp->kte_key < AGP_MAXKEYS)); 2970 alloc_info.agpa_key = entryp->kte_key; 2971 if (alloc_info.agpa_type == AGP_PHYSICAL) { 2972 alloc_info.agpa_physical = 2973 (uint32_t)(entryp->kte_pfnarray[0] << AGP_PAGE_SHIFT); 2974 } 2975 /* Update the memory pagse used */ 2976 st->asoft_pgused += alloc_info.agpa_pgcount; 2977 2978 if (ddi_copyout(&alloc_info, arg, 2979 sizeof (agp_allocate_t), flags) != 0) { 2980 2981 return (EFAULT); 2982 } 2983 2984 return (0); 2985 } 2986 2987 static int 2988 ioctl_agpgart_dealloc(agpgart_softstate_t *st, intptr_t arg) 2989 { 2990 int key; 2991 keytable_ent_t *keyent; 2992 2993 if (is_controlling_proc(st) < 0) { 2994 AGPDB_PRINT2((CE_WARN, 2995 "ioctl_agpgart_dealloc: not a controlling process")); 2996 return (EPERM); 2997 } 2998 key = (int)arg; 2999 if ((key >= AGP_MAXKEYS) || key < 0) { 3000 return (EINVAL); 3001 } 3002 keyent = &st->asoft_table[key]; 3003 if (!keyent->kte_memhdl) { 3004 return (EINVAL); 3005 } 3006 3007 if (agp_dealloc_mem(st, keyent)) 3008 return (EINVAL); 3009 3010 /* Update the memory pages used */ 3011 st->asoft_pgused -= keyent->kte_pages; 3012 bzero(keyent, sizeof (keytable_ent_t)); 3013 3014 return (0); 3015 } 3016 3017 static int 3018 ioctl_agpgart_bind(agpgart_softstate_t *st, void *arg, int flags) 3019 { 3020 agp_bind_t bind_info; 3021 keytable_ent_t *keyent; 3022 int key; 3023 uint32_t pg_offset; 3024 int retval = 0; 3025 3026 if (is_controlling_proc(st) < 0) { 3027 AGPDB_PRINT2((CE_WARN, 3028 "ioctl_agpgart_bind: not a controlling process")); 3029 return (EPERM); 3030 } 3031 3032 if (ddi_copyin(arg, &bind_info, sizeof (agp_bind_t), flags) != 0) { 3033 return (EFAULT); 3034 } 3035 3036 key = bind_info.agpb_key; 3037 if ((key >= AGP_MAXKEYS) || key < 0) { 3038 AGPDB_PRINT2((CE_WARN, "ioctl_agpgart_bind: invalid key")); 3039 return (EINVAL); 3040 } 3041 3042 if (IS_INTEL_830(st->asoft_devreg.agprd_arctype)) { 3043 if (AGP_PAGES2KB(bind_info.agpb_pgstart) < 3044 st->asoft_info.agpki_presize) { 3045 AGPDB_PRINT2((CE_WARN, 3046 "ioctl_agpgart_bind: bind to prealloc area " 3047 "pgstart = %dKB < presize = %ldKB", 3048 AGP_PAGES2KB(bind_info.agpb_pgstart), 3049 st->asoft_info.agpki_presize)); 3050 return (EINVAL); 3051 } 3052 } 3053 3054 pg_offset = bind_info.agpb_pgstart; 3055 keyent = &st->asoft_table[key]; 3056 if (!keyent->kte_memhdl) { 3057 AGPDB_PRINT2((CE_WARN, 3058 "ioctl_agpgart_bind: Key = 0x%x can't get keyenty", 3059 key)); 3060 return (EINVAL); 3061 } 3062 3063 if (keyent->kte_bound != 0) { 3064 AGPDB_PRINT2((CE_WARN, 3065 "ioctl_agpgart_bind: Key = 0x%x already bound", 3066 key)); 3067 return (EINVAL); 3068 } 3069 retval = agp_bind_key(st, keyent, pg_offset); 3070 3071 if (retval == 0) { 3072 keyent->kte_pgoff = pg_offset; 3073 keyent->kte_bound = 1; 3074 } 3075 3076 return (retval); 3077 } 3078 3079 static int 3080 ioctl_agpgart_unbind(agpgart_softstate_t *st, void *arg, int flags) 3081 { 3082 int key, retval = 0; 3083 agp_unbind_t unbindinfo; 3084 keytable_ent_t *keyent; 3085 3086 if (is_controlling_proc(st) < 0) { 3087 AGPDB_PRINT2((CE_WARN, 3088 "ioctl_agpgart_bind: not a controlling process")); 3089 return (EPERM); 3090 } 3091 3092 if (ddi_copyin(arg, &unbindinfo, sizeof (unbindinfo), flags) != 0) { 3093 return (EFAULT); 3094 } 3095 key = unbindinfo.agpu_key; 3096 if ((key >= AGP_MAXKEYS) || key < 0) { 3097 AGPDB_PRINT2((CE_WARN, "ioctl_agpgart_unbind: invalid key")); 3098 return (EINVAL); 3099 } 3100 keyent = &st->asoft_table[key]; 3101 if (!keyent->kte_bound) { 3102 return (EINVAL); 3103 } 3104 3105 if ((retval = agp_unbind_key(st, keyent)) != 0) 3106 return (retval); 3107 3108 return (0); 3109 } 3110 3111 /*ARGSUSED*/ 3112 static int 3113 agpgart_ioctl(dev_t dev, int cmd, intptr_t intarg, int flags, 3114 cred_t *credp, int *rvalp) 3115 { 3116 int instance; 3117 int retval = 0; 3118 void *arg = (void*)intarg; 3119 3120 agpgart_softstate_t *softstate; 3121 3122 instance = AGP_DEV2INST(dev); 3123 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 3124 if (softstate == NULL) { 3125 AGPDB_PRINT2((CE_WARN, "agpgart_ioctl: get soft state err")); 3126 return (ENXIO); 3127 } 3128 3129 if ((cmd != AGPIOC_INFO) && secpolicy_gart_access(credp)) { 3130 AGPDB_PRINT2((CE_WARN, "agpgart_ioctl: permission denied")); 3131 return (EPERM); 3132 } 3133 3134 mutex_enter(&softstate->asoft_instmutex); 3135 3136 switch (cmd) { 3137 case AGPIOC_INFO: 3138 retval = ioctl_agpgart_info(softstate, arg, flags); 3139 break; 3140 case AGPIOC_ACQUIRE: 3141 retval = ioctl_agpgart_acquire(softstate); 3142 break; 3143 case AGPIOC_RELEASE: 3144 retval = ioctl_agpgart_release(softstate); 3145 break; 3146 case AGPIOC_SETUP: 3147 retval = ioctl_agpgart_setup(softstate, arg, flags); 3148 break; 3149 case AGPIOC_ALLOCATE: 3150 retval = ioctl_agpgart_alloc(softstate, arg, flags); 3151 break; 3152 case AGPIOC_DEALLOCATE: 3153 retval = ioctl_agpgart_dealloc(softstate, intarg); 3154 break; 3155 case AGPIOC_BIND: 3156 retval = ioctl_agpgart_bind(softstate, arg, flags); 3157 break; 3158 case AGPIOC_UNBIND: 3159 retval = ioctl_agpgart_unbind(softstate, arg, flags); 3160 break; 3161 default: 3162 AGPDB_PRINT2((CE_WARN, "agpgart_ioctl: wrong argument")); 3163 retval = ENXIO; 3164 break; 3165 } 3166 3167 mutex_exit(&softstate->asoft_instmutex); 3168 return (retval); 3169 } 3170 3171 static int 3172 agpgart_segmap(dev_t dev, off_t off, struct as *asp, 3173 caddr_t *addrp, off_t len, unsigned int prot, 3174 unsigned int maxprot, unsigned int flags, cred_t *credp) 3175 { 3176 3177 struct agpgart_softstate *softstate; 3178 int instance; 3179 int rc = 0; 3180 3181 instance = AGP_DEV2INST(dev); 3182 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 3183 if (softstate == NULL) { 3184 AGPDB_PRINT2((CE_WARN, "agpgart_segmap: get soft state err")); 3185 return (ENXIO); 3186 } 3187 if (!AGP_ALIGNED(len)) 3188 return (EINVAL); 3189 3190 mutex_enter(&softstate->asoft_instmutex); 3191 3192 /* 3193 * Process must have gart map privilege or gart access privilege 3194 * to map agp memory. 3195 */ 3196 if (secpolicy_gart_map(credp)) { 3197 mutex_exit(&softstate->asoft_instmutex); 3198 AGPDB_PRINT2((CE_WARN, "agpgart_segmap: permission denied")); 3199 return (EPERM); 3200 } 3201 3202 rc = devmap_setup(dev, (offset_t)off, asp, addrp, 3203 (size_t)len, prot, maxprot, flags, credp); 3204 3205 mutex_exit(&softstate->asoft_instmutex); 3206 return (rc); 3207 } 3208 3209 /*ARGSUSED*/ 3210 static int 3211 agpgart_devmap(dev_t dev, devmap_cookie_t cookie, offset_t offset, size_t len, 3212 size_t *mappedlen, uint_t model) 3213 { 3214 struct agpgart_softstate *softstate; 3215 int instance, status; 3216 struct keytable_ent *mementry; 3217 offset_t local_offset; 3218 3219 instance = AGP_DEV2INST(dev); 3220 softstate = ddi_get_soft_state(agpgart_glob_soft_handle, instance); 3221 if (softstate == NULL) { 3222 AGPDB_PRINT2((CE_WARN, "agpgart_devmap: get soft state err")); 3223 return (ENXIO); 3224 } 3225 3226 3227 if (offset > MB2BYTES(softstate->asoft_info.agpki_apersize)) { 3228 AGPDB_PRINT2((CE_WARN, "agpgart_devmap: offset is too large")); 3229 return (EINVAL); 3230 } 3231 3232 /* 3233 * Can not find any memory now, so fail. 3234 */ 3235 3236 mementry = agp_find_bound_keyent(softstate, AGP_BYTES2PAGES(offset)); 3237 3238 if (mementry == NULL) { 3239 AGPDB_PRINT2((CE_WARN, 3240 "agpgart_devmap: can not find the proper keyent")); 3241 return (EINVAL); 3242 } 3243 3244 local_offset = offset - AGP_PAGES2BYTES(mementry->kte_pgoff); 3245 3246 if (len > (AGP_PAGES2BYTES(mementry->kte_pages) - local_offset)) { 3247 len = AGP_PAGES2BYTES(mementry->kte_pages) - local_offset; 3248 } 3249 3250 switch (mementry->kte_type) { 3251 case AGP_NORMAL: 3252 status = devmap_pmem_setup(cookie, softstate->asoft_dip, 3253 &agp_devmap_cb, 3254 PMEMP(mementry->kte_memhdl)->pmem_cookie, local_offset, 3255 len, PROT_ALL, (DEVMAP_DEFAULTS|IOMEM_DATA_UC_WR_COMBINE), 3256 &mem_dev_acc_attr); 3257 break; 3258 default: 3259 AGPDB_PRINT2((CE_WARN, 3260 "agpgart_devmap: not a valid memory type")); 3261 return (EINVAL); 3262 } 3263 3264 3265 if (status == 0) { 3266 *mappedlen = len; 3267 } else { 3268 *mappedlen = 0; 3269 AGPDB_PRINT2((CE_WARN, 3270 "agpgart_devmap: devmap interface failed")); 3271 return (EINVAL); 3272 } 3273 3274 return (0); 3275 } 3276 3277 static struct cb_ops agpgart_cb_ops = { 3278 agpgart_open, /* open() */ 3279 agpgart_close, /* close() */ 3280 nodev, /* strategy() */ 3281 nodev, /* print routine */ 3282 nodev, /* no dump routine */ 3283 nodev, /* read() */ 3284 nodev, /* write() */ 3285 agpgart_ioctl, /* agpgart_ioctl */ 3286 agpgart_devmap, /* devmap routine */ 3287 nodev, /* no longer use mmap routine */ 3288 agpgart_segmap, /* system segmap routine */ 3289 nochpoll, /* no chpoll routine */ 3290 ddi_prop_op, /* system prop operations */ 3291 0, /* not a STREAMS driver */ 3292 D_DEVMAP | D_MP, /* safe for multi-thread/multi-processor */ 3293 CB_REV, /* cb_ops version? */ 3294 nodev, /* cb_aread() */ 3295 nodev, /* cb_awrite() */ 3296 }; 3297 3298 static struct dev_ops agpgart_ops = { 3299 DEVO_REV, /* devo_rev */ 3300 0, /* devo_refcnt */ 3301 agpgart_getinfo, /* devo_getinfo */ 3302 nulldev, /* devo_identify */ 3303 nulldev, /* devo_probe */ 3304 agpgart_attach, /* devo_attach */ 3305 agpgart_detach, /* devo_detach */ 3306 nodev, /* devo_reset */ 3307 &agpgart_cb_ops, /* devo_cb_ops */ 3308 (struct bus_ops *)0, /* devo_bus_ops */ 3309 NULL, /* devo_power */ 3310 }; 3311 3312 static struct modldrv modldrv = { 3313 &mod_driverops, 3314 "AGP driver v1.9", 3315 &agpgart_ops, 3316 }; 3317 3318 static struct modlinkage modlinkage = { 3319 MODREV_1, /* MODREV_1 is indicated by manual */ 3320 {&modldrv, NULL, NULL, NULL} 3321 }; 3322 3323 static void *agpgart_glob_soft_handle; 3324 3325 int 3326 _init(void) 3327 { 3328 int ret = DDI_SUCCESS; 3329 3330 ret = ddi_soft_state_init(&agpgart_glob_soft_handle, 3331 sizeof (agpgart_softstate_t), 3332 AGPGART_MAX_INSTANCES); 3333 3334 if (ret != 0) { 3335 AGPDB_PRINT2((CE_WARN, 3336 "_init: soft state init error code=0x%x", ret)); 3337 return (ret); 3338 } 3339 3340 if ((ret = mod_install(&modlinkage)) != 0) { 3341 AGPDB_PRINT2((CE_WARN, 3342 "_init: mod install error code=0x%x", ret)); 3343 ddi_soft_state_fini(&agpgart_glob_soft_handle); 3344 return (ret); 3345 } 3346 3347 return (DDI_SUCCESS); 3348 } 3349 3350 int 3351 _info(struct modinfo *modinfop) 3352 { 3353 return (mod_info(&modlinkage, modinfop)); 3354 } 3355 3356 int 3357 _fini(void) 3358 { 3359 int ret; 3360 3361 if ((ret = mod_remove(&modlinkage)) == 0) { 3362 ddi_soft_state_fini(&agpgart_glob_soft_handle); 3363 } 3364 3365 return (ret); 3366 } 3367