1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Portions Copyright (c) 2010, Oracle and/or its affiliates. 23 * All rights reserved. 24 */ 25 26 /* 27 * immu_regs.c - File that operates on a IMMU unit's regsiters 28 */ 29 #include <sys/dditypes.h> 30 #include <sys/ddi.h> 31 #include <sys/archsystm.h> 32 #include <sys/x86_archext.h> 33 #include <sys/spl.h> 34 #include <sys/sysmacros.h> 35 #include <sys/immu.h> 36 #include <sys/cpu.h> 37 38 #define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \ 39 (uint32_t *)(immu->immu_regs_addr + (offset))) 40 #define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \ 41 (uint64_t *)(immu->immu_regs_addr + (offset))) 42 #define put_reg32(immu, offset, val) ddi_put32\ 43 ((immu)->immu_regs_handle, \ 44 (uint32_t *)(immu->immu_regs_addr + (offset)), val) 45 #define put_reg64(immu, offset, val) ddi_put64\ 46 ((immu)->immu_regs_handle, \ 47 (uint64_t *)(immu->immu_regs_addr + (offset)), val) 48 49 static void immu_regs_inv_wait(immu_inv_wait_t *iwp); 50 51 struct immu_flushops immu_regs_flushops = { 52 immu_regs_context_fsi, 53 immu_regs_context_dsi, 54 immu_regs_context_gbl, 55 immu_regs_iotlb_psi, 56 immu_regs_iotlb_dsi, 57 immu_regs_iotlb_gbl, 58 immu_regs_inv_wait 59 }; 60 61 /* 62 * wait max 60s for the hardware completion 63 */ 64 #define IMMU_MAX_WAIT_TIME 60000000 65 #define wait_completion(immu, offset, getf, completion, status) \ 66 { \ 67 clock_t stick = ddi_get_lbolt(); \ 68 clock_t ntick; \ 69 _NOTE(CONSTCOND) \ 70 while (1) { \ 71 status = getf(immu, offset); \ 72 ntick = ddi_get_lbolt(); \ 73 if (completion) { \ 74 break; \ 75 } \ 76 if (ntick - stick >= drv_usectohz(IMMU_MAX_WAIT_TIME)) { \ 77 ddi_err(DER_PANIC, NULL, \ 78 "immu wait completion time out"); \ 79 /*NOTREACHED*/ \ 80 } else { \ 81 ht_pause();\ 82 }\ 83 }\ 84 } 85 86 static ddi_device_acc_attr_t immu_regs_attr = { 87 DDI_DEVICE_ATTR_V0, 88 DDI_NEVERSWAP_ACC, 89 DDI_STRICTORDER_ACC, 90 }; 91 92 /* 93 * iotlb_flush() 94 * flush the iotlb cache 95 */ 96 static void 97 iotlb_flush(immu_t *immu, uint_t domain_id, 98 uint64_t addr, uint_t am, uint_t hint, immu_iotlb_inv_t type) 99 { 100 uint64_t command = 0, iva = 0; 101 uint_t iva_offset, iotlb_offset; 102 uint64_t status = 0; 103 104 /* no lock needed since cap and excap fields are RDONLY */ 105 iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap); 106 iotlb_offset = iva_offset + 8; 107 108 /* 109 * prepare drain read/write command 110 */ 111 if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) { 112 command |= TLB_INV_DRAIN_WRITE; 113 } 114 115 if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) { 116 command |= TLB_INV_DRAIN_READ; 117 } 118 119 /* 120 * if the hardward doesn't support page selective invalidation, we 121 * will use domain type. Otherwise, use global type 122 */ 123 switch (type) { 124 case IOTLB_PSI: 125 command |= TLB_INV_PAGE | TLB_INV_IVT | 126 TLB_INV_DID(domain_id); 127 iva = addr | am | TLB_IVA_HINT(hint); 128 break; 129 case IOTLB_DSI: 130 command |= TLB_INV_DOMAIN | TLB_INV_IVT | 131 TLB_INV_DID(domain_id); 132 break; 133 case IOTLB_GLOBAL: 134 command |= TLB_INV_GLOBAL | TLB_INV_IVT; 135 break; 136 default: 137 ddi_err(DER_MODE, NULL, "%s: incorrect iotlb flush type", 138 immu->immu_name); 139 return; 140 } 141 142 if (iva) 143 put_reg64(immu, iva_offset, iva); 144 put_reg64(immu, iotlb_offset, command); 145 wait_completion(immu, iotlb_offset, get_reg64, 146 (!(status & TLB_INV_IVT)), status); 147 } 148 149 /* 150 * immu_regs_iotlb_psi() 151 * iotlb page specific invalidation 152 */ 153 /*ARGSUSED*/ 154 void 155 immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages, 156 uint_t hint, immu_inv_wait_t *iwp) 157 { 158 int dvma_am; 159 int npg_am; 160 int max_am; 161 int am; 162 uint64_t align; 163 int npages_left; 164 int npages; 165 int i; 166 167 if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) { 168 immu_regs_iotlb_dsi(immu, did, iwp); 169 return; 170 } 171 172 max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap); 173 174 mutex_enter(&(immu->immu_regs_lock)); 175 176 npages_left = snpages; 177 for (i = 0; i < immu_flush_gran && npages_left > 0; i++) { 178 /* First calculate alignment of DVMA */ 179 180 if (dvma == 0) { 181 dvma_am = max_am; 182 } else { 183 for (align = (1 << 12), dvma_am = 1; 184 (dvma & align) == 0; align <<= 1, dvma_am++) 185 ; 186 dvma_am--; 187 } 188 189 /* Calculate the npg_am */ 190 npages = npages_left; 191 for (npg_am = 0, npages >>= 1; npages; npages >>= 1, npg_am++) 192 ; 193 194 am = MIN(max_am, MIN(dvma_am, npg_am)); 195 196 iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI); 197 198 npages = (1 << am); 199 npages_left -= npages; 200 dvma += (npages * IMMU_PAGESIZE); 201 } 202 203 if (npages_left) { 204 iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI); 205 } 206 mutex_exit(&(immu->immu_regs_lock)); 207 } 208 209 /* 210 * immu_regs_iotlb_dsi() 211 * domain specific invalidation 212 */ 213 /*ARGSUSED*/ 214 void 215 immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp) 216 { 217 mutex_enter(&(immu->immu_regs_lock)); 218 iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI); 219 mutex_exit(&(immu->immu_regs_lock)); 220 } 221 222 /* 223 * immu_regs_iotlb_gbl() 224 * global iotlb invalidation 225 */ 226 /*ARGSUSED*/ 227 void 228 immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp) 229 { 230 mutex_enter(&(immu->immu_regs_lock)); 231 iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL); 232 mutex_exit(&(immu->immu_regs_lock)); 233 } 234 235 236 static int 237 gaw2agaw(int gaw) 238 { 239 int r, agaw; 240 241 r = (gaw - 12) % 9; 242 243 if (r == 0) 244 agaw = gaw; 245 else 246 agaw = gaw + 9 - r; 247 248 if (agaw > 64) 249 agaw = 64; 250 251 return (agaw); 252 } 253 254 /* 255 * set_immu_agaw() 256 * calculate agaw for a IOMMU unit 257 */ 258 static int 259 set_agaw(immu_t *immu) 260 { 261 int mgaw, magaw, agaw; 262 uint_t bitpos; 263 int max_sagaw_mask, sagaw_mask, mask; 264 int nlevels; 265 266 /* 267 * mgaw is the maximum guest address width. 268 * Addresses above this value will be 269 * blocked by the IOMMU unit. 270 * sagaw is a bitmask that lists all the 271 * AGAWs supported by this IOMMU unit. 272 */ 273 mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap); 274 sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap); 275 276 magaw = gaw2agaw(mgaw); 277 278 /* 279 * Get bitpos corresponding to 280 * magaw 281 */ 282 283 /* 284 * Maximum SAGAW is specified by 285 * Vt-d spec. 286 */ 287 max_sagaw_mask = ((1 << 5) - 1); 288 289 if (sagaw_mask > max_sagaw_mask) { 290 ddi_err(DER_WARN, NULL, "%s: SAGAW bitmask (%x) " 291 "is larger than maximu SAGAW bitmask " 292 "(%x) specified by Intel Vt-d spec", 293 immu->immu_name, sagaw_mask, max_sagaw_mask); 294 return (DDI_FAILURE); 295 } 296 297 /* 298 * Find a supported AGAW <= magaw 299 * 300 * sagaw_mask bitpos AGAW (bits) nlevels 301 * ============================================== 302 * 0 0 0 0 1 0 30 2 303 * 0 0 0 1 0 1 39 3 304 * 0 0 1 0 0 2 48 4 305 * 0 1 0 0 0 3 57 5 306 * 1 0 0 0 0 4 64(66) 6 307 */ 308 mask = 1; 309 nlevels = 0; 310 agaw = 0; 311 for (mask = 1, bitpos = 0; bitpos < 5; 312 bitpos++, mask <<= 1) { 313 if (mask & sagaw_mask) { 314 nlevels = bitpos + 2; 315 agaw = 30 + (bitpos * 9); 316 } 317 } 318 319 /* calculated agaw can be > 64 */ 320 agaw = (agaw > 64) ? 64 : agaw; 321 322 if (agaw < 30 || agaw > magaw) { 323 ddi_err(DER_WARN, NULL, "%s: Calculated AGAW (%d) " 324 "is outside valid limits [30,%d] specified by Vt-d spec " 325 "and magaw", immu->immu_name, agaw, magaw); 326 return (DDI_FAILURE); 327 } 328 329 if (nlevels < 2 || nlevels > 6) { 330 ddi_err(DER_WARN, NULL, "%s: Calculated pagetable " 331 "level (%d) is outside valid limits [2,6]", 332 immu->immu_name, nlevels); 333 return (DDI_FAILURE); 334 } 335 336 ddi_err(DER_LOG, NULL, "Calculated pagetable " 337 "level (%d), agaw = %d", nlevels, agaw); 338 339 immu->immu_dvma_nlevels = nlevels; 340 immu->immu_dvma_agaw = agaw; 341 342 return (DDI_SUCCESS); 343 } 344 345 static int 346 setup_regs(immu_t *immu) 347 { 348 int error; 349 350 /* 351 * This lock may be acquired by the IOMMU interrupt handler 352 */ 353 mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER, 354 (void *)ipltospl(IMMU_INTR_IPL)); 355 356 /* 357 * map the register address space 358 */ 359 error = ddi_regs_map_setup(immu->immu_dip, 0, 360 (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 361 (offset_t)IMMU_REGSZ, &immu_regs_attr, 362 &(immu->immu_regs_handle)); 363 364 if (error == DDI_FAILURE) { 365 ddi_err(DER_WARN, NULL, "%s: Intel IOMMU register map failed", 366 immu->immu_name); 367 mutex_destroy(&(immu->immu_regs_lock)); 368 return (DDI_FAILURE); 369 } 370 371 /* 372 * get the register value 373 */ 374 immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP); 375 immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP); 376 377 /* 378 * if the hardware access is non-coherent, we need clflush 379 */ 380 if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) { 381 immu->immu_dvma_coherent = B_TRUE; 382 } else { 383 immu->immu_dvma_coherent = B_FALSE; 384 if (!is_x86_feature(x86_featureset, X86FSET_CLFSH)) { 385 ddi_err(DER_WARN, NULL, 386 "immu unit %s can't be enabled due to " 387 "missing clflush functionality", immu->immu_name); 388 ddi_regs_map_free(&(immu->immu_regs_handle)); 389 mutex_destroy(&(immu->immu_regs_lock)); 390 return (DDI_FAILURE); 391 } 392 } 393 394 /* Setup SNP and TM reserved fields */ 395 immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu); 396 immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu); 397 398 if (IMMU_ECAP_GET_CH(immu->immu_regs_excap) && immu_use_tm) 399 immu->immu_ptemask = PDTE_MASK_TM; 400 else 401 immu->immu_ptemask = 0; 402 403 /* 404 * Check for Mobile 4 series chipset 405 */ 406 if (immu_quirk_mobile4 == B_TRUE && 407 !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 408 ddi_err(DER_LOG, NULL, 409 "IMMU: Mobile 4 chipset quirk detected. " 410 "Force-setting RWBF"); 411 IMMU_CAP_SET_RWBF(immu->immu_regs_cap); 412 } 413 414 /* 415 * retrieve the maximum number of domains 416 */ 417 immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap); 418 419 /* 420 * calculate the agaw 421 */ 422 if (set_agaw(immu) != DDI_SUCCESS) { 423 ddi_regs_map_free(&(immu->immu_regs_handle)); 424 mutex_destroy(&(immu->immu_regs_lock)); 425 return (DDI_FAILURE); 426 } 427 immu->immu_regs_cmdval = 0; 428 429 immu->immu_flushops = &immu_regs_flushops; 430 431 return (DDI_SUCCESS); 432 } 433 434 /* ############### Functions exported ################## */ 435 436 /* 437 * immu_regs_setup() 438 * Setup mappings to a IMMU unit's registers 439 * so that they can be read/written 440 */ 441 void 442 immu_regs_setup(list_t *listp) 443 { 444 int i; 445 immu_t *immu; 446 447 for (i = 0; i < IMMU_MAXSEG; i++) { 448 immu = list_head(listp); 449 for (; immu; immu = list_next(listp, immu)) { 450 /* do your best, continue on error */ 451 if (setup_regs(immu) != DDI_SUCCESS) { 452 immu->immu_regs_setup = B_FALSE; 453 } else { 454 immu->immu_regs_setup = B_TRUE; 455 } 456 } 457 } 458 } 459 460 /* 461 * immu_regs_map() 462 */ 463 int 464 immu_regs_resume(immu_t *immu) 465 { 466 int error; 467 468 /* 469 * remap the register address space 470 */ 471 error = ddi_regs_map_setup(immu->immu_dip, 0, 472 (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 473 (offset_t)IMMU_REGSZ, &immu_regs_attr, 474 &(immu->immu_regs_handle)); 475 if (error != DDI_SUCCESS) { 476 return (DDI_FAILURE); 477 } 478 479 immu_regs_set_root_table(immu); 480 481 immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr, 482 immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr); 483 484 (void) immu_intr_handler((caddr_t)immu, NULL); 485 486 immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg); 487 488 immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value); 489 490 491 return (error); 492 } 493 494 /* 495 * immu_regs_suspend() 496 */ 497 void 498 immu_regs_suspend(immu_t *immu) 499 { 500 501 immu->immu_intrmap_running = B_FALSE; 502 503 /* Finally, unmap the regs */ 504 ddi_regs_map_free(&(immu->immu_regs_handle)); 505 } 506 507 /* 508 * immu_regs_startup() 509 * set a IMMU unit's registers to startup the unit 510 */ 511 void 512 immu_regs_startup(immu_t *immu) 513 { 514 uint32_t status; 515 516 if (immu->immu_regs_setup == B_FALSE) { 517 return; 518 } 519 520 mutex_enter(&(immu->immu_regs_lock)); 521 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 522 immu->immu_regs_cmdval | IMMU_GCMD_TE); 523 wait_completion(immu, IMMU_REG_GLOBAL_STS, 524 get_reg32, (status & IMMU_GSTS_TES), status); 525 immu->immu_regs_cmdval |= IMMU_GCMD_TE; 526 immu->immu_regs_running = B_TRUE; 527 mutex_exit(&(immu->immu_regs_lock)); 528 529 ddi_err(DER_NOTE, NULL, "%s running", immu->immu_name); 530 } 531 532 /* 533 * immu_regs_shutdown() 534 * shutdown a unit 535 */ 536 void 537 immu_regs_shutdown(immu_t *immu) 538 { 539 uint32_t status; 540 541 if (immu->immu_regs_running == B_FALSE) { 542 return; 543 } 544 545 mutex_enter(&(immu->immu_regs_lock)); 546 immu->immu_regs_cmdval &= ~IMMU_GCMD_TE; 547 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 548 immu->immu_regs_cmdval); 549 wait_completion(immu, IMMU_REG_GLOBAL_STS, 550 get_reg32, !(status & IMMU_GSTS_TES), status); 551 immu->immu_regs_running = B_FALSE; 552 mutex_exit(&(immu->immu_regs_lock)); 553 554 ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name); 555 } 556 557 /* 558 * immu_regs_intr() 559 * Set a IMMU unit regs to setup a IMMU unit's 560 * interrupt handler 561 */ 562 void 563 immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 564 uint32_t uaddr) 565 { 566 mutex_enter(&(immu->immu_regs_lock)); 567 immu->immu_regs_intr_msi_addr = msi_addr; 568 immu->immu_regs_intr_uaddr = uaddr; 569 immu->immu_regs_intr_msi_data = msi_data; 570 put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr); 571 put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr); 572 put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data); 573 put_reg32(immu, IMMU_REG_FEVNT_CON, 0); 574 mutex_exit(&(immu->immu_regs_lock)); 575 } 576 577 /* 578 * immu_regs_passthru_supported() 579 * Returns B_TRUE ifi passthru is supported 580 */ 581 boolean_t 582 immu_regs_passthru_supported(immu_t *immu) 583 { 584 if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) { 585 return (B_TRUE); 586 } 587 588 ddi_err(DER_WARN, NULL, "Passthru not supported"); 589 return (B_FALSE); 590 } 591 592 /* 593 * immu_regs_is_TM_reserved() 594 * Returns B_TRUE if TM field is reserved 595 */ 596 boolean_t 597 immu_regs_is_TM_reserved(immu_t *immu) 598 { 599 if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) || 600 IMMU_ECAP_GET_CH(immu->immu_regs_excap)) { 601 return (B_FALSE); 602 } 603 return (B_TRUE); 604 } 605 606 /* 607 * immu_regs_is_SNP_reserved() 608 * Returns B_TRUE if SNP field is reserved 609 */ 610 boolean_t 611 immu_regs_is_SNP_reserved(immu_t *immu) 612 { 613 614 return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE); 615 } 616 617 /* 618 * immu_regs_wbf_flush() 619 * If required and supported, write to IMMU 620 * unit's regs to flush DMA write buffer(s) 621 */ 622 void 623 immu_regs_wbf_flush(immu_t *immu) 624 { 625 uint32_t status; 626 627 if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 628 return; 629 } 630 631 mutex_enter(&(immu->immu_regs_lock)); 632 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 633 immu->immu_regs_cmdval | IMMU_GCMD_WBF); 634 wait_completion(immu, IMMU_REG_GLOBAL_STS, 635 get_reg32, (!(status & IMMU_GSTS_WBFS)), status); 636 mutex_exit(&(immu->immu_regs_lock)); 637 } 638 639 /* 640 * immu_regs_cpu_flush() 641 * flush the cpu cache line after CPU memory writes, so 642 * IOMMU can see the writes 643 */ 644 void 645 immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size) 646 { 647 uintptr_t startline, endline; 648 649 if (immu->immu_dvma_coherent == B_TRUE) 650 return; 651 652 startline = (uintptr_t)addr & ~(uintptr_t)(x86_clflush_size - 1); 653 endline = ((uintptr_t)addr + size - 1) & 654 ~(uintptr_t)(x86_clflush_size - 1); 655 while (startline <= endline) { 656 clflush_insn((caddr_t)startline); 657 startline += x86_clflush_size; 658 } 659 660 mfence_insn(); 661 } 662 663 /* 664 * immu_regs_context_flush() 665 * flush the context cache 666 */ 667 static void 668 context_flush(immu_t *immu, uint8_t function_mask, 669 uint16_t sid, uint_t did, immu_context_inv_t type) 670 { 671 uint64_t command = 0; 672 uint64_t status; 673 674 /* 675 * define the command 676 */ 677 switch (type) { 678 case CONTEXT_FSI: 679 command |= CCMD_INV_ICC | CCMD_INV_DEVICE 680 | CCMD_INV_DID(did) 681 | CCMD_INV_SID(sid) | CCMD_INV_FM(function_mask); 682 break; 683 case CONTEXT_DSI: 684 command |= CCMD_INV_ICC | CCMD_INV_DOMAIN 685 | CCMD_INV_DID(did); 686 break; 687 case CONTEXT_GLOBAL: 688 command |= CCMD_INV_ICC | CCMD_INV_GLOBAL; 689 break; 690 default: 691 ddi_err(DER_PANIC, NULL, 692 "%s: incorrect context cache flush type", 693 immu->immu_name); 694 /*NOTREACHED*/ 695 } 696 697 mutex_enter(&(immu->immu_regs_lock)); 698 put_reg64(immu, IMMU_REG_CONTEXT_CMD, command); 699 wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64, 700 (!(status & CCMD_INV_ICC)), status); 701 mutex_exit(&(immu->immu_regs_lock)); 702 } 703 704 /*ARGSUSED*/ 705 void 706 immu_regs_context_fsi(immu_t *immu, uint8_t function_mask, 707 uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp) 708 { 709 context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI); 710 } 711 712 /*ARGSUSED*/ 713 void 714 immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp) 715 { 716 context_flush(immu, 0, 0, domain_id, CONTEXT_DSI); 717 } 718 719 /*ARGSUSED*/ 720 void 721 immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp) 722 { 723 context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL); 724 } 725 726 /* 727 * Nothing to do, all register operations are synchronous. 728 */ 729 /*ARGSUSED*/ 730 static void 731 immu_regs_inv_wait(immu_inv_wait_t *iwp) 732 { 733 } 734 735 void 736 immu_regs_set_root_table(immu_t *immu) 737 { 738 uint32_t status; 739 740 mutex_enter(&(immu->immu_regs_lock)); 741 put_reg64(immu, IMMU_REG_ROOTENTRY, 742 immu->immu_ctx_root->hwpg_paddr); 743 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 744 immu->immu_regs_cmdval | IMMU_GCMD_SRTP); 745 wait_completion(immu, IMMU_REG_GLOBAL_STS, 746 get_reg32, (status & IMMU_GSTS_RTPS), status); 747 mutex_exit(&(immu->immu_regs_lock)); 748 } 749 750 751 /* enable queued invalidation interface */ 752 void 753 immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value) 754 { 755 uint32_t status; 756 757 if (immu_qinv_enable == B_FALSE) 758 return; 759 760 mutex_enter(&immu->immu_regs_lock); 761 immu->immu_qinv_reg_value = qinv_reg_value; 762 /* Initialize the Invalidation Queue Tail register to zero */ 763 put_reg64(immu, IMMU_REG_INVAL_QT, 0); 764 765 /* set invalidation queue base address register */ 766 put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value); 767 768 /* enable queued invalidation interface */ 769 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 770 immu->immu_regs_cmdval | IMMU_GCMD_QIE); 771 wait_completion(immu, IMMU_REG_GLOBAL_STS, 772 get_reg32, (status & IMMU_GSTS_QIES), status); 773 mutex_exit(&immu->immu_regs_lock); 774 775 immu->immu_regs_cmdval |= IMMU_GCMD_QIE; 776 immu->immu_qinv_running = B_TRUE; 777 778 } 779 780 /* enable interrupt remapping hardware unit */ 781 void 782 immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg) 783 { 784 uint32_t status; 785 786 if (immu_intrmap_enable == B_FALSE) 787 return; 788 789 /* set interrupt remap table pointer */ 790 mutex_enter(&(immu->immu_regs_lock)); 791 immu->immu_intrmap_irta_reg = irta_reg; 792 put_reg64(immu, IMMU_REG_IRTAR, irta_reg); 793 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 794 immu->immu_regs_cmdval | IMMU_GCMD_SIRTP); 795 wait_completion(immu, IMMU_REG_GLOBAL_STS, 796 get_reg32, (status & IMMU_GSTS_IRTPS), status); 797 mutex_exit(&(immu->immu_regs_lock)); 798 799 /* global flush intr entry cache */ 800 immu_qinv_intr_global(immu, &immu->immu_intrmap_inv_wait); 801 802 /* enable interrupt remapping */ 803 mutex_enter(&(immu->immu_regs_lock)); 804 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 805 immu->immu_regs_cmdval | IMMU_GCMD_IRE); 806 wait_completion(immu, IMMU_REG_GLOBAL_STS, 807 get_reg32, (status & IMMU_GSTS_IRES), 808 status); 809 immu->immu_regs_cmdval |= IMMU_GCMD_IRE; 810 811 /* set compatible mode */ 812 put_reg32(immu, IMMU_REG_GLOBAL_CMD, 813 immu->immu_regs_cmdval | IMMU_GCMD_CFI); 814 wait_completion(immu, IMMU_REG_GLOBAL_STS, 815 get_reg32, (status & IMMU_GSTS_CFIS), 816 status); 817 immu->immu_regs_cmdval |= IMMU_GCMD_CFI; 818 mutex_exit(&(immu->immu_regs_lock)); 819 820 immu->immu_intrmap_running = B_TRUE; 821 } 822 823 uint64_t 824 immu_regs_get64(immu_t *immu, uint_t reg) 825 { 826 return (get_reg64(immu, reg)); 827 } 828 829 uint32_t 830 immu_regs_get32(immu_t *immu, uint_t reg) 831 { 832 return (get_reg32(immu, reg)); 833 } 834 835 void 836 immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val) 837 { 838 put_reg64(immu, reg, val); 839 } 840 841 void 842 immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val) 843 { 844 put_reg32(immu, reg, val); 845 } 846