13a634bfcSVikram Hegde /* 23a634bfcSVikram Hegde * CDDL HEADER START 33a634bfcSVikram Hegde * 43a634bfcSVikram Hegde * The contents of this file are subject to the terms of the 53a634bfcSVikram Hegde * Common Development and Distribution License (the "License"). 63a634bfcSVikram Hegde * You may not use this file except in compliance with the License. 73a634bfcSVikram Hegde * 83a634bfcSVikram Hegde * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93a634bfcSVikram Hegde * or http://www.opensolaris.org/os/licensing. 103a634bfcSVikram Hegde * See the License for the specific language governing permissions 113a634bfcSVikram Hegde * and limitations under the License. 123a634bfcSVikram Hegde * 133a634bfcSVikram Hegde * When distributing Covered Code, include this CDDL HEADER in each 143a634bfcSVikram Hegde * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153a634bfcSVikram Hegde * If applicable, add the following below this CDDL HEADER, with the 163a634bfcSVikram Hegde * fields enclosed by brackets "[]" replaced with your own identifying 173a634bfcSVikram Hegde * information: Portions Copyright [yyyy] [name of copyright owner] 183a634bfcSVikram Hegde * 193a634bfcSVikram Hegde * CDDL HEADER END 203a634bfcSVikram Hegde */ 213a634bfcSVikram Hegde /* 223a634bfcSVikram Hegde * Portions Copyright 2010 Sun Microsystems, Inc. All rights reserved. 233a634bfcSVikram Hegde * Use is subject to license terms. 243a634bfcSVikram Hegde */ 253a634bfcSVikram Hegde 263a634bfcSVikram Hegde /* 273a634bfcSVikram Hegde * immu_regs.c - File that operates on a IMMU unit's regsiters 283a634bfcSVikram Hegde */ 293a634bfcSVikram Hegde #include <sys/dditypes.h> 303a634bfcSVikram Hegde #include <sys/ddi.h> 313a634bfcSVikram Hegde #include <sys/archsystm.h> 323a634bfcSVikram Hegde #include <sys/x86_archext.h> 333a634bfcSVikram Hegde #include <sys/spl.h> 34*e03dceedSVikram Hegde #include <sys/sysmacros.h> 353a634bfcSVikram Hegde #include <sys/immu.h> 363a634bfcSVikram Hegde 373a634bfcSVikram Hegde #define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \ 383a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset))) 393a634bfcSVikram Hegde #define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \ 403a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset))) 413a634bfcSVikram Hegde #define put_reg32(immu, offset, val) ddi_put32\ 423a634bfcSVikram Hegde ((immu)->immu_regs_handle, \ 433a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset)), val) 443a634bfcSVikram Hegde #define put_reg64(immu, offset, val) ddi_put64\ 453a634bfcSVikram Hegde ((immu)->immu_regs_handle, \ 463a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset)), val) 473a634bfcSVikram Hegde 483a634bfcSVikram Hegde /* 493a634bfcSVikram Hegde * wait max 60s for the hardware completion 503a634bfcSVikram Hegde */ 513a634bfcSVikram Hegde #define IMMU_MAX_WAIT_TIME 60000000 523a634bfcSVikram Hegde #define wait_completion(immu, offset, getf, completion, status) \ 533a634bfcSVikram Hegde { \ 543a634bfcSVikram Hegde clock_t stick = ddi_get_lbolt(); \ 553a634bfcSVikram Hegde clock_t ntick; \ 563a634bfcSVikram Hegde _NOTE(CONSTCOND) \ 573a634bfcSVikram Hegde while (1) { \ 583a634bfcSVikram Hegde status = getf(immu, offset); \ 593a634bfcSVikram Hegde ntick = ddi_get_lbolt(); \ 603a634bfcSVikram Hegde if (completion) { \ 613a634bfcSVikram Hegde break; \ 623a634bfcSVikram Hegde } \ 633a634bfcSVikram Hegde if (ntick - stick >= drv_usectohz(IMMU_MAX_WAIT_TIME)) { \ 643a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, \ 653a634bfcSVikram Hegde "immu wait completion time out"); \ 663a634bfcSVikram Hegde /*NOTREACHED*/ \ 673a634bfcSVikram Hegde } else { \ 683a634bfcSVikram Hegde iommu_cpu_nop();\ 693a634bfcSVikram Hegde }\ 703a634bfcSVikram Hegde }\ 713a634bfcSVikram Hegde } 723a634bfcSVikram Hegde 733a634bfcSVikram Hegde static ddi_device_acc_attr_t immu_regs_attr = { 743a634bfcSVikram Hegde DDI_DEVICE_ATTR_V0, 753a634bfcSVikram Hegde DDI_NEVERSWAP_ACC, 763a634bfcSVikram Hegde DDI_STRICTORDER_ACC, 773a634bfcSVikram Hegde }; 783a634bfcSVikram Hegde 793a634bfcSVikram Hegde /* 803a634bfcSVikram Hegde * iotlb_flush() 813a634bfcSVikram Hegde * flush the iotlb cache 823a634bfcSVikram Hegde */ 833a634bfcSVikram Hegde static void 843a634bfcSVikram Hegde iotlb_flush(immu_t *immu, uint_t domain_id, 853a634bfcSVikram Hegde uint64_t addr, uint_t am, uint_t hint, immu_iotlb_inv_t type) 863a634bfcSVikram Hegde { 873a634bfcSVikram Hegde uint64_t command = 0, iva = 0; 883a634bfcSVikram Hegde uint_t iva_offset, iotlb_offset; 893a634bfcSVikram Hegde uint64_t status = 0; 903a634bfcSVikram Hegde 913a634bfcSVikram Hegde /* no lock needed since cap and excap fields are RDONLY */ 923a634bfcSVikram Hegde iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap); 933a634bfcSVikram Hegde iotlb_offset = iva_offset + 8; 943a634bfcSVikram Hegde 953a634bfcSVikram Hegde /* 963a634bfcSVikram Hegde * prepare drain read/write command 973a634bfcSVikram Hegde */ 983a634bfcSVikram Hegde if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) { 993a634bfcSVikram Hegde command |= TLB_INV_DRAIN_WRITE; 1003a634bfcSVikram Hegde } 1013a634bfcSVikram Hegde 1023a634bfcSVikram Hegde if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) { 1033a634bfcSVikram Hegde command |= TLB_INV_DRAIN_READ; 1043a634bfcSVikram Hegde } 1053a634bfcSVikram Hegde 1063a634bfcSVikram Hegde /* 1073a634bfcSVikram Hegde * if the hardward doesn't support page selective invalidation, we 1083a634bfcSVikram Hegde * will use domain type. Otherwise, use global type 1093a634bfcSVikram Hegde */ 1103a634bfcSVikram Hegde switch (type) { 1113a634bfcSVikram Hegde case IOTLB_PSI: 112*e03dceedSVikram Hegde ASSERT(IMMU_CAP_GET_PSI(immu->immu_regs_cap)); 113*e03dceedSVikram Hegde ASSERT(am <= IMMU_CAP_GET_MAMV(immu->immu_regs_cap)); 114*e03dceedSVikram Hegde ASSERT(!(addr & IMMU_PAGEOFFSET)); 1153a634bfcSVikram Hegde command |= TLB_INV_PAGE | TLB_INV_IVT | 1163a634bfcSVikram Hegde TLB_INV_DID(domain_id); 1173a634bfcSVikram Hegde iva = addr | am | TLB_IVA_HINT(hint); 1183a634bfcSVikram Hegde break; 1193a634bfcSVikram Hegde case IOTLB_DSI: 1203a634bfcSVikram Hegde command |= TLB_INV_DOMAIN | TLB_INV_IVT | 1213a634bfcSVikram Hegde TLB_INV_DID(domain_id); 1223a634bfcSVikram Hegde break; 1233a634bfcSVikram Hegde case IOTLB_GLOBAL: 1243a634bfcSVikram Hegde command |= TLB_INV_GLOBAL | TLB_INV_IVT; 1253a634bfcSVikram Hegde break; 1263a634bfcSVikram Hegde default: 1273a634bfcSVikram Hegde ddi_err(DER_MODE, NULL, "%s: incorrect iotlb flush type", 1283a634bfcSVikram Hegde immu->immu_name); 1293a634bfcSVikram Hegde return; 1303a634bfcSVikram Hegde } 1313a634bfcSVikram Hegde 132*e03dceedSVikram Hegde ASSERT(!(status & TLB_INV_IVT)); 1333a634bfcSVikram Hegde if (iva) 1343a634bfcSVikram Hegde put_reg64(immu, iva_offset, iva); 1353a634bfcSVikram Hegde put_reg64(immu, iotlb_offset, command); 1363a634bfcSVikram Hegde wait_completion(immu, iotlb_offset, get_reg64, 1373a634bfcSVikram Hegde (!(status & TLB_INV_IVT)), status); 1383a634bfcSVikram Hegde } 1393a634bfcSVikram Hegde 1403a634bfcSVikram Hegde /* 1413a634bfcSVikram Hegde * iotlb_psi() 1423a634bfcSVikram Hegde * iotlb page specific invalidation 1433a634bfcSVikram Hegde */ 1443a634bfcSVikram Hegde static void 145*e03dceedSVikram Hegde iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages, 146*e03dceedSVikram Hegde uint_t hint) 1473a634bfcSVikram Hegde { 148*e03dceedSVikram Hegde int dvma_am; 149*e03dceedSVikram Hegde int npg_am; 150*e03dceedSVikram Hegde int max_am; 151*e03dceedSVikram Hegde int am; 152*e03dceedSVikram Hegde uint64_t align; 153*e03dceedSVikram Hegde int npages_left; 154*e03dceedSVikram Hegde int npages; 155*e03dceedSVikram Hegde int i; 156*e03dceedSVikram Hegde 157*e03dceedSVikram Hegde ASSERT(IMMU_CAP_GET_PSI(immu->immu_regs_cap)); 158*e03dceedSVikram Hegde ASSERT(dvma % IMMU_PAGESIZE == 0); 159*e03dceedSVikram Hegde 160*e03dceedSVikram Hegde max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap); 1613a634bfcSVikram Hegde 1623a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 1633a634bfcSVikram Hegde 164*e03dceedSVikram Hegde npages_left = snpages; 165*e03dceedSVikram Hegde for (i = 0; i < immu_flush_gran && npages_left > 0; i++) { 1663a634bfcSVikram Hegde /* First calculate alignment of DVMA */ 167*e03dceedSVikram Hegde 168*e03dceedSVikram Hegde if (dvma == 0) { 169*e03dceedSVikram Hegde dvma_am = max_am; 1703a634bfcSVikram Hegde } else { 171*e03dceedSVikram Hegde for (align = (1 << 12), dvma_am = 1; 172*e03dceedSVikram Hegde (dvma & align) == 0; align <<= 1, dvma_am++) 1733a634bfcSVikram Hegde ; 174*e03dceedSVikram Hegde dvma_am--; 1753a634bfcSVikram Hegde } 1763a634bfcSVikram Hegde 177*e03dceedSVikram Hegde /* Calculate the npg_am */ 178*e03dceedSVikram Hegde npages = npages_left; 179*e03dceedSVikram Hegde for (npg_am = 0, npages >>= 1; npages; npages >>= 1, npg_am++) 180*e03dceedSVikram Hegde ; 181*e03dceedSVikram Hegde 182*e03dceedSVikram Hegde am = MIN(max_am, MIN(dvma_am, npg_am)); 183*e03dceedSVikram Hegde 184*e03dceedSVikram Hegde iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI); 185*e03dceedSVikram Hegde 186*e03dceedSVikram Hegde npages = (1 << am); 187*e03dceedSVikram Hegde npages_left -= npages; 188*e03dceedSVikram Hegde dvma += (npages * IMMU_PAGESIZE); 189*e03dceedSVikram Hegde } 190*e03dceedSVikram Hegde 191*e03dceedSVikram Hegde if (npages_left) { 192*e03dceedSVikram Hegde iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI); 193*e03dceedSVikram Hegde } 1943a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 1953a634bfcSVikram Hegde } 1963a634bfcSVikram Hegde 1973a634bfcSVikram Hegde /* 1983a634bfcSVikram Hegde * iotlb_dsi() 1993a634bfcSVikram Hegde * domain specific invalidation 2003a634bfcSVikram Hegde */ 2013a634bfcSVikram Hegde static void 2023a634bfcSVikram Hegde iotlb_dsi(immu_t *immu, uint_t domain_id) 2033a634bfcSVikram Hegde { 2043a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 2053a634bfcSVikram Hegde iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI); 2063a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 2073a634bfcSVikram Hegde } 2083a634bfcSVikram Hegde 2093a634bfcSVikram Hegde /* 2103a634bfcSVikram Hegde * iotlb_global() 2113a634bfcSVikram Hegde * global iotlb invalidation 2123a634bfcSVikram Hegde */ 2133a634bfcSVikram Hegde static void 2143a634bfcSVikram Hegde iotlb_global(immu_t *immu) 2153a634bfcSVikram Hegde { 2163a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 2173a634bfcSVikram Hegde iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL); 2183a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 2193a634bfcSVikram Hegde } 2203a634bfcSVikram Hegde 2213a634bfcSVikram Hegde 2223a634bfcSVikram Hegde static int 2233a634bfcSVikram Hegde gaw2agaw(int gaw) 2243a634bfcSVikram Hegde { 2253a634bfcSVikram Hegde int r, agaw; 2263a634bfcSVikram Hegde 2273a634bfcSVikram Hegde r = (gaw - 12) % 9; 2283a634bfcSVikram Hegde 2293a634bfcSVikram Hegde if (r == 0) 2303a634bfcSVikram Hegde agaw = gaw; 2313a634bfcSVikram Hegde else 2323a634bfcSVikram Hegde agaw = gaw + 9 - r; 2333a634bfcSVikram Hegde 2343a634bfcSVikram Hegde if (agaw > 64) 2353a634bfcSVikram Hegde agaw = 64; 2363a634bfcSVikram Hegde 2373a634bfcSVikram Hegde return (agaw); 2383a634bfcSVikram Hegde } 2393a634bfcSVikram Hegde 2403a634bfcSVikram Hegde /* 2413a634bfcSVikram Hegde * set_immu_agaw() 2423a634bfcSVikram Hegde * calculate agaw for a IOMMU unit 2433a634bfcSVikram Hegde */ 2443a634bfcSVikram Hegde static int 2453a634bfcSVikram Hegde set_agaw(immu_t *immu) 2463a634bfcSVikram Hegde { 2473a634bfcSVikram Hegde int mgaw, magaw, agaw; 2483a634bfcSVikram Hegde uint_t bitpos; 2493a634bfcSVikram Hegde int max_sagaw_mask, sagaw_mask, mask; 2503a634bfcSVikram Hegde int nlevels; 2513a634bfcSVikram Hegde 2523a634bfcSVikram Hegde /* 2533a634bfcSVikram Hegde * mgaw is the maximum guest address width. 2543a634bfcSVikram Hegde * Addresses above this value will be 2553a634bfcSVikram Hegde * blocked by the IOMMU unit. 2563a634bfcSVikram Hegde * sagaw is a bitmask that lists all the 2573a634bfcSVikram Hegde * AGAWs supported by this IOMMU unit. 2583a634bfcSVikram Hegde */ 2593a634bfcSVikram Hegde mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap); 2603a634bfcSVikram Hegde sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap); 2613a634bfcSVikram Hegde 2623a634bfcSVikram Hegde magaw = gaw2agaw(mgaw); 2633a634bfcSVikram Hegde 2643a634bfcSVikram Hegde /* 2653a634bfcSVikram Hegde * Get bitpos corresponding to 2663a634bfcSVikram Hegde * magaw 2673a634bfcSVikram Hegde */ 2683a634bfcSVikram Hegde 2693a634bfcSVikram Hegde /* 2703a634bfcSVikram Hegde * Maximum SAGAW is specified by 2713a634bfcSVikram Hegde * Vt-d spec. 2723a634bfcSVikram Hegde */ 2733a634bfcSVikram Hegde max_sagaw_mask = ((1 << 5) - 1); 2743a634bfcSVikram Hegde 2753a634bfcSVikram Hegde if (sagaw_mask > max_sagaw_mask) { 2763a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: SAGAW bitmask (%x) " 2773a634bfcSVikram Hegde "is larger than maximu SAGAW bitmask " 2783a634bfcSVikram Hegde "(%x) specified by Intel Vt-d spec", 2793a634bfcSVikram Hegde immu->immu_name, sagaw_mask, max_sagaw_mask); 2803a634bfcSVikram Hegde return (DDI_FAILURE); 2813a634bfcSVikram Hegde } 2823a634bfcSVikram Hegde 2833a634bfcSVikram Hegde /* 2843a634bfcSVikram Hegde * Find a supported AGAW <= magaw 2853a634bfcSVikram Hegde * 2863a634bfcSVikram Hegde * sagaw_mask bitpos AGAW (bits) nlevels 2873a634bfcSVikram Hegde * ============================================== 2883a634bfcSVikram Hegde * 0 0 0 0 1 0 30 2 2893a634bfcSVikram Hegde * 0 0 0 1 0 1 39 3 2903a634bfcSVikram Hegde * 0 0 1 0 0 2 48 4 2913a634bfcSVikram Hegde * 0 1 0 0 0 3 57 5 2923a634bfcSVikram Hegde * 1 0 0 0 0 4 64(66) 6 2933a634bfcSVikram Hegde */ 2943a634bfcSVikram Hegde mask = 1; 2953a634bfcSVikram Hegde nlevels = 0; 2963a634bfcSVikram Hegde agaw = 0; 2973a634bfcSVikram Hegde for (mask = 1, bitpos = 0; bitpos < 5; 2983a634bfcSVikram Hegde bitpos++, mask <<= 1) { 2993a634bfcSVikram Hegde if (mask & sagaw_mask) { 3003a634bfcSVikram Hegde nlevels = bitpos + 2; 3013a634bfcSVikram Hegde agaw = 30 + (bitpos * 9); 3023a634bfcSVikram Hegde } 3033a634bfcSVikram Hegde } 3043a634bfcSVikram Hegde 3053a634bfcSVikram Hegde /* calculated agaw can be > 64 */ 3063a634bfcSVikram Hegde agaw = (agaw > 64) ? 64 : agaw; 3073a634bfcSVikram Hegde 3083a634bfcSVikram Hegde if (agaw < 30 || agaw > magaw) { 3093a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated AGAW (%d) " 3103a634bfcSVikram Hegde "is outside valid limits [30,%d] specified by Vt-d spec " 3113a634bfcSVikram Hegde "and magaw", immu->immu_name, agaw, magaw); 3123a634bfcSVikram Hegde return (DDI_FAILURE); 3133a634bfcSVikram Hegde } 3143a634bfcSVikram Hegde 3153a634bfcSVikram Hegde if (nlevels < 2 || nlevels > 6) { 3163a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated pagetable " 3173a634bfcSVikram Hegde "level (%d) is outside valid limits [2,6]", 3183a634bfcSVikram Hegde immu->immu_name, nlevels); 3193a634bfcSVikram Hegde return (DDI_FAILURE); 3203a634bfcSVikram Hegde } 3213a634bfcSVikram Hegde 3223a634bfcSVikram Hegde ddi_err(DER_LOG, NULL, "Calculated pagetable " 3233a634bfcSVikram Hegde "level (%d), agaw = %d", nlevels, agaw); 3243a634bfcSVikram Hegde 3253a634bfcSVikram Hegde immu->immu_dvma_nlevels = nlevels; 3263a634bfcSVikram Hegde immu->immu_dvma_agaw = agaw; 3273a634bfcSVikram Hegde 3283a634bfcSVikram Hegde return (DDI_SUCCESS); 3293a634bfcSVikram Hegde } 3303a634bfcSVikram Hegde 3313a634bfcSVikram Hegde static int 3323a634bfcSVikram Hegde setup_regs(immu_t *immu) 3333a634bfcSVikram Hegde { 3343a634bfcSVikram Hegde int error; 3353a634bfcSVikram Hegde 3363a634bfcSVikram Hegde ASSERT(immu); 3373a634bfcSVikram Hegde ASSERT(immu->immu_name); 3383a634bfcSVikram Hegde 3393a634bfcSVikram Hegde /* 3403a634bfcSVikram Hegde * This lock may be acquired by the IOMMU interrupt handler 3413a634bfcSVikram Hegde */ 3423a634bfcSVikram Hegde mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER, 3433a634bfcSVikram Hegde (void *)ipltospl(IMMU_INTR_IPL)); 3443a634bfcSVikram Hegde 3453a634bfcSVikram Hegde /* 3463a634bfcSVikram Hegde * map the register address space 3473a634bfcSVikram Hegde */ 3483a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0, 3493a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 3503a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr, 3513a634bfcSVikram Hegde &(immu->immu_regs_handle)); 3523a634bfcSVikram Hegde 3533a634bfcSVikram Hegde if (error == DDI_FAILURE) { 3543a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Intel IOMMU register map failed", 3553a634bfcSVikram Hegde immu->immu_name); 3563a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 3573a634bfcSVikram Hegde return (DDI_FAILURE); 3583a634bfcSVikram Hegde } 3593a634bfcSVikram Hegde 3603a634bfcSVikram Hegde /* 3613a634bfcSVikram Hegde * get the register value 3623a634bfcSVikram Hegde */ 3633a634bfcSVikram Hegde immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP); 3643a634bfcSVikram Hegde immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP); 3653a634bfcSVikram Hegde 3663a634bfcSVikram Hegde /* 3673a634bfcSVikram Hegde * if the hardware access is non-coherent, we need clflush 3683a634bfcSVikram Hegde */ 3693a634bfcSVikram Hegde if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) { 3703a634bfcSVikram Hegde immu->immu_dvma_coherent = B_TRUE; 3713a634bfcSVikram Hegde } else { 3723a634bfcSVikram Hegde immu->immu_dvma_coherent = B_FALSE; 3733a634bfcSVikram Hegde if (!(x86_feature & X86_CLFSH)) { 3743a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, 3753a634bfcSVikram Hegde "immu unit %s can't be enabled due to " 3763a634bfcSVikram Hegde "missing clflush functionality", immu->immu_name); 3773a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 3783a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 3793a634bfcSVikram Hegde return (DDI_FAILURE); 3803a634bfcSVikram Hegde } 3813a634bfcSVikram Hegde } 3823a634bfcSVikram Hegde 383*e03dceedSVikram Hegde /* Setup SNP and TM reserved fields */ 384*e03dceedSVikram Hegde immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu); 385*e03dceedSVikram Hegde immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu); 386*e03dceedSVikram Hegde 3873a634bfcSVikram Hegde /* 3883a634bfcSVikram Hegde * Check for Mobile 4 series chipset 3893a634bfcSVikram Hegde */ 3903a634bfcSVikram Hegde if (immu_quirk_mobile4 == B_TRUE && 3913a634bfcSVikram Hegde !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 3923a634bfcSVikram Hegde ddi_err(DER_LOG, NULL, 3933a634bfcSVikram Hegde "IMMU: Mobile 4 chipset quirk detected. " 3943a634bfcSVikram Hegde "Force-setting RWBF"); 3953a634bfcSVikram Hegde IMMU_CAP_SET_RWBF(immu->immu_regs_cap); 3963a634bfcSVikram Hegde ASSERT(IMMU_CAP_GET_RWBF(immu->immu_regs_cap)); 3973a634bfcSVikram Hegde } 3983a634bfcSVikram Hegde 3993a634bfcSVikram Hegde /* 4003a634bfcSVikram Hegde * retrieve the maximum number of domains 4013a634bfcSVikram Hegde */ 4023a634bfcSVikram Hegde immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap); 4033a634bfcSVikram Hegde 4043a634bfcSVikram Hegde /* 4053a634bfcSVikram Hegde * calculate the agaw 4063a634bfcSVikram Hegde */ 4073a634bfcSVikram Hegde if (set_agaw(immu) != DDI_SUCCESS) { 4083a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 4093a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 4103a634bfcSVikram Hegde return (DDI_FAILURE); 4113a634bfcSVikram Hegde } 4123a634bfcSVikram Hegde immu->immu_regs_cmdval = 0; 4133a634bfcSVikram Hegde 4143a634bfcSVikram Hegde return (DDI_SUCCESS); 4153a634bfcSVikram Hegde } 4163a634bfcSVikram Hegde 4173a634bfcSVikram Hegde /* ############### Functions exported ################## */ 4183a634bfcSVikram Hegde 4193a634bfcSVikram Hegde /* 4203a634bfcSVikram Hegde * immu_regs_setup() 4213a634bfcSVikram Hegde * Setup mappings to a IMMU unit's registers 4223a634bfcSVikram Hegde * so that they can be read/written 4233a634bfcSVikram Hegde */ 4243a634bfcSVikram Hegde void 4253a634bfcSVikram Hegde immu_regs_setup(list_t *listp) 4263a634bfcSVikram Hegde { 4273a634bfcSVikram Hegde int i; 4283a634bfcSVikram Hegde immu_t *immu; 4293a634bfcSVikram Hegde 4303a634bfcSVikram Hegde for (i = 0; i < IMMU_MAXSEG; i++) { 4313a634bfcSVikram Hegde immu = list_head(listp); 4323a634bfcSVikram Hegde for (; immu; immu = list_next(listp, immu)) { 4333a634bfcSVikram Hegde /* do your best, continue on error */ 4343a634bfcSVikram Hegde if (setup_regs(immu) != DDI_SUCCESS) { 4353a634bfcSVikram Hegde immu->immu_regs_setup = B_FALSE; 4363a634bfcSVikram Hegde } else { 4373a634bfcSVikram Hegde immu->immu_regs_setup = B_TRUE; 4383a634bfcSVikram Hegde } 4393a634bfcSVikram Hegde } 4403a634bfcSVikram Hegde } 4413a634bfcSVikram Hegde } 4423a634bfcSVikram Hegde 4433a634bfcSVikram Hegde /* 4443a634bfcSVikram Hegde * immu_regs_map() 4453a634bfcSVikram Hegde */ 4463a634bfcSVikram Hegde int 4473a634bfcSVikram Hegde immu_regs_resume(immu_t *immu) 4483a634bfcSVikram Hegde { 4493a634bfcSVikram Hegde int error; 4503a634bfcSVikram Hegde 4513a634bfcSVikram Hegde /* 4523a634bfcSVikram Hegde * remap the register address space 4533a634bfcSVikram Hegde */ 4543a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0, 4553a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 4563a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr, 4573a634bfcSVikram Hegde &(immu->immu_regs_handle)); 4583a634bfcSVikram Hegde if (error != DDI_SUCCESS) { 4593a634bfcSVikram Hegde return (DDI_FAILURE); 4603a634bfcSVikram Hegde } 4613a634bfcSVikram Hegde 4623a634bfcSVikram Hegde immu_regs_set_root_table(immu); 4633a634bfcSVikram Hegde 4643a634bfcSVikram Hegde immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr, 4653a634bfcSVikram Hegde immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr); 4663a634bfcSVikram Hegde 4673a634bfcSVikram Hegde (void) immu_intr_handler(immu); 4683a634bfcSVikram Hegde 4693a634bfcSVikram Hegde immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg); 4703a634bfcSVikram Hegde 4713a634bfcSVikram Hegde immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value); 4723a634bfcSVikram Hegde 4733a634bfcSVikram Hegde 4743a634bfcSVikram Hegde return (error); 4753a634bfcSVikram Hegde } 4763a634bfcSVikram Hegde 4773a634bfcSVikram Hegde /* 4783a634bfcSVikram Hegde * immu_regs_suspend() 4793a634bfcSVikram Hegde */ 4803a634bfcSVikram Hegde void 4813a634bfcSVikram Hegde immu_regs_suspend(immu_t *immu) 4823a634bfcSVikram Hegde { 4833a634bfcSVikram Hegde 4843a634bfcSVikram Hegde immu->immu_intrmap_running = B_FALSE; 4853a634bfcSVikram Hegde 4863a634bfcSVikram Hegde /* Finally, unmap the regs */ 4873a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 4883a634bfcSVikram Hegde } 4893a634bfcSVikram Hegde 4903a634bfcSVikram Hegde /* 4913a634bfcSVikram Hegde * immu_regs_startup() 4923a634bfcSVikram Hegde * set a IMMU unit's registers to startup the unit 4933a634bfcSVikram Hegde */ 4943a634bfcSVikram Hegde void 4953a634bfcSVikram Hegde immu_regs_startup(immu_t *immu) 4963a634bfcSVikram Hegde { 4973a634bfcSVikram Hegde uint32_t status; 4983a634bfcSVikram Hegde 4993a634bfcSVikram Hegde if (immu->immu_regs_setup == B_FALSE) { 5003a634bfcSVikram Hegde return; 5013a634bfcSVikram Hegde } 5023a634bfcSVikram Hegde 5033a634bfcSVikram Hegde ASSERT(immu->immu_regs_running == B_FALSE); 5043a634bfcSVikram Hegde 5053a634bfcSVikram Hegde ASSERT(MUTEX_HELD(&(immu->immu_lock))); 5063a634bfcSVikram Hegde 5073a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5083a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 5093a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_TE); 5103a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 5113a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_TES), status); 5123a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_TE; 5133a634bfcSVikram Hegde immu->immu_regs_running = B_TRUE; 5143a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5153a634bfcSVikram Hegde 5163a634bfcSVikram Hegde ddi_err(DER_NOTE, NULL, "IMMU %s running", immu->immu_name); 5173a634bfcSVikram Hegde } 5183a634bfcSVikram Hegde 5193a634bfcSVikram Hegde /* 5203a634bfcSVikram Hegde * immu_regs_shutdown() 5213a634bfcSVikram Hegde * shutdown a unit 5223a634bfcSVikram Hegde */ 5233a634bfcSVikram Hegde void 5243a634bfcSVikram Hegde immu_regs_shutdown(immu_t *immu) 5253a634bfcSVikram Hegde { 5263a634bfcSVikram Hegde uint32_t status; 5273a634bfcSVikram Hegde 5283a634bfcSVikram Hegde if (immu->immu_regs_running == B_FALSE) { 5293a634bfcSVikram Hegde return; 5303a634bfcSVikram Hegde } 5313a634bfcSVikram Hegde 5323a634bfcSVikram Hegde ASSERT(immu->immu_regs_setup == B_TRUE); 5333a634bfcSVikram Hegde 5343a634bfcSVikram Hegde ASSERT(MUTEX_HELD(&(immu->immu_lock))); 5353a634bfcSVikram Hegde 5363a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5373a634bfcSVikram Hegde immu->immu_regs_cmdval &= ~IMMU_GCMD_TE; 5383a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 5393a634bfcSVikram Hegde immu->immu_regs_cmdval); 5403a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 5413a634bfcSVikram Hegde get_reg32, !(status & IMMU_GSTS_TES), status); 5423a634bfcSVikram Hegde immu->immu_regs_running = B_FALSE; 5433a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5443a634bfcSVikram Hegde 5453a634bfcSVikram Hegde ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name); 5463a634bfcSVikram Hegde } 5473a634bfcSVikram Hegde 5483a634bfcSVikram Hegde /* 5493a634bfcSVikram Hegde * immu_regs_intr() 5503a634bfcSVikram Hegde * Set a IMMU unit regs to setup a IMMU unit's 5513a634bfcSVikram Hegde * interrupt handler 5523a634bfcSVikram Hegde */ 5533a634bfcSVikram Hegde void 5543a634bfcSVikram Hegde immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 5553a634bfcSVikram Hegde uint32_t uaddr) 5563a634bfcSVikram Hegde { 5573a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5583a634bfcSVikram Hegde immu->immu_regs_intr_msi_addr = msi_addr; 5593a634bfcSVikram Hegde immu->immu_regs_intr_uaddr = uaddr; 5603a634bfcSVikram Hegde immu->immu_regs_intr_msi_data = msi_data; 5613a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr); 5623a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr); 5633a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data); 5643a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_CON, 0); 5653a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5663a634bfcSVikram Hegde } 5673a634bfcSVikram Hegde 5683a634bfcSVikram Hegde /* 5693a634bfcSVikram Hegde * immu_regs_passthru_supported() 5703a634bfcSVikram Hegde * Returns B_TRUE ifi passthru is supported 5713a634bfcSVikram Hegde */ 5723a634bfcSVikram Hegde boolean_t 5733a634bfcSVikram Hegde immu_regs_passthru_supported(immu_t *immu) 5743a634bfcSVikram Hegde { 5753a634bfcSVikram Hegde if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) { 5763a634bfcSVikram Hegde return (B_TRUE); 5773a634bfcSVikram Hegde } 5783a634bfcSVikram Hegde 5793a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "Passthru not supported"); 5803a634bfcSVikram Hegde return (B_FALSE); 5813a634bfcSVikram Hegde } 5823a634bfcSVikram Hegde 5833a634bfcSVikram Hegde /* 5843a634bfcSVikram Hegde * immu_regs_is_TM_reserved() 5853a634bfcSVikram Hegde * Returns B_TRUE if TM field is reserved 5863a634bfcSVikram Hegde */ 5873a634bfcSVikram Hegde boolean_t 5883a634bfcSVikram Hegde immu_regs_is_TM_reserved(immu_t *immu) 5893a634bfcSVikram Hegde { 5903a634bfcSVikram Hegde if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) || 5913a634bfcSVikram Hegde IMMU_ECAP_GET_CH(immu->immu_regs_excap)) { 5923a634bfcSVikram Hegde return (B_FALSE); 5933a634bfcSVikram Hegde } 5943a634bfcSVikram Hegde return (B_TRUE); 5953a634bfcSVikram Hegde } 5963a634bfcSVikram Hegde 5973a634bfcSVikram Hegde /* 5983a634bfcSVikram Hegde * immu_regs_is_SNP_reserved() 5993a634bfcSVikram Hegde * Returns B_TRUE if SNP field is reserved 6003a634bfcSVikram Hegde */ 6013a634bfcSVikram Hegde boolean_t 6023a634bfcSVikram Hegde immu_regs_is_SNP_reserved(immu_t *immu) 6033a634bfcSVikram Hegde { 6043a634bfcSVikram Hegde 6053a634bfcSVikram Hegde return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE); 6063a634bfcSVikram Hegde } 6073a634bfcSVikram Hegde 6083a634bfcSVikram Hegde /* 6093a634bfcSVikram Hegde * immu_regs_wbf_flush() 6103a634bfcSVikram Hegde * If required and supported, write to IMMU 6113a634bfcSVikram Hegde * unit's regs to flush DMA write buffer(s) 6123a634bfcSVikram Hegde */ 6133a634bfcSVikram Hegde void 6143a634bfcSVikram Hegde immu_regs_wbf_flush(immu_t *immu) 6153a634bfcSVikram Hegde { 6163a634bfcSVikram Hegde uint32_t status; 6173a634bfcSVikram Hegde 6183a634bfcSVikram Hegde if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 6193a634bfcSVikram Hegde return; 6203a634bfcSVikram Hegde } 6213a634bfcSVikram Hegde 6223a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 6233a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 6243a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_WBF); 6253a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 6263a634bfcSVikram Hegde get_reg32, (!(status & IMMU_GSTS_WBFS)), status); 6273a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 6283a634bfcSVikram Hegde } 6293a634bfcSVikram Hegde 6303a634bfcSVikram Hegde /* 6313a634bfcSVikram Hegde * immu_regs_cpu_flush() 6323a634bfcSVikram Hegde * flush the cpu cache line after CPU memory writes, so 6333a634bfcSVikram Hegde * IOMMU can see the writes 6343a634bfcSVikram Hegde */ 6353a634bfcSVikram Hegde void 6363a634bfcSVikram Hegde immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size) 6373a634bfcSVikram Hegde { 638*e03dceedSVikram Hegde uint64_t i; 6393a634bfcSVikram Hegde 6403a634bfcSVikram Hegde ASSERT(immu); 6413a634bfcSVikram Hegde 6423a634bfcSVikram Hegde if (immu->immu_dvma_coherent == B_TRUE) 6433a634bfcSVikram Hegde return; 6443a634bfcSVikram Hegde 645*e03dceedSVikram Hegde for (i = 0; i < size; i += x86_clflush_size, addr += x86_clflush_size) { 646*e03dceedSVikram Hegde clflush_insn(addr); 6473a634bfcSVikram Hegde } 6483a634bfcSVikram Hegde 6493a634bfcSVikram Hegde mfence_insn(); 6503a634bfcSVikram Hegde } 6513a634bfcSVikram Hegde 6523a634bfcSVikram Hegde void 6533a634bfcSVikram Hegde immu_regs_iotlb_flush(immu_t *immu, uint_t domainid, uint64_t dvma, 6543a634bfcSVikram Hegde uint64_t count, uint_t hint, immu_iotlb_inv_t type) 6553a634bfcSVikram Hegde { 6563a634bfcSVikram Hegde ASSERT(immu); 6573a634bfcSVikram Hegde 658*e03dceedSVikram Hegde #ifndef TEST 659*e03dceedSVikram Hegde if (type == IOTLB_PSI && !IMMU_CAP_GET_PSI(immu->immu_regs_cap)) { 660*e03dceedSVikram Hegde dvma = 0; 661*e03dceedSVikram Hegde count = 0; 662*e03dceedSVikram Hegde hint = 0; 663*e03dceedSVikram Hegde type = IOTLB_DSI; 664*e03dceedSVikram Hegde } 665*e03dceedSVikram Hegde #else 666*e03dceedSVikram Hegde if (type == IOTLB_PSI) { 667*e03dceedSVikram Hegde dvma = 0; 668*e03dceedSVikram Hegde count = 0; 669*e03dceedSVikram Hegde hint = 0; 670*e03dceedSVikram Hegde type = IOTLB_DSI; 671*e03dceedSVikram Hegde } 672*e03dceedSVikram Hegde #endif 673*e03dceedSVikram Hegde 674*e03dceedSVikram Hegde 6753a634bfcSVikram Hegde switch (type) { 6763a634bfcSVikram Hegde case IOTLB_PSI: 6773a634bfcSVikram Hegde ASSERT(domainid > 0); 6783a634bfcSVikram Hegde ASSERT(count > 0); 6793a634bfcSVikram Hegde iotlb_psi(immu, domainid, dvma, count, hint); 6803a634bfcSVikram Hegde break; 6813a634bfcSVikram Hegde case IOTLB_DSI: 6823a634bfcSVikram Hegde ASSERT(domainid > 0); 6833a634bfcSVikram Hegde ASSERT(dvma == 0); 6843a634bfcSVikram Hegde ASSERT(count == 0); 6853a634bfcSVikram Hegde ASSERT(hint == 0); 6863a634bfcSVikram Hegde iotlb_dsi(immu, domainid); 6873a634bfcSVikram Hegde break; 6883a634bfcSVikram Hegde case IOTLB_GLOBAL: 6893a634bfcSVikram Hegde ASSERT(domainid == 0); 6903a634bfcSVikram Hegde ASSERT(dvma == 0); 6913a634bfcSVikram Hegde ASSERT(count == 0); 6923a634bfcSVikram Hegde ASSERT(hint == 0); 6933a634bfcSVikram Hegde iotlb_global(immu); 6943a634bfcSVikram Hegde break; 6953a634bfcSVikram Hegde default: 6963a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, "invalid IOTLB invalidation type: %d", 6973a634bfcSVikram Hegde type); 6983a634bfcSVikram Hegde /*NOTREACHED*/ 6993a634bfcSVikram Hegde } 7003a634bfcSVikram Hegde } 7013a634bfcSVikram Hegde 7023a634bfcSVikram Hegde /* 7033a634bfcSVikram Hegde * immu_regs_context_flush() 7043a634bfcSVikram Hegde * flush the context cache 7053a634bfcSVikram Hegde */ 7063a634bfcSVikram Hegde void 7073a634bfcSVikram Hegde immu_regs_context_flush(immu_t *immu, uint8_t function_mask, 7083a634bfcSVikram Hegde uint16_t sid, uint_t did, immu_context_inv_t type) 7093a634bfcSVikram Hegde { 7103a634bfcSVikram Hegde uint64_t command = 0; 7113a634bfcSVikram Hegde uint64_t status; 7123a634bfcSVikram Hegde 7133a634bfcSVikram Hegde ASSERT(immu); 7143a634bfcSVikram Hegde ASSERT(rw_write_held(&(immu->immu_ctx_rwlock))); 7153a634bfcSVikram Hegde 7163a634bfcSVikram Hegde /* 7173a634bfcSVikram Hegde * define the command 7183a634bfcSVikram Hegde */ 7193a634bfcSVikram Hegde switch (type) { 7203a634bfcSVikram Hegde case CONTEXT_FSI: 7213a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DEVICE 7223a634bfcSVikram Hegde | CCMD_INV_DID(did) 7233a634bfcSVikram Hegde | CCMD_INV_SID(sid) | CCMD_INV_FM(function_mask); 7243a634bfcSVikram Hegde break; 7253a634bfcSVikram Hegde case CONTEXT_DSI: 7263a634bfcSVikram Hegde ASSERT(function_mask == 0); 7273a634bfcSVikram Hegde ASSERT(sid == 0); 7283a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DOMAIN 7293a634bfcSVikram Hegde | CCMD_INV_DID(did); 7303a634bfcSVikram Hegde break; 7313a634bfcSVikram Hegde case CONTEXT_GLOBAL: 7323a634bfcSVikram Hegde ASSERT(function_mask == 0); 7333a634bfcSVikram Hegde ASSERT(sid == 0); 7343a634bfcSVikram Hegde ASSERT(did == 0); 7353a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_GLOBAL; 7363a634bfcSVikram Hegde break; 7373a634bfcSVikram Hegde default: 7383a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, 7393a634bfcSVikram Hegde "%s: incorrect context cache flush type", 7403a634bfcSVikram Hegde immu->immu_name); 7413a634bfcSVikram Hegde /*NOTREACHED*/ 7423a634bfcSVikram Hegde } 7433a634bfcSVikram Hegde 7443a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 745*e03dceedSVikram Hegde ASSERT(!(get_reg64(immu, IMMU_REG_CONTEXT_CMD) & CCMD_INV_ICC)); 7463a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_CONTEXT_CMD, command); 7473a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64, 7483a634bfcSVikram Hegde (!(status & CCMD_INV_ICC)), status); 7493a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 7503a634bfcSVikram Hegde } 7513a634bfcSVikram Hegde 7523a634bfcSVikram Hegde void 7533a634bfcSVikram Hegde immu_regs_set_root_table(immu_t *immu) 7543a634bfcSVikram Hegde { 7553a634bfcSVikram Hegde uint32_t status; 7563a634bfcSVikram Hegde 7573a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 7583a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_ROOTENTRY, 7593a634bfcSVikram Hegde immu->immu_ctx_root->hwpg_paddr); 7603a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 7613a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SRTP); 7623a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 7633a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_RTPS), status); 7643a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 7653a634bfcSVikram Hegde } 7663a634bfcSVikram Hegde 7673a634bfcSVikram Hegde 7683a634bfcSVikram Hegde /* enable queued invalidation interface */ 7693a634bfcSVikram Hegde void 7703a634bfcSVikram Hegde immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value) 7713a634bfcSVikram Hegde { 7723a634bfcSVikram Hegde uint32_t status; 7733a634bfcSVikram Hegde 7743a634bfcSVikram Hegde if (immu_qinv_enable == B_FALSE) 7753a634bfcSVikram Hegde return; 7763a634bfcSVikram Hegde 7773a634bfcSVikram Hegde mutex_enter(&immu->immu_regs_lock); 7783a634bfcSVikram Hegde immu->immu_qinv_reg_value = qinv_reg_value; 7793a634bfcSVikram Hegde /* Initialize the Invalidation Queue Tail register to zero */ 7803a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QT, 0); 7813a634bfcSVikram Hegde 7823a634bfcSVikram Hegde /* set invalidation queue base address register */ 7833a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value); 7843a634bfcSVikram Hegde 7853a634bfcSVikram Hegde /* enable queued invalidation interface */ 7863a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 7873a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_QIE); 7883a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 7893a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_QIES), status); 7903a634bfcSVikram Hegde mutex_exit(&immu->immu_regs_lock); 7913a634bfcSVikram Hegde 7923a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_QIE; 7933a634bfcSVikram Hegde immu->immu_qinv_running = B_TRUE; 7943a634bfcSVikram Hegde 7953a634bfcSVikram Hegde } 7963a634bfcSVikram Hegde 7973a634bfcSVikram Hegde /* enable interrupt remapping hardware unit */ 7983a634bfcSVikram Hegde void 7993a634bfcSVikram Hegde immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg) 8003a634bfcSVikram Hegde { 8013a634bfcSVikram Hegde uint32_t status; 8023a634bfcSVikram Hegde 8033a634bfcSVikram Hegde if (immu_intrmap_enable == B_FALSE) 8043a634bfcSVikram Hegde return; 8053a634bfcSVikram Hegde 8063a634bfcSVikram Hegde /* set interrupt remap table pointer */ 8073a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 8083a634bfcSVikram Hegde immu->immu_intrmap_irta_reg = irta_reg; 8093a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_IRTAR, irta_reg); 8103a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 8113a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SIRTP); 8123a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 8133a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRTPS), status); 8143a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 8153a634bfcSVikram Hegde 8163a634bfcSVikram Hegde /* global flush intr entry cache */ 8173a634bfcSVikram Hegde if (immu_qinv_enable == B_TRUE) 8183a634bfcSVikram Hegde immu_qinv_intr_global(immu); 8193a634bfcSVikram Hegde 8203a634bfcSVikram Hegde /* enable interrupt remapping */ 8213a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 8223a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 8233a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_IRE); 8243a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 8253a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRES), 8263a634bfcSVikram Hegde status); 8273a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_IRE; 8283a634bfcSVikram Hegde 8293a634bfcSVikram Hegde /* set compatible mode */ 8303a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 8313a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_CFI); 8323a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 8333a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_CFIS), 8343a634bfcSVikram Hegde status); 8353a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_CFI; 8363a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 8373a634bfcSVikram Hegde 8383a634bfcSVikram Hegde immu->immu_intrmap_running = B_TRUE; 8393a634bfcSVikram Hegde } 8403a634bfcSVikram Hegde 8413a634bfcSVikram Hegde uint64_t 8423a634bfcSVikram Hegde immu_regs_get64(immu_t *immu, uint_t reg) 8433a634bfcSVikram Hegde { 8443a634bfcSVikram Hegde return (get_reg64(immu, reg)); 8453a634bfcSVikram Hegde } 8463a634bfcSVikram Hegde 8473a634bfcSVikram Hegde uint32_t 8483a634bfcSVikram Hegde immu_regs_get32(immu_t *immu, uint_t reg) 8493a634bfcSVikram Hegde { 8503a634bfcSVikram Hegde return (get_reg32(immu, reg)); 8513a634bfcSVikram Hegde } 8523a634bfcSVikram Hegde 8533a634bfcSVikram Hegde void 8543a634bfcSVikram Hegde immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val) 8553a634bfcSVikram Hegde { 8563a634bfcSVikram Hegde put_reg64(immu, reg, val); 8573a634bfcSVikram Hegde } 8583a634bfcSVikram Hegde 8593a634bfcSVikram Hegde void 8603a634bfcSVikram Hegde immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val) 8613a634bfcSVikram Hegde { 8623a634bfcSVikram Hegde put_reg32(immu, reg, val); 8633a634bfcSVikram Hegde } 864