13a634bfcSVikram Hegde /* 23a634bfcSVikram Hegde * CDDL HEADER START 33a634bfcSVikram Hegde * 43a634bfcSVikram Hegde * The contents of this file are subject to the terms of the 53a634bfcSVikram Hegde * Common Development and Distribution License (the "License"). 63a634bfcSVikram Hegde * You may not use this file except in compliance with the License. 73a634bfcSVikram Hegde * 83a634bfcSVikram Hegde * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93a634bfcSVikram Hegde * or http://www.opensolaris.org/os/licensing. 103a634bfcSVikram Hegde * See the License for the specific language governing permissions 113a634bfcSVikram Hegde * and limitations under the License. 123a634bfcSVikram Hegde * 133a634bfcSVikram Hegde * When distributing Covered Code, include this CDDL HEADER in each 143a634bfcSVikram Hegde * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153a634bfcSVikram Hegde * If applicable, add the following below this CDDL HEADER, with the 163a634bfcSVikram Hegde * fields enclosed by brackets "[]" replaced with your own identifying 173a634bfcSVikram Hegde * information: Portions Copyright [yyyy] [name of copyright owner] 183a634bfcSVikram Hegde * 193a634bfcSVikram Hegde * CDDL HEADER END 203a634bfcSVikram Hegde */ 213a634bfcSVikram Hegde /* 22*c94adbf9SFrank Van Der Linden * Portions Copyright (c) 2010, Oracle and/or its affiliates. 23*c94adbf9SFrank Van Der Linden * All rights reserved. 243a634bfcSVikram Hegde */ 253a634bfcSVikram Hegde 263a634bfcSVikram Hegde /* 273a634bfcSVikram Hegde * immu_regs.c - File that operates on a IMMU unit's regsiters 283a634bfcSVikram Hegde */ 293a634bfcSVikram Hegde #include <sys/dditypes.h> 303a634bfcSVikram Hegde #include <sys/ddi.h> 313a634bfcSVikram Hegde #include <sys/archsystm.h> 323a634bfcSVikram Hegde #include <sys/x86_archext.h> 333a634bfcSVikram Hegde #include <sys/spl.h> 34e03dceedSVikram Hegde #include <sys/sysmacros.h> 353a634bfcSVikram Hegde #include <sys/immu.h> 363a634bfcSVikram Hegde 373a634bfcSVikram Hegde #define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \ 383a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset))) 393a634bfcSVikram Hegde #define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \ 403a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset))) 413a634bfcSVikram Hegde #define put_reg32(immu, offset, val) ddi_put32\ 423a634bfcSVikram Hegde ((immu)->immu_regs_handle, \ 433a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset)), val) 443a634bfcSVikram Hegde #define put_reg64(immu, offset, val) ddi_put64\ 453a634bfcSVikram Hegde ((immu)->immu_regs_handle, \ 463a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset)), val) 473a634bfcSVikram Hegde 48*c94adbf9SFrank Van Der Linden struct immu_flushops immu_regs_flushops = { 49*c94adbf9SFrank Van Der Linden immu_regs_context_fsi, 50*c94adbf9SFrank Van Der Linden immu_regs_context_dsi, 51*c94adbf9SFrank Van Der Linden immu_regs_context_gbl, 52*c94adbf9SFrank Van Der Linden immu_regs_iotlb_psi, 53*c94adbf9SFrank Van Der Linden immu_regs_iotlb_dsi, 54*c94adbf9SFrank Van Der Linden immu_regs_iotlb_gbl 55*c94adbf9SFrank Van Der Linden }; 56*c94adbf9SFrank Van Der Linden 573a634bfcSVikram Hegde /* 583a634bfcSVikram Hegde * wait max 60s for the hardware completion 593a634bfcSVikram Hegde */ 603a634bfcSVikram Hegde #define IMMU_MAX_WAIT_TIME 60000000 613a634bfcSVikram Hegde #define wait_completion(immu, offset, getf, completion, status) \ 623a634bfcSVikram Hegde { \ 633a634bfcSVikram Hegde clock_t stick = ddi_get_lbolt(); \ 643a634bfcSVikram Hegde clock_t ntick; \ 653a634bfcSVikram Hegde _NOTE(CONSTCOND) \ 663a634bfcSVikram Hegde while (1) { \ 673a634bfcSVikram Hegde status = getf(immu, offset); \ 683a634bfcSVikram Hegde ntick = ddi_get_lbolt(); \ 693a634bfcSVikram Hegde if (completion) { \ 703a634bfcSVikram Hegde break; \ 713a634bfcSVikram Hegde } \ 723a634bfcSVikram Hegde if (ntick - stick >= drv_usectohz(IMMU_MAX_WAIT_TIME)) { \ 733a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, \ 743a634bfcSVikram Hegde "immu wait completion time out"); \ 753a634bfcSVikram Hegde /*NOTREACHED*/ \ 763a634bfcSVikram Hegde } else { \ 773a634bfcSVikram Hegde iommu_cpu_nop();\ 783a634bfcSVikram Hegde }\ 793a634bfcSVikram Hegde }\ 803a634bfcSVikram Hegde } 813a634bfcSVikram Hegde 823a634bfcSVikram Hegde static ddi_device_acc_attr_t immu_regs_attr = { 833a634bfcSVikram Hegde DDI_DEVICE_ATTR_V0, 843a634bfcSVikram Hegde DDI_NEVERSWAP_ACC, 853a634bfcSVikram Hegde DDI_STRICTORDER_ACC, 863a634bfcSVikram Hegde }; 873a634bfcSVikram Hegde 883a634bfcSVikram Hegde /* 893a634bfcSVikram Hegde * iotlb_flush() 903a634bfcSVikram Hegde * flush the iotlb cache 913a634bfcSVikram Hegde */ 923a634bfcSVikram Hegde static void 933a634bfcSVikram Hegde iotlb_flush(immu_t *immu, uint_t domain_id, 943a634bfcSVikram Hegde uint64_t addr, uint_t am, uint_t hint, immu_iotlb_inv_t type) 953a634bfcSVikram Hegde { 963a634bfcSVikram Hegde uint64_t command = 0, iva = 0; 973a634bfcSVikram Hegde uint_t iva_offset, iotlb_offset; 983a634bfcSVikram Hegde uint64_t status = 0; 993a634bfcSVikram Hegde 1003a634bfcSVikram Hegde /* no lock needed since cap and excap fields are RDONLY */ 1013a634bfcSVikram Hegde iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap); 1023a634bfcSVikram Hegde iotlb_offset = iva_offset + 8; 1033a634bfcSVikram Hegde 1043a634bfcSVikram Hegde /* 1053a634bfcSVikram Hegde * prepare drain read/write command 1063a634bfcSVikram Hegde */ 1073a634bfcSVikram Hegde if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) { 1083a634bfcSVikram Hegde command |= TLB_INV_DRAIN_WRITE; 1093a634bfcSVikram Hegde } 1103a634bfcSVikram Hegde 1113a634bfcSVikram Hegde if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) { 1123a634bfcSVikram Hegde command |= TLB_INV_DRAIN_READ; 1133a634bfcSVikram Hegde } 1143a634bfcSVikram Hegde 1153a634bfcSVikram Hegde /* 1163a634bfcSVikram Hegde * if the hardward doesn't support page selective invalidation, we 1173a634bfcSVikram Hegde * will use domain type. Otherwise, use global type 1183a634bfcSVikram Hegde */ 1193a634bfcSVikram Hegde switch (type) { 1203a634bfcSVikram Hegde case IOTLB_PSI: 121e03dceedSVikram Hegde ASSERT(IMMU_CAP_GET_PSI(immu->immu_regs_cap)); 122e03dceedSVikram Hegde ASSERT(am <= IMMU_CAP_GET_MAMV(immu->immu_regs_cap)); 123e03dceedSVikram Hegde ASSERT(!(addr & IMMU_PAGEOFFSET)); 1243a634bfcSVikram Hegde command |= TLB_INV_PAGE | TLB_INV_IVT | 1253a634bfcSVikram Hegde TLB_INV_DID(domain_id); 1263a634bfcSVikram Hegde iva = addr | am | TLB_IVA_HINT(hint); 1273a634bfcSVikram Hegde break; 1283a634bfcSVikram Hegde case IOTLB_DSI: 1293a634bfcSVikram Hegde command |= TLB_INV_DOMAIN | TLB_INV_IVT | 1303a634bfcSVikram Hegde TLB_INV_DID(domain_id); 1313a634bfcSVikram Hegde break; 1323a634bfcSVikram Hegde case IOTLB_GLOBAL: 1333a634bfcSVikram Hegde command |= TLB_INV_GLOBAL | TLB_INV_IVT; 1343a634bfcSVikram Hegde break; 1353a634bfcSVikram Hegde default: 1363a634bfcSVikram Hegde ddi_err(DER_MODE, NULL, "%s: incorrect iotlb flush type", 1373a634bfcSVikram Hegde immu->immu_name); 1383a634bfcSVikram Hegde return; 1393a634bfcSVikram Hegde } 1403a634bfcSVikram Hegde 1413a634bfcSVikram Hegde if (iva) 1423a634bfcSVikram Hegde put_reg64(immu, iva_offset, iva); 1433a634bfcSVikram Hegde put_reg64(immu, iotlb_offset, command); 1443a634bfcSVikram Hegde wait_completion(immu, iotlb_offset, get_reg64, 1453a634bfcSVikram Hegde (!(status & TLB_INV_IVT)), status); 1463a634bfcSVikram Hegde } 1473a634bfcSVikram Hegde 1483a634bfcSVikram Hegde /* 149*c94adbf9SFrank Van Der Linden * immu_regs_iotlb_psi() 1503a634bfcSVikram Hegde * iotlb page specific invalidation 1513a634bfcSVikram Hegde */ 152*c94adbf9SFrank Van Der Linden void 153*c94adbf9SFrank Van Der Linden immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages, 154e03dceedSVikram Hegde uint_t hint) 1553a634bfcSVikram Hegde { 156e03dceedSVikram Hegde int dvma_am; 157e03dceedSVikram Hegde int npg_am; 158e03dceedSVikram Hegde int max_am; 159e03dceedSVikram Hegde int am; 160e03dceedSVikram Hegde uint64_t align; 161e03dceedSVikram Hegde int npages_left; 162e03dceedSVikram Hegde int npages; 163e03dceedSVikram Hegde int i; 164e03dceedSVikram Hegde 165*c94adbf9SFrank Van Der Linden if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) { 166*c94adbf9SFrank Van Der Linden immu_regs_iotlb_dsi(immu, did); 167*c94adbf9SFrank Van Der Linden return; 168*c94adbf9SFrank Van Der Linden } 169*c94adbf9SFrank Van Der Linden 170e03dceedSVikram Hegde ASSERT(dvma % IMMU_PAGESIZE == 0); 171e03dceedSVikram Hegde 172e03dceedSVikram Hegde max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap); 1733a634bfcSVikram Hegde 1743a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 1753a634bfcSVikram Hegde 176e03dceedSVikram Hegde npages_left = snpages; 177e03dceedSVikram Hegde for (i = 0; i < immu_flush_gran && npages_left > 0; i++) { 1783a634bfcSVikram Hegde /* First calculate alignment of DVMA */ 179e03dceedSVikram Hegde 180e03dceedSVikram Hegde if (dvma == 0) { 181e03dceedSVikram Hegde dvma_am = max_am; 1823a634bfcSVikram Hegde } else { 183e03dceedSVikram Hegde for (align = (1 << 12), dvma_am = 1; 184e03dceedSVikram Hegde (dvma & align) == 0; align <<= 1, dvma_am++) 1853a634bfcSVikram Hegde ; 186e03dceedSVikram Hegde dvma_am--; 1873a634bfcSVikram Hegde } 1883a634bfcSVikram Hegde 189e03dceedSVikram Hegde /* Calculate the npg_am */ 190e03dceedSVikram Hegde npages = npages_left; 191e03dceedSVikram Hegde for (npg_am = 0, npages >>= 1; npages; npages >>= 1, npg_am++) 192e03dceedSVikram Hegde ; 193e03dceedSVikram Hegde 194e03dceedSVikram Hegde am = MIN(max_am, MIN(dvma_am, npg_am)); 195e03dceedSVikram Hegde 196e03dceedSVikram Hegde iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI); 197e03dceedSVikram Hegde 198e03dceedSVikram Hegde npages = (1 << am); 199e03dceedSVikram Hegde npages_left -= npages; 200e03dceedSVikram Hegde dvma += (npages * IMMU_PAGESIZE); 201e03dceedSVikram Hegde } 202e03dceedSVikram Hegde 203e03dceedSVikram Hegde if (npages_left) { 204e03dceedSVikram Hegde iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI); 205e03dceedSVikram Hegde } 2063a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 2073a634bfcSVikram Hegde } 2083a634bfcSVikram Hegde 2093a634bfcSVikram Hegde /* 210*c94adbf9SFrank Van Der Linden * immu_regs_iotlb_dsi() 2113a634bfcSVikram Hegde * domain specific invalidation 2123a634bfcSVikram Hegde */ 213*c94adbf9SFrank Van Der Linden void 214*c94adbf9SFrank Van Der Linden immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id) 2153a634bfcSVikram Hegde { 2163a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 2173a634bfcSVikram Hegde iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI); 2183a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 2193a634bfcSVikram Hegde } 2203a634bfcSVikram Hegde 2213a634bfcSVikram Hegde /* 222*c94adbf9SFrank Van Der Linden * immu_regs_iotlb_gbl() 2233a634bfcSVikram Hegde * global iotlb invalidation 2243a634bfcSVikram Hegde */ 225*c94adbf9SFrank Van Der Linden void 226*c94adbf9SFrank Van Der Linden immu_regs_iotlb_gbl(immu_t *immu) 2273a634bfcSVikram Hegde { 2283a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 2293a634bfcSVikram Hegde iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL); 2303a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 2313a634bfcSVikram Hegde } 2323a634bfcSVikram Hegde 2333a634bfcSVikram Hegde 2343a634bfcSVikram Hegde static int 2353a634bfcSVikram Hegde gaw2agaw(int gaw) 2363a634bfcSVikram Hegde { 2373a634bfcSVikram Hegde int r, agaw; 2383a634bfcSVikram Hegde 2393a634bfcSVikram Hegde r = (gaw - 12) % 9; 2403a634bfcSVikram Hegde 2413a634bfcSVikram Hegde if (r == 0) 2423a634bfcSVikram Hegde agaw = gaw; 2433a634bfcSVikram Hegde else 2443a634bfcSVikram Hegde agaw = gaw + 9 - r; 2453a634bfcSVikram Hegde 2463a634bfcSVikram Hegde if (agaw > 64) 2473a634bfcSVikram Hegde agaw = 64; 2483a634bfcSVikram Hegde 2493a634bfcSVikram Hegde return (agaw); 2503a634bfcSVikram Hegde } 2513a634bfcSVikram Hegde 2523a634bfcSVikram Hegde /* 2533a634bfcSVikram Hegde * set_immu_agaw() 2543a634bfcSVikram Hegde * calculate agaw for a IOMMU unit 2553a634bfcSVikram Hegde */ 2563a634bfcSVikram Hegde static int 2573a634bfcSVikram Hegde set_agaw(immu_t *immu) 2583a634bfcSVikram Hegde { 2593a634bfcSVikram Hegde int mgaw, magaw, agaw; 2603a634bfcSVikram Hegde uint_t bitpos; 2613a634bfcSVikram Hegde int max_sagaw_mask, sagaw_mask, mask; 2623a634bfcSVikram Hegde int nlevels; 2633a634bfcSVikram Hegde 2643a634bfcSVikram Hegde /* 2653a634bfcSVikram Hegde * mgaw is the maximum guest address width. 2663a634bfcSVikram Hegde * Addresses above this value will be 2673a634bfcSVikram Hegde * blocked by the IOMMU unit. 2683a634bfcSVikram Hegde * sagaw is a bitmask that lists all the 2693a634bfcSVikram Hegde * AGAWs supported by this IOMMU unit. 2703a634bfcSVikram Hegde */ 2713a634bfcSVikram Hegde mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap); 2723a634bfcSVikram Hegde sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap); 2733a634bfcSVikram Hegde 2743a634bfcSVikram Hegde magaw = gaw2agaw(mgaw); 2753a634bfcSVikram Hegde 2763a634bfcSVikram Hegde /* 2773a634bfcSVikram Hegde * Get bitpos corresponding to 2783a634bfcSVikram Hegde * magaw 2793a634bfcSVikram Hegde */ 2803a634bfcSVikram Hegde 2813a634bfcSVikram Hegde /* 2823a634bfcSVikram Hegde * Maximum SAGAW is specified by 2833a634bfcSVikram Hegde * Vt-d spec. 2843a634bfcSVikram Hegde */ 2853a634bfcSVikram Hegde max_sagaw_mask = ((1 << 5) - 1); 2863a634bfcSVikram Hegde 2873a634bfcSVikram Hegde if (sagaw_mask > max_sagaw_mask) { 2883a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: SAGAW bitmask (%x) " 2893a634bfcSVikram Hegde "is larger than maximu SAGAW bitmask " 2903a634bfcSVikram Hegde "(%x) specified by Intel Vt-d spec", 2913a634bfcSVikram Hegde immu->immu_name, sagaw_mask, max_sagaw_mask); 2923a634bfcSVikram Hegde return (DDI_FAILURE); 2933a634bfcSVikram Hegde } 2943a634bfcSVikram Hegde 2953a634bfcSVikram Hegde /* 2963a634bfcSVikram Hegde * Find a supported AGAW <= magaw 2973a634bfcSVikram Hegde * 2983a634bfcSVikram Hegde * sagaw_mask bitpos AGAW (bits) nlevels 2993a634bfcSVikram Hegde * ============================================== 3003a634bfcSVikram Hegde * 0 0 0 0 1 0 30 2 3013a634bfcSVikram Hegde * 0 0 0 1 0 1 39 3 3023a634bfcSVikram Hegde * 0 0 1 0 0 2 48 4 3033a634bfcSVikram Hegde * 0 1 0 0 0 3 57 5 3043a634bfcSVikram Hegde * 1 0 0 0 0 4 64(66) 6 3053a634bfcSVikram Hegde */ 3063a634bfcSVikram Hegde mask = 1; 3073a634bfcSVikram Hegde nlevels = 0; 3083a634bfcSVikram Hegde agaw = 0; 3093a634bfcSVikram Hegde for (mask = 1, bitpos = 0; bitpos < 5; 3103a634bfcSVikram Hegde bitpos++, mask <<= 1) { 3113a634bfcSVikram Hegde if (mask & sagaw_mask) { 3123a634bfcSVikram Hegde nlevels = bitpos + 2; 3133a634bfcSVikram Hegde agaw = 30 + (bitpos * 9); 3143a634bfcSVikram Hegde } 3153a634bfcSVikram Hegde } 3163a634bfcSVikram Hegde 3173a634bfcSVikram Hegde /* calculated agaw can be > 64 */ 3183a634bfcSVikram Hegde agaw = (agaw > 64) ? 64 : agaw; 3193a634bfcSVikram Hegde 3203a634bfcSVikram Hegde if (agaw < 30 || agaw > magaw) { 3213a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated AGAW (%d) " 3223a634bfcSVikram Hegde "is outside valid limits [30,%d] specified by Vt-d spec " 3233a634bfcSVikram Hegde "and magaw", immu->immu_name, agaw, magaw); 3243a634bfcSVikram Hegde return (DDI_FAILURE); 3253a634bfcSVikram Hegde } 3263a634bfcSVikram Hegde 3273a634bfcSVikram Hegde if (nlevels < 2 || nlevels > 6) { 3283a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated pagetable " 3293a634bfcSVikram Hegde "level (%d) is outside valid limits [2,6]", 3303a634bfcSVikram Hegde immu->immu_name, nlevels); 3313a634bfcSVikram Hegde return (DDI_FAILURE); 3323a634bfcSVikram Hegde } 3333a634bfcSVikram Hegde 3343a634bfcSVikram Hegde ddi_err(DER_LOG, NULL, "Calculated pagetable " 3353a634bfcSVikram Hegde "level (%d), agaw = %d", nlevels, agaw); 3363a634bfcSVikram Hegde 3373a634bfcSVikram Hegde immu->immu_dvma_nlevels = nlevels; 3383a634bfcSVikram Hegde immu->immu_dvma_agaw = agaw; 3393a634bfcSVikram Hegde 3403a634bfcSVikram Hegde return (DDI_SUCCESS); 3413a634bfcSVikram Hegde } 3423a634bfcSVikram Hegde 3433a634bfcSVikram Hegde static int 3443a634bfcSVikram Hegde setup_regs(immu_t *immu) 3453a634bfcSVikram Hegde { 3463a634bfcSVikram Hegde int error; 3473a634bfcSVikram Hegde 3483a634bfcSVikram Hegde ASSERT(immu); 3493a634bfcSVikram Hegde ASSERT(immu->immu_name); 3503a634bfcSVikram Hegde 3513a634bfcSVikram Hegde /* 3523a634bfcSVikram Hegde * This lock may be acquired by the IOMMU interrupt handler 3533a634bfcSVikram Hegde */ 3543a634bfcSVikram Hegde mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER, 3553a634bfcSVikram Hegde (void *)ipltospl(IMMU_INTR_IPL)); 3563a634bfcSVikram Hegde 3573a634bfcSVikram Hegde /* 3583a634bfcSVikram Hegde * map the register address space 3593a634bfcSVikram Hegde */ 3603a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0, 3613a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 3623a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr, 3633a634bfcSVikram Hegde &(immu->immu_regs_handle)); 3643a634bfcSVikram Hegde 3653a634bfcSVikram Hegde if (error == DDI_FAILURE) { 3663a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Intel IOMMU register map failed", 3673a634bfcSVikram Hegde immu->immu_name); 3683a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 3693a634bfcSVikram Hegde return (DDI_FAILURE); 3703a634bfcSVikram Hegde } 3713a634bfcSVikram Hegde 3723a634bfcSVikram Hegde /* 3733a634bfcSVikram Hegde * get the register value 3743a634bfcSVikram Hegde */ 3753a634bfcSVikram Hegde immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP); 3763a634bfcSVikram Hegde immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP); 3773a634bfcSVikram Hegde 3783a634bfcSVikram Hegde /* 3793a634bfcSVikram Hegde * if the hardware access is non-coherent, we need clflush 3803a634bfcSVikram Hegde */ 3813a634bfcSVikram Hegde if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) { 3823a634bfcSVikram Hegde immu->immu_dvma_coherent = B_TRUE; 3833a634bfcSVikram Hegde } else { 3843a634bfcSVikram Hegde immu->immu_dvma_coherent = B_FALSE; 3853a634bfcSVikram Hegde if (!(x86_feature & X86_CLFSH)) { 3863a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, 3873a634bfcSVikram Hegde "immu unit %s can't be enabled due to " 3883a634bfcSVikram Hegde "missing clflush functionality", immu->immu_name); 3893a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 3903a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 3913a634bfcSVikram Hegde return (DDI_FAILURE); 3923a634bfcSVikram Hegde } 3933a634bfcSVikram Hegde } 3943a634bfcSVikram Hegde 395e03dceedSVikram Hegde /* Setup SNP and TM reserved fields */ 396e03dceedSVikram Hegde immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu); 397e03dceedSVikram Hegde immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu); 398e03dceedSVikram Hegde 3993a634bfcSVikram Hegde /* 4003a634bfcSVikram Hegde * Check for Mobile 4 series chipset 4013a634bfcSVikram Hegde */ 4023a634bfcSVikram Hegde if (immu_quirk_mobile4 == B_TRUE && 4033a634bfcSVikram Hegde !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 4043a634bfcSVikram Hegde ddi_err(DER_LOG, NULL, 4053a634bfcSVikram Hegde "IMMU: Mobile 4 chipset quirk detected. " 4063a634bfcSVikram Hegde "Force-setting RWBF"); 4073a634bfcSVikram Hegde IMMU_CAP_SET_RWBF(immu->immu_regs_cap); 4083a634bfcSVikram Hegde ASSERT(IMMU_CAP_GET_RWBF(immu->immu_regs_cap)); 4093a634bfcSVikram Hegde } 4103a634bfcSVikram Hegde 4113a634bfcSVikram Hegde /* 4123a634bfcSVikram Hegde * retrieve the maximum number of domains 4133a634bfcSVikram Hegde */ 4143a634bfcSVikram Hegde immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap); 4153a634bfcSVikram Hegde 4163a634bfcSVikram Hegde /* 4173a634bfcSVikram Hegde * calculate the agaw 4183a634bfcSVikram Hegde */ 4193a634bfcSVikram Hegde if (set_agaw(immu) != DDI_SUCCESS) { 4203a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 4213a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock)); 4223a634bfcSVikram Hegde return (DDI_FAILURE); 4233a634bfcSVikram Hegde } 4243a634bfcSVikram Hegde immu->immu_regs_cmdval = 0; 4253a634bfcSVikram Hegde 426*c94adbf9SFrank Van Der Linden immu->immu_flushops = &immu_regs_flushops; 427*c94adbf9SFrank Van Der Linden 4283a634bfcSVikram Hegde return (DDI_SUCCESS); 4293a634bfcSVikram Hegde } 4303a634bfcSVikram Hegde 4313a634bfcSVikram Hegde /* ############### Functions exported ################## */ 4323a634bfcSVikram Hegde 4333a634bfcSVikram Hegde /* 4343a634bfcSVikram Hegde * immu_regs_setup() 4353a634bfcSVikram Hegde * Setup mappings to a IMMU unit's registers 4363a634bfcSVikram Hegde * so that they can be read/written 4373a634bfcSVikram Hegde */ 4383a634bfcSVikram Hegde void 4393a634bfcSVikram Hegde immu_regs_setup(list_t *listp) 4403a634bfcSVikram Hegde { 4413a634bfcSVikram Hegde int i; 4423a634bfcSVikram Hegde immu_t *immu; 4433a634bfcSVikram Hegde 4443a634bfcSVikram Hegde for (i = 0; i < IMMU_MAXSEG; i++) { 4453a634bfcSVikram Hegde immu = list_head(listp); 4463a634bfcSVikram Hegde for (; immu; immu = list_next(listp, immu)) { 4473a634bfcSVikram Hegde /* do your best, continue on error */ 4483a634bfcSVikram Hegde if (setup_regs(immu) != DDI_SUCCESS) { 4493a634bfcSVikram Hegde immu->immu_regs_setup = B_FALSE; 4503a634bfcSVikram Hegde } else { 4513a634bfcSVikram Hegde immu->immu_regs_setup = B_TRUE; 4523a634bfcSVikram Hegde } 4533a634bfcSVikram Hegde } 4543a634bfcSVikram Hegde } 4553a634bfcSVikram Hegde } 4563a634bfcSVikram Hegde 4573a634bfcSVikram Hegde /* 4583a634bfcSVikram Hegde * immu_regs_map() 4593a634bfcSVikram Hegde */ 4603a634bfcSVikram Hegde int 4613a634bfcSVikram Hegde immu_regs_resume(immu_t *immu) 4623a634bfcSVikram Hegde { 4633a634bfcSVikram Hegde int error; 4643a634bfcSVikram Hegde 4653a634bfcSVikram Hegde /* 4663a634bfcSVikram Hegde * remap the register address space 4673a634bfcSVikram Hegde */ 4683a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0, 4693a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0, 4703a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr, 4713a634bfcSVikram Hegde &(immu->immu_regs_handle)); 4723a634bfcSVikram Hegde if (error != DDI_SUCCESS) { 4733a634bfcSVikram Hegde return (DDI_FAILURE); 4743a634bfcSVikram Hegde } 4753a634bfcSVikram Hegde 4763a634bfcSVikram Hegde immu_regs_set_root_table(immu); 4773a634bfcSVikram Hegde 4783a634bfcSVikram Hegde immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr, 4793a634bfcSVikram Hegde immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr); 4803a634bfcSVikram Hegde 4813a634bfcSVikram Hegde (void) immu_intr_handler(immu); 4823a634bfcSVikram Hegde 4833a634bfcSVikram Hegde immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg); 4843a634bfcSVikram Hegde 4853a634bfcSVikram Hegde immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value); 4863a634bfcSVikram Hegde 4873a634bfcSVikram Hegde 4883a634bfcSVikram Hegde return (error); 4893a634bfcSVikram Hegde } 4903a634bfcSVikram Hegde 4913a634bfcSVikram Hegde /* 4923a634bfcSVikram Hegde * immu_regs_suspend() 4933a634bfcSVikram Hegde */ 4943a634bfcSVikram Hegde void 4953a634bfcSVikram Hegde immu_regs_suspend(immu_t *immu) 4963a634bfcSVikram Hegde { 4973a634bfcSVikram Hegde 4983a634bfcSVikram Hegde immu->immu_intrmap_running = B_FALSE; 4993a634bfcSVikram Hegde 5003a634bfcSVikram Hegde /* Finally, unmap the regs */ 5013a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle)); 5023a634bfcSVikram Hegde } 5033a634bfcSVikram Hegde 5043a634bfcSVikram Hegde /* 5053a634bfcSVikram Hegde * immu_regs_startup() 5063a634bfcSVikram Hegde * set a IMMU unit's registers to startup the unit 5073a634bfcSVikram Hegde */ 5083a634bfcSVikram Hegde void 5093a634bfcSVikram Hegde immu_regs_startup(immu_t *immu) 5103a634bfcSVikram Hegde { 5113a634bfcSVikram Hegde uint32_t status; 5123a634bfcSVikram Hegde 5133a634bfcSVikram Hegde if (immu->immu_regs_setup == B_FALSE) { 5143a634bfcSVikram Hegde return; 5153a634bfcSVikram Hegde } 5163a634bfcSVikram Hegde 5173a634bfcSVikram Hegde ASSERT(immu->immu_regs_running == B_FALSE); 5183a634bfcSVikram Hegde 5193a634bfcSVikram Hegde ASSERT(MUTEX_HELD(&(immu->immu_lock))); 5203a634bfcSVikram Hegde 5213a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5223a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 5233a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_TE); 5243a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 5253a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_TES), status); 5263a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_TE; 5273a634bfcSVikram Hegde immu->immu_regs_running = B_TRUE; 5283a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5293a634bfcSVikram Hegde 5303a634bfcSVikram Hegde ddi_err(DER_NOTE, NULL, "IMMU %s running", immu->immu_name); 5313a634bfcSVikram Hegde } 5323a634bfcSVikram Hegde 5333a634bfcSVikram Hegde /* 5343a634bfcSVikram Hegde * immu_regs_shutdown() 5353a634bfcSVikram Hegde * shutdown a unit 5363a634bfcSVikram Hegde */ 5373a634bfcSVikram Hegde void 5383a634bfcSVikram Hegde immu_regs_shutdown(immu_t *immu) 5393a634bfcSVikram Hegde { 5403a634bfcSVikram Hegde uint32_t status; 5413a634bfcSVikram Hegde 5423a634bfcSVikram Hegde if (immu->immu_regs_running == B_FALSE) { 5433a634bfcSVikram Hegde return; 5443a634bfcSVikram Hegde } 5453a634bfcSVikram Hegde 5463a634bfcSVikram Hegde ASSERT(immu->immu_regs_setup == B_TRUE); 5473a634bfcSVikram Hegde 5483a634bfcSVikram Hegde ASSERT(MUTEX_HELD(&(immu->immu_lock))); 5493a634bfcSVikram Hegde 5503a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5513a634bfcSVikram Hegde immu->immu_regs_cmdval &= ~IMMU_GCMD_TE; 5523a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 5533a634bfcSVikram Hegde immu->immu_regs_cmdval); 5543a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 5553a634bfcSVikram Hegde get_reg32, !(status & IMMU_GSTS_TES), status); 5563a634bfcSVikram Hegde immu->immu_regs_running = B_FALSE; 5573a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5583a634bfcSVikram Hegde 5593a634bfcSVikram Hegde ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name); 5603a634bfcSVikram Hegde } 5613a634bfcSVikram Hegde 5623a634bfcSVikram Hegde /* 5633a634bfcSVikram Hegde * immu_regs_intr() 5643a634bfcSVikram Hegde * Set a IMMU unit regs to setup a IMMU unit's 5653a634bfcSVikram Hegde * interrupt handler 5663a634bfcSVikram Hegde */ 5673a634bfcSVikram Hegde void 5683a634bfcSVikram Hegde immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 5693a634bfcSVikram Hegde uint32_t uaddr) 5703a634bfcSVikram Hegde { 5713a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 5723a634bfcSVikram Hegde immu->immu_regs_intr_msi_addr = msi_addr; 5733a634bfcSVikram Hegde immu->immu_regs_intr_uaddr = uaddr; 5743a634bfcSVikram Hegde immu->immu_regs_intr_msi_data = msi_data; 5753a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr); 5763a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr); 5773a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data); 5783a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_CON, 0); 5793a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 5803a634bfcSVikram Hegde } 5813a634bfcSVikram Hegde 5823a634bfcSVikram Hegde /* 5833a634bfcSVikram Hegde * immu_regs_passthru_supported() 5843a634bfcSVikram Hegde * Returns B_TRUE ifi passthru is supported 5853a634bfcSVikram Hegde */ 5863a634bfcSVikram Hegde boolean_t 5873a634bfcSVikram Hegde immu_regs_passthru_supported(immu_t *immu) 5883a634bfcSVikram Hegde { 5893a634bfcSVikram Hegde if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) { 5903a634bfcSVikram Hegde return (B_TRUE); 5913a634bfcSVikram Hegde } 5923a634bfcSVikram Hegde 5933a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "Passthru not supported"); 5943a634bfcSVikram Hegde return (B_FALSE); 5953a634bfcSVikram Hegde } 5963a634bfcSVikram Hegde 5973a634bfcSVikram Hegde /* 5983a634bfcSVikram Hegde * immu_regs_is_TM_reserved() 5993a634bfcSVikram Hegde * Returns B_TRUE if TM field is reserved 6003a634bfcSVikram Hegde */ 6013a634bfcSVikram Hegde boolean_t 6023a634bfcSVikram Hegde immu_regs_is_TM_reserved(immu_t *immu) 6033a634bfcSVikram Hegde { 6043a634bfcSVikram Hegde if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) || 6053a634bfcSVikram Hegde IMMU_ECAP_GET_CH(immu->immu_regs_excap)) { 6063a634bfcSVikram Hegde return (B_FALSE); 6073a634bfcSVikram Hegde } 6083a634bfcSVikram Hegde return (B_TRUE); 6093a634bfcSVikram Hegde } 6103a634bfcSVikram Hegde 6113a634bfcSVikram Hegde /* 6123a634bfcSVikram Hegde * immu_regs_is_SNP_reserved() 6133a634bfcSVikram Hegde * Returns B_TRUE if SNP field is reserved 6143a634bfcSVikram Hegde */ 6153a634bfcSVikram Hegde boolean_t 6163a634bfcSVikram Hegde immu_regs_is_SNP_reserved(immu_t *immu) 6173a634bfcSVikram Hegde { 6183a634bfcSVikram Hegde 6193a634bfcSVikram Hegde return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE); 6203a634bfcSVikram Hegde } 6213a634bfcSVikram Hegde 6223a634bfcSVikram Hegde /* 6233a634bfcSVikram Hegde * immu_regs_wbf_flush() 6243a634bfcSVikram Hegde * If required and supported, write to IMMU 6253a634bfcSVikram Hegde * unit's regs to flush DMA write buffer(s) 6263a634bfcSVikram Hegde */ 6273a634bfcSVikram Hegde void 6283a634bfcSVikram Hegde immu_regs_wbf_flush(immu_t *immu) 6293a634bfcSVikram Hegde { 6303a634bfcSVikram Hegde uint32_t status; 6313a634bfcSVikram Hegde 6323a634bfcSVikram Hegde if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) { 6333a634bfcSVikram Hegde return; 6343a634bfcSVikram Hegde } 6353a634bfcSVikram Hegde 6363a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 6373a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 6383a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_WBF); 6393a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 6403a634bfcSVikram Hegde get_reg32, (!(status & IMMU_GSTS_WBFS)), status); 6413a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 6423a634bfcSVikram Hegde } 6433a634bfcSVikram Hegde 6443a634bfcSVikram Hegde /* 6453a634bfcSVikram Hegde * immu_regs_cpu_flush() 6463a634bfcSVikram Hegde * flush the cpu cache line after CPU memory writes, so 6473a634bfcSVikram Hegde * IOMMU can see the writes 6483a634bfcSVikram Hegde */ 6493a634bfcSVikram Hegde void 6503a634bfcSVikram Hegde immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size) 6513a634bfcSVikram Hegde { 652e03dceedSVikram Hegde uint64_t i; 6533a634bfcSVikram Hegde 6543a634bfcSVikram Hegde ASSERT(immu); 6553a634bfcSVikram Hegde 6563a634bfcSVikram Hegde if (immu->immu_dvma_coherent == B_TRUE) 6573a634bfcSVikram Hegde return; 6583a634bfcSVikram Hegde 659e03dceedSVikram Hegde for (i = 0; i < size; i += x86_clflush_size, addr += x86_clflush_size) { 660e03dceedSVikram Hegde clflush_insn(addr); 6613a634bfcSVikram Hegde } 6623a634bfcSVikram Hegde 6633a634bfcSVikram Hegde mfence_insn(); 6643a634bfcSVikram Hegde } 6653a634bfcSVikram Hegde 6663a634bfcSVikram Hegde /* 6673a634bfcSVikram Hegde * immu_regs_context_flush() 6683a634bfcSVikram Hegde * flush the context cache 6693a634bfcSVikram Hegde */ 670*c94adbf9SFrank Van Der Linden static void 671*c94adbf9SFrank Van Der Linden context_flush(immu_t *immu, uint8_t function_mask, 6723a634bfcSVikram Hegde uint16_t sid, uint_t did, immu_context_inv_t type) 6733a634bfcSVikram Hegde { 6743a634bfcSVikram Hegde uint64_t command = 0; 6753a634bfcSVikram Hegde uint64_t status; 6763a634bfcSVikram Hegde 6773a634bfcSVikram Hegde ASSERT(immu); 6783a634bfcSVikram Hegde ASSERT(rw_write_held(&(immu->immu_ctx_rwlock))); 6793a634bfcSVikram Hegde 6803a634bfcSVikram Hegde /* 6813a634bfcSVikram Hegde * define the command 6823a634bfcSVikram Hegde */ 6833a634bfcSVikram Hegde switch (type) { 6843a634bfcSVikram Hegde case CONTEXT_FSI: 6853a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DEVICE 6863a634bfcSVikram Hegde | CCMD_INV_DID(did) 6873a634bfcSVikram Hegde | CCMD_INV_SID(sid) | CCMD_INV_FM(function_mask); 6883a634bfcSVikram Hegde break; 6893a634bfcSVikram Hegde case CONTEXT_DSI: 6903a634bfcSVikram Hegde ASSERT(function_mask == 0); 6913a634bfcSVikram Hegde ASSERT(sid == 0); 6923a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DOMAIN 6933a634bfcSVikram Hegde | CCMD_INV_DID(did); 6943a634bfcSVikram Hegde break; 6953a634bfcSVikram Hegde case CONTEXT_GLOBAL: 6963a634bfcSVikram Hegde ASSERT(function_mask == 0); 6973a634bfcSVikram Hegde ASSERT(sid == 0); 6983a634bfcSVikram Hegde ASSERT(did == 0); 6993a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_GLOBAL; 7003a634bfcSVikram Hegde break; 7013a634bfcSVikram Hegde default: 7023a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, 7033a634bfcSVikram Hegde "%s: incorrect context cache flush type", 7043a634bfcSVikram Hegde immu->immu_name); 7053a634bfcSVikram Hegde /*NOTREACHED*/ 7063a634bfcSVikram Hegde } 7073a634bfcSVikram Hegde 7083a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 709e03dceedSVikram Hegde ASSERT(!(get_reg64(immu, IMMU_REG_CONTEXT_CMD) & CCMD_INV_ICC)); 7103a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_CONTEXT_CMD, command); 7113a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64, 7123a634bfcSVikram Hegde (!(status & CCMD_INV_ICC)), status); 7133a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 7143a634bfcSVikram Hegde } 7153a634bfcSVikram Hegde 7163a634bfcSVikram Hegde void 717*c94adbf9SFrank Van Der Linden immu_regs_context_fsi(immu_t *immu, uint8_t function_mask, 718*c94adbf9SFrank Van Der Linden uint16_t source_id, uint_t domain_id) 719*c94adbf9SFrank Van Der Linden { 720*c94adbf9SFrank Van Der Linden context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI); 721*c94adbf9SFrank Van Der Linden } 722*c94adbf9SFrank Van Der Linden 723*c94adbf9SFrank Van Der Linden void 724*c94adbf9SFrank Van Der Linden immu_regs_context_dsi(immu_t *immu, uint_t domain_id) 725*c94adbf9SFrank Van Der Linden { 726*c94adbf9SFrank Van Der Linden context_flush(immu, 0, 0, domain_id, CONTEXT_DSI); 727*c94adbf9SFrank Van Der Linden } 728*c94adbf9SFrank Van Der Linden 729*c94adbf9SFrank Van Der Linden void 730*c94adbf9SFrank Van Der Linden immu_regs_context_gbl(immu_t *immu) 731*c94adbf9SFrank Van Der Linden { 732*c94adbf9SFrank Van Der Linden context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL); 733*c94adbf9SFrank Van Der Linden } 734*c94adbf9SFrank Van Der Linden 735*c94adbf9SFrank Van Der Linden void 7363a634bfcSVikram Hegde immu_regs_set_root_table(immu_t *immu) 7373a634bfcSVikram Hegde { 7383a634bfcSVikram Hegde uint32_t status; 7393a634bfcSVikram Hegde 7403a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 7413a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_ROOTENTRY, 7423a634bfcSVikram Hegde immu->immu_ctx_root->hwpg_paddr); 7433a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 7443a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SRTP); 7453a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 7463a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_RTPS), status); 7473a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 7483a634bfcSVikram Hegde } 7493a634bfcSVikram Hegde 7503a634bfcSVikram Hegde 7513a634bfcSVikram Hegde /* enable queued invalidation interface */ 7523a634bfcSVikram Hegde void 7533a634bfcSVikram Hegde immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value) 7543a634bfcSVikram Hegde { 7553a634bfcSVikram Hegde uint32_t status; 7563a634bfcSVikram Hegde 7573a634bfcSVikram Hegde if (immu_qinv_enable == B_FALSE) 7583a634bfcSVikram Hegde return; 7593a634bfcSVikram Hegde 7603a634bfcSVikram Hegde mutex_enter(&immu->immu_regs_lock); 7613a634bfcSVikram Hegde immu->immu_qinv_reg_value = qinv_reg_value; 7623a634bfcSVikram Hegde /* Initialize the Invalidation Queue Tail register to zero */ 7633a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QT, 0); 7643a634bfcSVikram Hegde 7653a634bfcSVikram Hegde /* set invalidation queue base address register */ 7663a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value); 7673a634bfcSVikram Hegde 7683a634bfcSVikram Hegde /* enable queued invalidation interface */ 7693a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 7703a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_QIE); 7713a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 7723a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_QIES), status); 7733a634bfcSVikram Hegde mutex_exit(&immu->immu_regs_lock); 7743a634bfcSVikram Hegde 7753a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_QIE; 7763a634bfcSVikram Hegde immu->immu_qinv_running = B_TRUE; 7773a634bfcSVikram Hegde 7783a634bfcSVikram Hegde } 7793a634bfcSVikram Hegde 7803a634bfcSVikram Hegde /* enable interrupt remapping hardware unit */ 7813a634bfcSVikram Hegde void 7823a634bfcSVikram Hegde immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg) 7833a634bfcSVikram Hegde { 7843a634bfcSVikram Hegde uint32_t status; 7853a634bfcSVikram Hegde 7863a634bfcSVikram Hegde if (immu_intrmap_enable == B_FALSE) 7873a634bfcSVikram Hegde return; 7883a634bfcSVikram Hegde 7893a634bfcSVikram Hegde /* set interrupt remap table pointer */ 7903a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 7913a634bfcSVikram Hegde immu->immu_intrmap_irta_reg = irta_reg; 7923a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_IRTAR, irta_reg); 7933a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 7943a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SIRTP); 7953a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 7963a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRTPS), status); 7973a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 7983a634bfcSVikram Hegde 7993a634bfcSVikram Hegde /* global flush intr entry cache */ 8003a634bfcSVikram Hegde if (immu_qinv_enable == B_TRUE) 8013a634bfcSVikram Hegde immu_qinv_intr_global(immu); 8023a634bfcSVikram Hegde 8033a634bfcSVikram Hegde /* enable interrupt remapping */ 8043a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock)); 8053a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 8063a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_IRE); 8073a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 8083a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRES), 8093a634bfcSVikram Hegde status); 8103a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_IRE; 8113a634bfcSVikram Hegde 8123a634bfcSVikram Hegde /* set compatible mode */ 8133a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD, 8143a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_CFI); 8153a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS, 8163a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_CFIS), 8173a634bfcSVikram Hegde status); 8183a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_CFI; 8193a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock)); 8203a634bfcSVikram Hegde 8213a634bfcSVikram Hegde immu->immu_intrmap_running = B_TRUE; 8223a634bfcSVikram Hegde } 8233a634bfcSVikram Hegde 8243a634bfcSVikram Hegde uint64_t 8253a634bfcSVikram Hegde immu_regs_get64(immu_t *immu, uint_t reg) 8263a634bfcSVikram Hegde { 8273a634bfcSVikram Hegde return (get_reg64(immu, reg)); 8283a634bfcSVikram Hegde } 8293a634bfcSVikram Hegde 8303a634bfcSVikram Hegde uint32_t 8313a634bfcSVikram Hegde immu_regs_get32(immu_t *immu, uint_t reg) 8323a634bfcSVikram Hegde { 8333a634bfcSVikram Hegde return (get_reg32(immu, reg)); 8343a634bfcSVikram Hegde } 8353a634bfcSVikram Hegde 8363a634bfcSVikram Hegde void 8373a634bfcSVikram Hegde immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val) 8383a634bfcSVikram Hegde { 8393a634bfcSVikram Hegde put_reg64(immu, reg, val); 8403a634bfcSVikram Hegde } 8413a634bfcSVikram Hegde 8423a634bfcSVikram Hegde void 8433a634bfcSVikram Hegde immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val) 8443a634bfcSVikram Hegde { 8453a634bfcSVikram Hegde put_reg32(immu, reg, val); 8463a634bfcSVikram Hegde } 847