/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. */ #ifndef _AMD_IOMMU_IMPL_H #define _AMD_IOMMU_IMPL_H #ifdef __cplusplus extern "C" { #endif #include #ifdef _KERNEL #define AMD_IOMMU_PCI_PROG_IF (0x0) #define AMD_IOMMU_CAP (0x3) #define AMD_IOMMU_REG_SIZE (0x2028) #define AMD_IOMMU_DEVTBL_SZ (16) #define AMD_IOMMU_CMDBUF_SZ (15) #define AMD_IOMMU_EVENTLOG_SZ (15) #define AMD_IOMMU_DEVENT_SZ (32) #define AMD_IOMMU_CMD_SZ (16) #define AMD_IOMMU_EVENT_SZ (16) /* Capability Register offsets */ #define AMD_IOMMU_CAP_HDR_OFF (0x00) #define AMD_IOMMU_CAP_ADDR_LOW_OFF (0x04) #define AMD_IOMMU_CAP_ADDR_HI_OFF (0x08) #define AMD_IOMMU_CAP_RANGE_OFF (0x0C) #define AMD_IOMMU_CAP_MISC_OFF (0x10) /* ControL Registers offsets */ #define AMD_IOMMU_DEVTBL_REG_OFF (0x00) #define AMD_IOMMU_CMDBUF_REG_OFF (0x08) #define AMD_IOMMU_EVENTLOG_REG_OFF (0x10) #define AMD_IOMMU_CTRL_REG_OFF (0x18) #define AMD_IOMMU_EXCL_BASE_REG_OFF (0x20) #define AMD_IOMMU_EXCL_LIM_REG_OFF (0x28) #define AMD_IOMMU_CMDBUF_HEAD_REG_OFF (0x2000) #define AMD_IOMMU_CMDBUF_TAIL_REG_OFF (0x2008) #define AMD_IOMMU_EVENTLOG_HEAD_REG_OFF (0x2010) #define AMD_IOMMU_EVENTLOG_TAIL_REG_OFF (0x2018) #define AMD_IOMMU_STATUS_REG_OFF (0x2020) /* Capability Header Register Bits */ #define AMD_IOMMU_CAP_NPCACHE (26 << 16 | 26) #define AMD_IOMMU_CAP_HTTUN (25 << 16 | 25) #define AMD_IOMMU_CAP_IOTLB (24 << 16 | 24) #define AMD_IOMMU_CAP_TYPE (18 << 16 | 16) #define AMD_IOMMU_CAP_ID (7 << 16 | 0) /* Capability Range Register bits */ #define AMD_IOMMU_LAST_DEVFN (31 << 16 | 24) #define AMD_IOMMU_FIRST_DEVFN (23 << 16 | 16) #define AMD_IOMMU_RNG_BUS (15 << 16 | 8) #define AMD_IOMMU_RNG_VALID (7 << 16 | 7) #define AMD_IOMMU_HT_UNITID (4 << 16 | 0) /* Capability Misc Register bits */ #define AMD_IOMMU_HT_ATSRSV (22 << 16 | 22) #define AMD_IOMMU_VA_SIZE (21 << 16 | 15) #define AMD_IOMMU_PA_SIZE (14 << 16 | 8) #define AMD_IOMMU_MSINUM (4 << 16 | 0) /* Device Table Base Address register bits */ #define AMD_IOMMU_DEVTABBASE (51 << 16 | 12) #define AMD_IOMMU_DEVTABSIZE (8 << 16 | 0) /* Command Buffer Base Address register bits */ #define AMD_IOMMU_COMLEN (59 << 16 | 56) #define AMD_IOMMU_COMBASE (51 << 16 | 12) #define AMD_IOMMU_CMDBUF_MINSZ (8) #define AMD_IOMMU_CMDBUF_MAXSZ (15) /* Event Log Base Address register bits */ #define AMD_IOMMU_EVENTLEN (59 << 16 | 56) #define AMD_IOMMU_EVENTBASE (51 << 16 | 12) #define AMD_IOMMU_EVENTLOG_MINSZ (8) #define AMD_IOMMU_EVENTLOG_MAXSZ (15) /* Control register bits */ #define AMD_IOMMU_CMDBUF_ENABLE (12 << 16 | 12) #define AMD_IOMMU_ISOC (11 << 16 | 11) #define AMD_IOMMU_COHERENT (10 << 16 | 10) #define AMD_IOMMU_RESPASSPW (9 << 16 | 9) #define AMD_IOMMU_PASSPW (8 << 16 | 8) #define AMD_IOMMU_INVTO (7 << 16 | 5) #define AMD_IOMMU_COMWAITINT_ENABLE (4 << 16 | 4) #define AMD_IOMMU_EVENTINT_ENABLE (3 << 16 | 3) #define AMD_IOMMU_EVENTLOG_ENABLE (2 << 16 | 2) #define AMD_IOMMU_HT_TUN_ENABLE (1 << 16 | 1) #define AMD_IOMMU_ENABLE (0 << 16 | 0) /* Exclusion Base Register bits */ #define AMD_IOMMU_EXCL_BASE_ADDR (51 << 16 | 12) #define AMD_IOMMU_EXCL_BASE_ALLOW (1 << 16 | 1) #define AMD_IOMMU_EXCL_BASE_EXEN (0 << 16 | 0) /* Exclusion Limit Register bits */ #define AMD_IOMMU_EXCL_LIM (51 << 16 | 12) /* Command Buffer Head Pointer Register bits */ #define AMD_IOMMU_CMDHEADPTR (18 << 16 | 4) /* Command Buffer Tail Pointer Register bits */ #define AMD_IOMMU_CMDTAILPTR (18 << 16 | 4) /* Event Log Head Pointer Register bits */ #define AMD_IOMMU_EVENTHEADPTR (18 << 16 | 4) /* Event Log Tail Pointer Register bits */ #define AMD_IOMMU_EVENTTAILPTR (18 << 16 | 4) /* Status Register bits */ #define AMD_IOMMU_CMDBUF_RUN (4 << 16 | 4) #define AMD_IOMMU_EVENT_LOG_RUN (3 << 16 | 3) #define AMD_IOMMU_COMWAIT_INT (2 << 16 | 2) #define AMD_IOMMU_EVENT_LOG_INT (1 << 16 | 1) #define AMD_IOMMU_EVENT_OVERFLOW_INT (0 << 16 | 0) /* Device Table Bits */ /* size in bytes of each device table entry */ #define AMD_IOMMU_DEVTBL_ENTRY_SZ (32) /* Interrupt Remapping related Device Table bits */ #define AMD_IOMMU_DEVTBL_LINT1PASS ((191-128) << 16 | (191-128)) #define AMD_IOMMU_DEVTBL_LINT0PASS ((190-128) << 16 | (190-128)) #define AMD_IOMMU_DEVTBL_INTCTL ((189-128) << 16 | (188-128)) #define AMD_IOMMU_DEVTBL_NMIPASS ((186-128) << 16 | (186-128)) #define AMD_IOMMU_DEVTBL_EXTINTPAS ((185-128) << 16 | (185-128)) #define AMD_IOMMU_DEVTBL_INITPASS ((184-128) << 16 | (184-128)) #define AMD_IOMMU_DEVTBL_INTR_ROOT ((179-128) << 16 | (134-128)) #define AMD_IOMMU_DEVTBL_IG ((133-128) << 16 | (133-128)) #define AMD_IOMMU_DEVTBL_INTTABLEN ((132-128) << 16 | (129-128)) #define AMD_IOMMU_DEVTBL_IV ((128-128) << 16 | (128-128)) /* DMA Remapping related Device Table Bits */ #define AMD_IOMMU_DEVTBL_SYSMGT ((105-64) << 16 | (104-64)) #define AMD_IOMMU_DEVTBL_EX ((103-64) << 16 | (103-64)) #define AMD_IOMMU_DEVTBL_SD ((102-64) << 16 | (102-64)) #define AMD_IOMMU_DEVTBL_CACHE ((101-64) << 16 | (101-64)) #define AMD_IOMMU_DEVTBL_IOCTL ((100-64) << 16 | (99-64)) #define AMD_IOMMU_DEVTBL_SA ((98-64) << 16 | (98-64)) #define AMD_IOMMU_DEVTBL_SE ((97-64) << 16 | (97-64)) #define AMD_IOMMU_DEVTBL_IOTLB ((96-64) << 16 | (96-64)) #define AMD_IOMMU_DEVTBL_DOMAINID ((79-64) << 16 | (64-64)) #define AMD_IOMMU_DEVTBL_IW (62 << 16 | 62) #define AMD_IOMMU_DEVTBL_IR (61 << 16 | 61) #define AMD_IOMMU_DEVTBL_ROOT_PGTBL (51 << 16 | 12) #define AMD_IOMMU_DEVTBL_PG_MODE (11 << 16 | 9) #define AMD_IOMMU_DEVTBL_TV (1 << 16 | 1) #define AMD_IOMMU_DEVTBL_V (0 << 16 | 0) #define BUS_DEVFN_TO_BDF(b, devfn) (devfn) #define AMD_IOMMU_ALIAS_HASH_SZ (256) #define AMD_IOMMU_REG_ADDR_LOCKED (0x1) /* * IOMMU Command bits */ typedef enum { AMD_IOMMU_CMD_INVAL = 0, AMD_IOMMU_CMD_COMPL_WAIT, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, AMD_IOMMU_CMD_INVAL_IOTLB_PAGES, AMD_IOMMU_CMD_INVAL_INTR_TABLE, } amd_iommu_cmd_t; typedef enum { AMD_IOMMU_CMD_FLAGS_NONE = 0, AMD_IOMMU_CMD_FLAGS_COMPL_WAIT = 1, AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_F = 2, AMD_IOMMU_CMD_FLAGS_COMPL_WAIT_S = 4, AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL = 8, AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S = 16, AMD_IOMMU_CMD_FLAGS_IOTLB_INVAL_S = 32 } amd_iommu_cmd_flags_t; /* Common command bits */ #define AMD_IOMMU_CMD_OPCODE (31 << 16 | 28) /* Completion Wait command bits */ #define AMD_IOMMU_CMD_COMPL_WAIT_S (0 << 16 | 0) #define AMD_IOMMU_CMD_COMPL_WAIT_I (1 << 16 | 1) #define AMD_IOMMU_CMD_COMPL_WAIT_F (2 << 16 | 2) #define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_LO (31 << 16 | 3) #define AMD_IOMMU_CMD_COMPL_WAIT_STORE_ADDR_HI (19 << 16 | 0) /* Invalidate Device Table entry command bits */ #define AMD_IOMMU_CMD_INVAL_DEVTAB_DEVICEID (15 << 16 | 0) /* Invalidate IOMMU Pages command bits */ #define AMD_IOMMU_CMD_INVAL_PAGES_DOMAINID (15 << 16 | 0) #define AMD_IOMMU_CMD_INVAL_PAGES_S (0 << 16 | 0) #define AMD_IOMMU_CMD_INVAL_PAGES_PDE (1 << 16 | 1) #define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_LO (31 << 16 | 12) #define AMD_IOMMU_CMD_INVAL_PAGES_ADDR_HI (63 << 16 | 32) /* Invalidate IOTLB command bits */ #define AMD_IOMMU_CMD_INVAL_IOTLB_DEVICEID (15 << 16 | 0) #define AMD_IOMMU_CMD_INVAL_IOTLB_MAXPEND (31 << 16 | 24) #define AMD_IOMMU_CMD_INVAL_IOTLB_QUEUEID (15 << 16 | 0) #define AMD_IOMMU_CMD_INVAL_IOTLB_S (0 << 16 | 0) #define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_LO (31 << 16 | 12) #define AMD_IOMMU_CMD_INVAL_IOTLB_ADDR_HI (31 << 16 | 0) #define AMD_IOMMU_DEFAULT_MAXPEND (10) /* Invalidate Interrupt Table bits */ #define AMD_IOMMU_CMD_INVAL_INTR_DEVICEID (15 << 16 | 0) #if defined(__amd64) #define dmac_cookie_addr dmac_laddress #else #define dmac_cookie_addr dmac_address #endif #define AMD_IOMMU_TABLE_ALIGN ((1ULL << 12) - 1) #define AMD_IOMMU_MAX_DEVICEID (0xFFFF) /* * DMA sync macros * TODO: optimize sync only small ranges */ #define SYNC_FORDEV(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORDEV) #define SYNC_FORKERN(h) (void) ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORKERNEL) #define WAIT_SEC(s) drv_usecwait(1000000*(s)) #define CMD2OFF(c) ((c) << 4) #define OFF2CMD(o) ((o) >> 4) typedef union split { uint64_t u64; uint32_t u32[2]; } split_t; #define BITPOS_START(b) ((b) >> 16) #define BITPOS_END(b) ((b) & 0xFFFF) #define START_MASK64(s) (((s) == 63) ? ~((uint64_t)0) : \ (uint64_t)((1ULL << ((s)+1)) - 1)) #define START_MASK32(s) (((s) == 31) ? ~((uint32_t)0) : \ (uint32_t)((1ULL << ((s)+1)) - 1)) #define START_MASK16(s) (((s) == 15) ? ~((uint16_t)0) : \ (uint16_t)((1ULL << ((s)+1)) - 1)) #define START_MASK8(s) (((s) == 7) ? ~((uint8_t)0) : \ (uint8_t)((1ULL << ((s)+1)) - 1)) #define END_MASK(e) ((1ULL << (e)) - 1) #define BIT_MASK64(s, e) (uint64_t)(START_MASK64(s) & ~END_MASK(e)) #define BIT_MASK32(s, e) (uint32_t)(START_MASK32(s) & ~END_MASK(e)) #define BIT_MASK16(s, e) (uint16_t)(START_MASK16(s) & ~END_MASK(e)) #define BIT_MASK8(s, e) (uint8_t)(START_MASK8(s) & ~END_MASK(e)) #define AMD_IOMMU_REG_GET64_IMPL(rp, b) \ (((*(rp)) & (START_MASK64(BITPOS_START(b)))) >> BITPOS_END(b)) #define AMD_IOMMU_REG_GET64(rp, b) \ ((amd_iommu_64bit_bug) ? amd_iommu_reg_get64_workaround(rp, b) : \ AMD_IOMMU_REG_GET64_IMPL(rp, b)) #define AMD_IOMMU_REG_GET32(rp, b) \ (((*(rp)) & (START_MASK32(BITPOS_START(b)))) >> BITPOS_END(b)) #define AMD_IOMMU_REG_GET16(rp, b) \ (((*(rp)) & (START_MASK16(BITPOS_START(b)))) >> BITPOS_END(b)) #define AMD_IOMMU_REG_GET8(rp, b) \ (((*(rp)) & (START_MASK8(BITPOS_START(b)))) >> BITPOS_END(b)) #define AMD_IOMMU_REG_SET64_IMPL(rp, b, v) \ ((*(rp)) = \ (((uint64_t)(*(rp)) & ~(BIT_MASK64(BITPOS_START(b), BITPOS_END(b)))) \ | ((uint64_t)(v) << BITPOS_END(b)))) #define AMD_IOMMU_REG_SET64(rp, b, v) \ (void) ((amd_iommu_64bit_bug) ? \ amd_iommu_reg_set64_workaround(rp, b, v) : \ AMD_IOMMU_REG_SET64_IMPL(rp, b, v)) #define AMD_IOMMU_REG_SET32(rp, b, v) \ ((*(rp)) = \ (((uint32_t)(*(rp)) & ~(BIT_MASK32(BITPOS_START(b), BITPOS_END(b)))) \ | ((uint32_t)(v) << BITPOS_END(b)))) #define AMD_IOMMU_REG_SET16(rp, b, v) \ ((*(rp)) = \ (((uint16_t)(*(rp)) & ~(BIT_MASK16(BITPOS_START(b), BITPOS_END(b)))) \ | ((uint16_t)(v) << BITPOS_END(b)))) #define AMD_IOMMU_REG_SET8(rp, b, v) \ ((*(rp)) = \ (((uint8_t)(*(rp)) & ~(BIT_MASK8(BITPOS_START(b), BITPOS_END(b)))) \ | ((uint8_t)(v) << BITPOS_END(b)))) /* * Cast a 64 bit pointer to a uint64_t * */ #define REGADDR64(a) ((uint64_t *)(uintptr_t)(a)) typedef enum { AMD_IOMMU_INTR_INVALID = 0, AMD_IOMMU_INTR_TABLE, AMD_IOMMU_INTR_ALLOCED, AMD_IOMMU_INTR_HANDLER, AMD_IOMMU_INTR_ENABLED } amd_iommu_intr_state_t; typedef struct amd_iommu { kmutex_t aiomt_mutex; kmutex_t aiomt_eventlock; kmutex_t aiomt_cmdlock; dev_info_t *aiomt_dip; uint16_t aiomt_bdf; int aiomt_idx; iommulib_handle_t aiomt_iommulib_handle; iommulib_ops_t *aiomt_iommulib_ops; uint32_t aiomt_cap_hdr; uint8_t aiomt_npcache; uint8_t aiomt_httun; uint8_t aiomt_iotlb; uint8_t aiomt_captype; uint8_t aiomt_capid; uint32_t aiomt_low_addr32; uint32_t aiomt_hi_addr32; uint64_t aiomt_reg_pa; uint64_t aiomt_va; uint64_t aiomt_reg_va; uint32_t aiomt_range; uint8_t aiomt_rng_bus; uint8_t aiomt_first_devfn; uint8_t aiomt_last_devfn; uint8_t aiomt_rng_valid; uint8_t aiomt_ht_unitid; uint32_t aiomt_misc; uint8_t aiomt_htatsresv; uint8_t aiomt_vasize; uint8_t aiomt_pasize; uint8_t aiomt_msinum; uint8_t aiomt_reg_pages; uint32_t aiomt_reg_size; uint32_t aiomt_devtbl_sz; uint32_t aiomt_cmdbuf_sz; uint32_t aiomt_eventlog_sz; caddr_t aiomt_devtbl; caddr_t aiomt_cmdbuf; caddr_t aiomt_eventlog; uint32_t *aiomt_cmd_tail; uint32_t *aiomt_event_head; ddi_dma_handle_t aiomt_dmahdl; void *aiomt_dma_bufva; uint64_t aiomt_dma_mem_realsz; ddi_acc_handle_t aiomt_dma_mem_hdl; ddi_dma_cookie_t aiomt_buf_dma_cookie; uint_t aiomt_buf_dma_ncookie; amd_iommu_intr_state_t aiomt_intr_state; ddi_intr_handle_t *aiomt_intr_htable; uint32_t aiomt_intr_htable_sz; uint32_t aiomt_actual_intrs; uint32_t aiomt_intr_cap; uint64_t aiomt_reg_devtbl_va; uint64_t aiomt_reg_cmdbuf_va; uint64_t aiomt_reg_eventlog_va; uint64_t aiomt_reg_ctrl_va; uint64_t aiomt_reg_excl_base_va; uint64_t aiomt_reg_excl_lim_va; uint64_t aiomt_reg_cmdbuf_head_va; uint64_t aiomt_reg_cmdbuf_tail_va; uint64_t aiomt_reg_eventlog_head_va; uint64_t aiomt_reg_eventlog_tail_va; uint64_t aiomt_reg_status_va; struct amd_iommu *aiomt_next; } amd_iommu_t; typedef struct amd_iommu_dma_devtbl_ent { uint16_t de_domainid; uint8_t de_R; uint8_t de_W; caddr_t de_root_pgtbl; uint8_t de_pgmode; } amd_iommu_dma_devtbl_entry_t; typedef struct amd_iommu_alias { uint16_t al_bdf; uint16_t al_src_bdf; struct amd_iommu_alias *al_next; } amd_iommu_alias_t; typedef struct amd_iommu_cmdargs { uint64_t ca_addr; uint16_t ca_domainid; uint16_t ca_deviceid; } amd_iommu_cmdargs_t; struct amd_iommu_page_table; typedef struct amd_iommu_page_table_hash { kmutex_t ampt_lock; struct amd_iommu_page_table **ampt_hash; } amd_iommu_page_table_hash_t; typedef enum { AMD_IOMMU_LOG_INVALID_OP = 0, AMD_IOMMU_LOG_DISPLAY, AMD_IOMMU_LOG_DISCARD } amd_iommu_log_op_t; typedef enum { AMD_IOMMU_DEBUG_NONE = 0, AMD_IOMMU_DEBUG_ALLOCHDL = 0x1, AMD_IOMMU_DEBUG_FREEHDL = 0x2, AMD_IOMMU_DEBUG_BIND = 0x4, AMD_IOMMU_DEBUG_UNBIND = 0x8, AMD_IOMMU_DEBUG_WIN = 0x10, AMD_IOMMU_DEBUG_PAGE_TABLES = 0x20, AMD_IOMMU_DEBUG_DEVTBL = 0x40, AMD_IOMMU_DEBUG_CMDBUF = 0x80, AMD_IOMMU_DEBUG_EVENTLOG = 0x100, AMD_IOMMU_DEBUG_ACPI = 0x200, AMD_IOMMU_DEBUG_PA2VA = 0x400, AMD_IOMMU_DEBUG_TABLES = 0x800, AMD_IOMMU_DEBUG_EXCL = 0x1000, AMD_IOMMU_DEBUG_INTR = 0x2000 } amd_iommu_debug_t; extern const char *amd_iommu_modname; extern kmutex_t amd_iommu_global_lock; extern amd_iommu_alias_t **amd_iommu_alias; extern amd_iommu_page_table_hash_t amd_iommu_page_table_hash; extern ddi_device_acc_attr_t amd_iommu_devacc; extern amd_iommu_debug_t amd_iommu_debug; extern uint8_t amd_iommu_htatsresv; extern uint8_t amd_iommu_vasize; extern uint8_t amd_iommu_pasize; extern int amd_iommu_64bit_bug; extern int amd_iommu_unity_map; extern int amd_iommu_no_RW_perms; extern int amd_iommu_no_unmap; extern int amd_iommu_pageva_inval_all; extern int amd_iommu_disable; extern char *amd_iommu_disable_list; extern uint64_t amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits); extern uint64_t amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value); extern dev_info_t *amd_iommu_pci_dip(dev_info_t *rdip, const char *path); int amd_iommu_cmd(amd_iommu_t *iommu, amd_iommu_cmd_t cmd, amd_iommu_cmdargs_t *cmdargs, amd_iommu_cmd_flags_t flags, int lock_held); int amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt); void amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt); int amd_iommu_read_log(amd_iommu_t *iommu, amd_iommu_log_op_t op); void amd_iommu_read_boot_props(void); void amd_iommu_lookup_conf_props(dev_info_t *dip); #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _AMD_IOMMU_IMPL_H */