1b77cf11fSRob Herring /* SPDX-License-Identifier: GPL-2.0 */ 2b77cf11fSRob Herring #ifndef __IO_PGTABLE_H 3b77cf11fSRob Herring #define __IO_PGTABLE_H 4a2d3a382SWill Deacon 5b77cf11fSRob Herring #include <linux/bitops.h> 6a2d3a382SWill Deacon #include <linux/iommu.h> 7b77cf11fSRob Herring 8b77cf11fSRob Herring /* 9b77cf11fSRob Herring * Public API for use by IOMMU drivers 10b77cf11fSRob Herring */ 11b77cf11fSRob Herring enum io_pgtable_fmt { 12b77cf11fSRob Herring ARM_32_LPAE_S1, 13b77cf11fSRob Herring ARM_32_LPAE_S2, 14b77cf11fSRob Herring ARM_64_LPAE_S1, 15b77cf11fSRob Herring ARM_64_LPAE_S2, 16b77cf11fSRob Herring ARM_V7S, 17d08d42deSRob Herring ARM_MALI_LPAE, 18b77cf11fSRob Herring IO_PGTABLE_NUM_FMTS, 19b77cf11fSRob Herring }; 20b77cf11fSRob Herring 21b77cf11fSRob Herring /** 22298f7889SWill Deacon * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management. 23b77cf11fSRob Herring * 24b77cf11fSRob Herring * @tlb_flush_all: Synchronously invalidate the entire TLB context. 253445545bSWill Deacon * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state 263445545bSWill Deacon * (sometimes referred to as the "walk cache") for a virtual 273445545bSWill Deacon * address range. 283445545bSWill Deacon * @tlb_flush_leaf: Synchronously invalidate all leaf TLB state for a virtual 293445545bSWill Deacon * address range. 30abfd6fe0SWill Deacon * @tlb_add_page: Optional callback to queue up leaf TLB invalidation for a 313951c41aSWill Deacon * single page. IOMMUs that cannot batch TLB invalidation 323951c41aSWill Deacon * operations efficiently will typically issue them here, but 333951c41aSWill Deacon * others may decide to update the iommu_iotlb_gather structure 343951c41aSWill Deacon * and defer the invalidation until iommu_tlb_sync() instead. 35b77cf11fSRob Herring * 36b77cf11fSRob Herring * Note that these can all be called in atomic context and must therefore 37b77cf11fSRob Herring * not block. 38b77cf11fSRob Herring */ 39298f7889SWill Deacon struct iommu_flush_ops { 40b77cf11fSRob Herring void (*tlb_flush_all)(void *cookie); 413445545bSWill Deacon void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule, 423445545bSWill Deacon void *cookie); 433445545bSWill Deacon void (*tlb_flush_leaf)(unsigned long iova, size_t size, size_t granule, 443445545bSWill Deacon void *cookie); 453951c41aSWill Deacon void (*tlb_add_page)(struct iommu_iotlb_gather *gather, 463951c41aSWill Deacon unsigned long iova, size_t granule, void *cookie); 47b77cf11fSRob Herring }; 48b77cf11fSRob Herring 49b77cf11fSRob Herring /** 50b77cf11fSRob Herring * struct io_pgtable_cfg - Configuration data for a set of page tables. 51b77cf11fSRob Herring * 52b77cf11fSRob Herring * @quirks: A bitmap of hardware quirks that require some special 53b77cf11fSRob Herring * action by the low-level page table allocator. 54b77cf11fSRob Herring * @pgsize_bitmap: A bitmap of page sizes supported by this set of page 55b77cf11fSRob Herring * tables. 56b77cf11fSRob Herring * @ias: Input address (iova) size, in bits. 57b77cf11fSRob Herring * @oas: Output address (paddr) size, in bits. 584f41845bSWill Deacon * @coherent_walk A flag to indicate whether or not page table walks made 594f41845bSWill Deacon * by the IOMMU are coherent with the CPU caches. 60b77cf11fSRob Herring * @tlb: TLB management callbacks for this set of tables. 61b77cf11fSRob Herring * @iommu_dev: The device representing the DMA configuration for the 62b77cf11fSRob Herring * page table walker. 63b77cf11fSRob Herring */ 64b77cf11fSRob Herring struct io_pgtable_cfg { 65b77cf11fSRob Herring /* 66b77cf11fSRob Herring * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in 67b77cf11fSRob Herring * stage 1 PTEs, for hardware which insists on validating them 68b77cf11fSRob Herring * even in non-secure state where they should normally be ignored. 69b77cf11fSRob Herring * 70b77cf11fSRob Herring * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and 71b77cf11fSRob Herring * IOMMU_NOEXEC flags and map everything with full access, for 72b77cf11fSRob Herring * hardware which does not implement the permissions of a given 73b77cf11fSRob Herring * format, and/or requires some format-specific default value. 74b77cf11fSRob Herring * 75b77cf11fSRob Herring * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid 76b77cf11fSRob Herring * (unmapped) entries but the hardware might do so anyway, perform 77b77cf11fSRob Herring * TLB maintenance when mapping as well as when unmapping. 78b77cf11fSRob Herring * 794c019de6SYong Wu * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend 804c019de6SYong Wu * to support up to 34 bits PA where the bit32 and bit33 are 814c019de6SYong Wu * encoded in the bit9 and bit4 of the PTE respectively. 82b77cf11fSRob Herring * 83b77cf11fSRob Herring * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs 84b77cf11fSRob Herring * on unmap, for DMA domains using the flush queue mechanism for 85b77cf11fSRob Herring * delayed invalidation. 86*db690301SRobin Murphy * 87*db690301SRobin Murphy * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table 88*db690301SRobin Murphy * for use in the upper half of a split address space. 89b77cf11fSRob Herring */ 90b77cf11fSRob Herring #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) 91b77cf11fSRob Herring #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) 92b77cf11fSRob Herring #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) 9373d50811SYong Wu #define IO_PGTABLE_QUIRK_ARM_MTK_EXT BIT(3) 944f41845bSWill Deacon #define IO_PGTABLE_QUIRK_NON_STRICT BIT(4) 95*db690301SRobin Murphy #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) 96b77cf11fSRob Herring unsigned long quirks; 97b77cf11fSRob Herring unsigned long pgsize_bitmap; 98b77cf11fSRob Herring unsigned int ias; 99b77cf11fSRob Herring unsigned int oas; 1004f41845bSWill Deacon bool coherent_walk; 101298f7889SWill Deacon const struct iommu_flush_ops *tlb; 102b77cf11fSRob Herring struct device *iommu_dev; 103b77cf11fSRob Herring 104b77cf11fSRob Herring /* Low-level data specific to the table format */ 105b77cf11fSRob Herring union { 106b77cf11fSRob Herring struct { 107d1e5f26fSRobin Murphy u64 ttbr; 108fb485eb1SRobin Murphy struct { 109fb485eb1SRobin Murphy u32 ips:3; 110fb485eb1SRobin Murphy u32 tg:2; 111fb485eb1SRobin Murphy u32 sh:2; 112fb485eb1SRobin Murphy u32 orgn:2; 113fb485eb1SRobin Murphy u32 irgn:2; 114fb485eb1SRobin Murphy u32 tsz:6; 115fb485eb1SRobin Murphy } tcr; 116205577abSRobin Murphy u64 mair; 117b77cf11fSRob Herring } arm_lpae_s1_cfg; 118b77cf11fSRob Herring 119b77cf11fSRob Herring struct { 120b77cf11fSRob Herring u64 vttbr; 121ac4b80e5SWill Deacon struct { 122ac4b80e5SWill Deacon u32 ps:3; 123ac4b80e5SWill Deacon u32 tg:2; 124ac4b80e5SWill Deacon u32 sh:2; 125ac4b80e5SWill Deacon u32 orgn:2; 126ac4b80e5SWill Deacon u32 irgn:2; 127ac4b80e5SWill Deacon u32 sl:2; 128ac4b80e5SWill Deacon u32 tsz:6; 129ac4b80e5SWill Deacon } vtcr; 130b77cf11fSRob Herring } arm_lpae_s2_cfg; 131b77cf11fSRob Herring 132b77cf11fSRob Herring struct { 133d1e5f26fSRobin Murphy u32 ttbr; 134b77cf11fSRob Herring u32 tcr; 135b77cf11fSRob Herring u32 nmrr; 136b77cf11fSRob Herring u32 prrr; 137b77cf11fSRob Herring } arm_v7s_cfg; 138d08d42deSRob Herring 139d08d42deSRob Herring struct { 140d08d42deSRob Herring u64 transtab; 141d08d42deSRob Herring u64 memattr; 142d08d42deSRob Herring } arm_mali_lpae_cfg; 143b77cf11fSRob Herring }; 144b77cf11fSRob Herring }; 145b77cf11fSRob Herring 146b77cf11fSRob Herring /** 147b77cf11fSRob Herring * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. 148b77cf11fSRob Herring * 149b77cf11fSRob Herring * @map: Map a physically contiguous memory region. 150b77cf11fSRob Herring * @unmap: Unmap a physically contiguous memory region. 151b77cf11fSRob Herring * @iova_to_phys: Translate iova to physical address. 152b77cf11fSRob Herring * 153b77cf11fSRob Herring * These functions map directly onto the iommu_ops member functions with 154b77cf11fSRob Herring * the same names. 155b77cf11fSRob Herring */ 156b77cf11fSRob Herring struct io_pgtable_ops { 157b77cf11fSRob Herring int (*map)(struct io_pgtable_ops *ops, unsigned long iova, 158b77cf11fSRob Herring phys_addr_t paddr, size_t size, int prot); 159b77cf11fSRob Herring size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, 160a2d3a382SWill Deacon size_t size, struct iommu_iotlb_gather *gather); 161b77cf11fSRob Herring phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, 162b77cf11fSRob Herring unsigned long iova); 163b77cf11fSRob Herring }; 164b77cf11fSRob Herring 165b77cf11fSRob Herring /** 166b77cf11fSRob Herring * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. 167b77cf11fSRob Herring * 168b77cf11fSRob Herring * @fmt: The page table format. 169b77cf11fSRob Herring * @cfg: The page table configuration. This will be modified to represent 170b77cf11fSRob Herring * the configuration actually provided by the allocator (e.g. the 171b77cf11fSRob Herring * pgsize_bitmap may be restricted). 172b77cf11fSRob Herring * @cookie: An opaque token provided by the IOMMU driver and passed back to 173b77cf11fSRob Herring * the callback routines in cfg->tlb. 174b77cf11fSRob Herring */ 175b77cf11fSRob Herring struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, 176b77cf11fSRob Herring struct io_pgtable_cfg *cfg, 177b77cf11fSRob Herring void *cookie); 178b77cf11fSRob Herring 179b77cf11fSRob Herring /** 180b77cf11fSRob Herring * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller 181b77cf11fSRob Herring * *must* ensure that the page table is no longer 182b77cf11fSRob Herring * live, but the TLB can be dirty. 183b77cf11fSRob Herring * 184b77cf11fSRob Herring * @ops: The ops returned from alloc_io_pgtable_ops. 185b77cf11fSRob Herring */ 186b77cf11fSRob Herring void free_io_pgtable_ops(struct io_pgtable_ops *ops); 187b77cf11fSRob Herring 188b77cf11fSRob Herring 189b77cf11fSRob Herring /* 190b77cf11fSRob Herring * Internal structures for page table allocator implementations. 191b77cf11fSRob Herring */ 192b77cf11fSRob Herring 193b77cf11fSRob Herring /** 194b77cf11fSRob Herring * struct io_pgtable - Internal structure describing a set of page tables. 195b77cf11fSRob Herring * 196b77cf11fSRob Herring * @fmt: The page table format. 197b77cf11fSRob Herring * @cookie: An opaque token provided by the IOMMU driver and passed back to 198b77cf11fSRob Herring * any callback routines. 199b77cf11fSRob Herring * @cfg: A copy of the page table configuration. 200b77cf11fSRob Herring * @ops: The page table operations in use for this set of page tables. 201b77cf11fSRob Herring */ 202b77cf11fSRob Herring struct io_pgtable { 203b77cf11fSRob Herring enum io_pgtable_fmt fmt; 204b77cf11fSRob Herring void *cookie; 205b77cf11fSRob Herring struct io_pgtable_cfg cfg; 206b77cf11fSRob Herring struct io_pgtable_ops ops; 207b77cf11fSRob Herring }; 208b77cf11fSRob Herring 209b77cf11fSRob Herring #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) 210b77cf11fSRob Herring 211b77cf11fSRob Herring static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) 212b77cf11fSRob Herring { 213b77cf11fSRob Herring iop->cfg.tlb->tlb_flush_all(iop->cookie); 214b77cf11fSRob Herring } 215b77cf11fSRob Herring 21610b7a7d9SWill Deacon static inline void 21710b7a7d9SWill Deacon io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova, 21810b7a7d9SWill Deacon size_t size, size_t granule) 219b77cf11fSRob Herring { 22010b7a7d9SWill Deacon iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie); 221b77cf11fSRob Herring } 222b77cf11fSRob Herring 22310b7a7d9SWill Deacon static inline void 22410b7a7d9SWill Deacon io_pgtable_tlb_flush_leaf(struct io_pgtable *iop, unsigned long iova, 22510b7a7d9SWill Deacon size_t size, size_t granule) 226b77cf11fSRob Herring { 22710b7a7d9SWill Deacon iop->cfg.tlb->tlb_flush_leaf(iova, size, granule, iop->cookie); 22810b7a7d9SWill Deacon } 22910b7a7d9SWill Deacon 230abfd6fe0SWill Deacon static inline void 2313951c41aSWill Deacon io_pgtable_tlb_add_page(struct io_pgtable *iop, 2323951c41aSWill Deacon struct iommu_iotlb_gather * gather, unsigned long iova, 233abfd6fe0SWill Deacon size_t granule) 234b77cf11fSRob Herring { 235abfd6fe0SWill Deacon if (iop->cfg.tlb->tlb_add_page) 2363951c41aSWill Deacon iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); 237b77cf11fSRob Herring } 238b77cf11fSRob Herring 239b77cf11fSRob Herring /** 240b77cf11fSRob Herring * struct io_pgtable_init_fns - Alloc/free a set of page tables for a 241b77cf11fSRob Herring * particular format. 242b77cf11fSRob Herring * 243b77cf11fSRob Herring * @alloc: Allocate a set of page tables described by cfg. 244b77cf11fSRob Herring * @free: Free the page tables associated with iop. 245b77cf11fSRob Herring */ 246b77cf11fSRob Herring struct io_pgtable_init_fns { 247b77cf11fSRob Herring struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); 248b77cf11fSRob Herring void (*free)(struct io_pgtable *iop); 249b77cf11fSRob Herring }; 250b77cf11fSRob Herring 251b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; 252b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; 253b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; 254b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; 255b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; 256d08d42deSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; 257b77cf11fSRob Herring 258b77cf11fSRob Herring #endif /* __IO_PGTABLE_H */ 259