xref: /linux/include/linux/io-pgtable.h (revision db6903010aa5b7691d8234a179f4cf71a119445a)
1b77cf11fSRob Herring /* SPDX-License-Identifier: GPL-2.0 */
2b77cf11fSRob Herring #ifndef __IO_PGTABLE_H
3b77cf11fSRob Herring #define __IO_PGTABLE_H
4a2d3a382SWill Deacon 
5b77cf11fSRob Herring #include <linux/bitops.h>
6a2d3a382SWill Deacon #include <linux/iommu.h>
7b77cf11fSRob Herring 
8b77cf11fSRob Herring /*
9b77cf11fSRob Herring  * Public API for use by IOMMU drivers
10b77cf11fSRob Herring  */
11b77cf11fSRob Herring enum io_pgtable_fmt {
12b77cf11fSRob Herring 	ARM_32_LPAE_S1,
13b77cf11fSRob Herring 	ARM_32_LPAE_S2,
14b77cf11fSRob Herring 	ARM_64_LPAE_S1,
15b77cf11fSRob Herring 	ARM_64_LPAE_S2,
16b77cf11fSRob Herring 	ARM_V7S,
17d08d42deSRob Herring 	ARM_MALI_LPAE,
18b77cf11fSRob Herring 	IO_PGTABLE_NUM_FMTS,
19b77cf11fSRob Herring };
20b77cf11fSRob Herring 
21b77cf11fSRob Herring /**
22298f7889SWill Deacon  * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management.
23b77cf11fSRob Herring  *
24b77cf11fSRob Herring  * @tlb_flush_all:  Synchronously invalidate the entire TLB context.
253445545bSWill Deacon  * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state
263445545bSWill Deacon  *                  (sometimes referred to as the "walk cache") for a virtual
273445545bSWill Deacon  *                  address range.
283445545bSWill Deacon  * @tlb_flush_leaf: Synchronously invalidate all leaf TLB state for a virtual
293445545bSWill Deacon  *                  address range.
30abfd6fe0SWill Deacon  * @tlb_add_page:   Optional callback to queue up leaf TLB invalidation for a
313951c41aSWill Deacon  *                  single page.  IOMMUs that cannot batch TLB invalidation
323951c41aSWill Deacon  *                  operations efficiently will typically issue them here, but
333951c41aSWill Deacon  *                  others may decide to update the iommu_iotlb_gather structure
343951c41aSWill Deacon  *                  and defer the invalidation until iommu_tlb_sync() instead.
35b77cf11fSRob Herring  *
36b77cf11fSRob Herring  * Note that these can all be called in atomic context and must therefore
37b77cf11fSRob Herring  * not block.
38b77cf11fSRob Herring  */
39298f7889SWill Deacon struct iommu_flush_ops {
40b77cf11fSRob Herring 	void (*tlb_flush_all)(void *cookie);
413445545bSWill Deacon 	void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
423445545bSWill Deacon 			       void *cookie);
433445545bSWill Deacon 	void (*tlb_flush_leaf)(unsigned long iova, size_t size, size_t granule,
443445545bSWill Deacon 			       void *cookie);
453951c41aSWill Deacon 	void (*tlb_add_page)(struct iommu_iotlb_gather *gather,
463951c41aSWill Deacon 			     unsigned long iova, size_t granule, void *cookie);
47b77cf11fSRob Herring };
48b77cf11fSRob Herring 
49b77cf11fSRob Herring /**
50b77cf11fSRob Herring  * struct io_pgtable_cfg - Configuration data for a set of page tables.
51b77cf11fSRob Herring  *
52b77cf11fSRob Herring  * @quirks:        A bitmap of hardware quirks that require some special
53b77cf11fSRob Herring  *                 action by the low-level page table allocator.
54b77cf11fSRob Herring  * @pgsize_bitmap: A bitmap of page sizes supported by this set of page
55b77cf11fSRob Herring  *                 tables.
56b77cf11fSRob Herring  * @ias:           Input address (iova) size, in bits.
57b77cf11fSRob Herring  * @oas:           Output address (paddr) size, in bits.
584f41845bSWill Deacon  * @coherent_walk  A flag to indicate whether or not page table walks made
594f41845bSWill Deacon  *                 by the IOMMU are coherent with the CPU caches.
60b77cf11fSRob Herring  * @tlb:           TLB management callbacks for this set of tables.
61b77cf11fSRob Herring  * @iommu_dev:     The device representing the DMA configuration for the
62b77cf11fSRob Herring  *                 page table walker.
63b77cf11fSRob Herring  */
64b77cf11fSRob Herring struct io_pgtable_cfg {
65b77cf11fSRob Herring 	/*
66b77cf11fSRob Herring 	 * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in
67b77cf11fSRob Herring 	 *	stage 1 PTEs, for hardware which insists on validating them
68b77cf11fSRob Herring 	 *	even in	non-secure state where they should normally be ignored.
69b77cf11fSRob Herring 	 *
70b77cf11fSRob Herring 	 * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and
71b77cf11fSRob Herring 	 *	IOMMU_NOEXEC flags and map everything with full access, for
72b77cf11fSRob Herring 	 *	hardware which does not implement the permissions of a given
73b77cf11fSRob Herring 	 *	format, and/or requires some format-specific default value.
74b77cf11fSRob Herring 	 *
75b77cf11fSRob Herring 	 * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid
76b77cf11fSRob Herring 	 *	(unmapped) entries but the hardware might do so anyway, perform
77b77cf11fSRob Herring 	 *	TLB maintenance when mapping as well as when unmapping.
78b77cf11fSRob Herring 	 *
794c019de6SYong Wu 	 * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend
804c019de6SYong Wu 	 *	to support up to 34 bits PA where the bit32 and bit33 are
814c019de6SYong Wu 	 *	encoded in the bit9 and bit4 of the PTE respectively.
82b77cf11fSRob Herring 	 *
83b77cf11fSRob Herring 	 * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
84b77cf11fSRob Herring 	 *	on unmap, for DMA domains using the flush queue mechanism for
85b77cf11fSRob Herring 	 *	delayed invalidation.
86*db690301SRobin Murphy 	 *
87*db690301SRobin Murphy 	 * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
88*db690301SRobin Murphy 	 *	for use in the upper half of a split address space.
89b77cf11fSRob Herring 	 */
90b77cf11fSRob Herring 	#define IO_PGTABLE_QUIRK_ARM_NS		BIT(0)
91b77cf11fSRob Herring 	#define IO_PGTABLE_QUIRK_NO_PERMS	BIT(1)
92b77cf11fSRob Herring 	#define IO_PGTABLE_QUIRK_TLBI_ON_MAP	BIT(2)
9373d50811SYong Wu 	#define IO_PGTABLE_QUIRK_ARM_MTK_EXT	BIT(3)
944f41845bSWill Deacon 	#define IO_PGTABLE_QUIRK_NON_STRICT	BIT(4)
95*db690301SRobin Murphy 	#define IO_PGTABLE_QUIRK_ARM_TTBR1	BIT(5)
96b77cf11fSRob Herring 	unsigned long			quirks;
97b77cf11fSRob Herring 	unsigned long			pgsize_bitmap;
98b77cf11fSRob Herring 	unsigned int			ias;
99b77cf11fSRob Herring 	unsigned int			oas;
1004f41845bSWill Deacon 	bool				coherent_walk;
101298f7889SWill Deacon 	const struct iommu_flush_ops	*tlb;
102b77cf11fSRob Herring 	struct device			*iommu_dev;
103b77cf11fSRob Herring 
104b77cf11fSRob Herring 	/* Low-level data specific to the table format */
105b77cf11fSRob Herring 	union {
106b77cf11fSRob Herring 		struct {
107d1e5f26fSRobin Murphy 			u64	ttbr;
108fb485eb1SRobin Murphy 			struct {
109fb485eb1SRobin Murphy 				u32	ips:3;
110fb485eb1SRobin Murphy 				u32	tg:2;
111fb485eb1SRobin Murphy 				u32	sh:2;
112fb485eb1SRobin Murphy 				u32	orgn:2;
113fb485eb1SRobin Murphy 				u32	irgn:2;
114fb485eb1SRobin Murphy 				u32	tsz:6;
115fb485eb1SRobin Murphy 			}	tcr;
116205577abSRobin Murphy 			u64	mair;
117b77cf11fSRob Herring 		} arm_lpae_s1_cfg;
118b77cf11fSRob Herring 
119b77cf11fSRob Herring 		struct {
120b77cf11fSRob Herring 			u64	vttbr;
121ac4b80e5SWill Deacon 			struct {
122ac4b80e5SWill Deacon 				u32	ps:3;
123ac4b80e5SWill Deacon 				u32	tg:2;
124ac4b80e5SWill Deacon 				u32	sh:2;
125ac4b80e5SWill Deacon 				u32	orgn:2;
126ac4b80e5SWill Deacon 				u32	irgn:2;
127ac4b80e5SWill Deacon 				u32	sl:2;
128ac4b80e5SWill Deacon 				u32	tsz:6;
129ac4b80e5SWill Deacon 			}	vtcr;
130b77cf11fSRob Herring 		} arm_lpae_s2_cfg;
131b77cf11fSRob Herring 
132b77cf11fSRob Herring 		struct {
133d1e5f26fSRobin Murphy 			u32	ttbr;
134b77cf11fSRob Herring 			u32	tcr;
135b77cf11fSRob Herring 			u32	nmrr;
136b77cf11fSRob Herring 			u32	prrr;
137b77cf11fSRob Herring 		} arm_v7s_cfg;
138d08d42deSRob Herring 
139d08d42deSRob Herring 		struct {
140d08d42deSRob Herring 			u64	transtab;
141d08d42deSRob Herring 			u64	memattr;
142d08d42deSRob Herring 		} arm_mali_lpae_cfg;
143b77cf11fSRob Herring 	};
144b77cf11fSRob Herring };
145b77cf11fSRob Herring 
146b77cf11fSRob Herring /**
147b77cf11fSRob Herring  * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
148b77cf11fSRob Herring  *
149b77cf11fSRob Herring  * @map:          Map a physically contiguous memory region.
150b77cf11fSRob Herring  * @unmap:        Unmap a physically contiguous memory region.
151b77cf11fSRob Herring  * @iova_to_phys: Translate iova to physical address.
152b77cf11fSRob Herring  *
153b77cf11fSRob Herring  * These functions map directly onto the iommu_ops member functions with
154b77cf11fSRob Herring  * the same names.
155b77cf11fSRob Herring  */
156b77cf11fSRob Herring struct io_pgtable_ops {
157b77cf11fSRob Herring 	int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
158b77cf11fSRob Herring 		   phys_addr_t paddr, size_t size, int prot);
159b77cf11fSRob Herring 	size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
160a2d3a382SWill Deacon 			size_t size, struct iommu_iotlb_gather *gather);
161b77cf11fSRob Herring 	phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
162b77cf11fSRob Herring 				    unsigned long iova);
163b77cf11fSRob Herring };
164b77cf11fSRob Herring 
165b77cf11fSRob Herring /**
166b77cf11fSRob Herring  * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
167b77cf11fSRob Herring  *
168b77cf11fSRob Herring  * @fmt:    The page table format.
169b77cf11fSRob Herring  * @cfg:    The page table configuration. This will be modified to represent
170b77cf11fSRob Herring  *          the configuration actually provided by the allocator (e.g. the
171b77cf11fSRob Herring  *          pgsize_bitmap may be restricted).
172b77cf11fSRob Herring  * @cookie: An opaque token provided by the IOMMU driver and passed back to
173b77cf11fSRob Herring  *          the callback routines in cfg->tlb.
174b77cf11fSRob Herring  */
175b77cf11fSRob Herring struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
176b77cf11fSRob Herring 					    struct io_pgtable_cfg *cfg,
177b77cf11fSRob Herring 					    void *cookie);
178b77cf11fSRob Herring 
179b77cf11fSRob Herring /**
180b77cf11fSRob Herring  * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller
181b77cf11fSRob Herring  *                         *must* ensure that the page table is no longer
182b77cf11fSRob Herring  *                         live, but the TLB can be dirty.
183b77cf11fSRob Herring  *
184b77cf11fSRob Herring  * @ops: The ops returned from alloc_io_pgtable_ops.
185b77cf11fSRob Herring  */
186b77cf11fSRob Herring void free_io_pgtable_ops(struct io_pgtable_ops *ops);
187b77cf11fSRob Herring 
188b77cf11fSRob Herring 
189b77cf11fSRob Herring /*
190b77cf11fSRob Herring  * Internal structures for page table allocator implementations.
191b77cf11fSRob Herring  */
192b77cf11fSRob Herring 
193b77cf11fSRob Herring /**
194b77cf11fSRob Herring  * struct io_pgtable - Internal structure describing a set of page tables.
195b77cf11fSRob Herring  *
196b77cf11fSRob Herring  * @fmt:    The page table format.
197b77cf11fSRob Herring  * @cookie: An opaque token provided by the IOMMU driver and passed back to
198b77cf11fSRob Herring  *          any callback routines.
199b77cf11fSRob Herring  * @cfg:    A copy of the page table configuration.
200b77cf11fSRob Herring  * @ops:    The page table operations in use for this set of page tables.
201b77cf11fSRob Herring  */
202b77cf11fSRob Herring struct io_pgtable {
203b77cf11fSRob Herring 	enum io_pgtable_fmt	fmt;
204b77cf11fSRob Herring 	void			*cookie;
205b77cf11fSRob Herring 	struct io_pgtable_cfg	cfg;
206b77cf11fSRob Herring 	struct io_pgtable_ops	ops;
207b77cf11fSRob Herring };
208b77cf11fSRob Herring 
209b77cf11fSRob Herring #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
210b77cf11fSRob Herring 
211b77cf11fSRob Herring static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
212b77cf11fSRob Herring {
213b77cf11fSRob Herring 	iop->cfg.tlb->tlb_flush_all(iop->cookie);
214b77cf11fSRob Herring }
215b77cf11fSRob Herring 
21610b7a7d9SWill Deacon static inline void
21710b7a7d9SWill Deacon io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova,
21810b7a7d9SWill Deacon 			  size_t size, size_t granule)
219b77cf11fSRob Herring {
22010b7a7d9SWill Deacon 	iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie);
221b77cf11fSRob Herring }
222b77cf11fSRob Herring 
22310b7a7d9SWill Deacon static inline void
22410b7a7d9SWill Deacon io_pgtable_tlb_flush_leaf(struct io_pgtable *iop, unsigned long iova,
22510b7a7d9SWill Deacon 			  size_t size, size_t granule)
226b77cf11fSRob Herring {
22710b7a7d9SWill Deacon 	iop->cfg.tlb->tlb_flush_leaf(iova, size, granule, iop->cookie);
22810b7a7d9SWill Deacon }
22910b7a7d9SWill Deacon 
230abfd6fe0SWill Deacon static inline void
2313951c41aSWill Deacon io_pgtable_tlb_add_page(struct io_pgtable *iop,
2323951c41aSWill Deacon 			struct iommu_iotlb_gather * gather, unsigned long iova,
233abfd6fe0SWill Deacon 			size_t granule)
234b77cf11fSRob Herring {
235abfd6fe0SWill Deacon 	if (iop->cfg.tlb->tlb_add_page)
2363951c41aSWill Deacon 		iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
237b77cf11fSRob Herring }
238b77cf11fSRob Herring 
239b77cf11fSRob Herring /**
240b77cf11fSRob Herring  * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
241b77cf11fSRob Herring  *                              particular format.
242b77cf11fSRob Herring  *
243b77cf11fSRob Herring  * @alloc: Allocate a set of page tables described by cfg.
244b77cf11fSRob Herring  * @free:  Free the page tables associated with iop.
245b77cf11fSRob Herring  */
246b77cf11fSRob Herring struct io_pgtable_init_fns {
247b77cf11fSRob Herring 	struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
248b77cf11fSRob Herring 	void (*free)(struct io_pgtable *iop);
249b77cf11fSRob Herring };
250b77cf11fSRob Herring 
251b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
252b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
253b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
254b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
255b77cf11fSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
256d08d42deSRob Herring extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
257b77cf11fSRob Herring 
258b77cf11fSRob Herring #endif /* __IO_PGTABLE_H */
259