/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. * */ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H #include #include #include #include #include "iommufd_private.h" struct iommu_domain; /* * Each io_pagetable is composed of intervals of areas which cover regions of * the iova that are backed by something. iova not covered by areas is not * populated in the page table. Each area is fully populated with pages. * * iovas are in byte units, but must be iopt->iova_alignment aligned. * * pages can be NULL, this means some other thread is still working on setting * up or tearing down the area. When observed under the write side of the * domain_rwsem a NULL pages must mean the area is still being setup and no * domains are filled. * * storage_domain points at an arbitrary iommu_domain that is holding the PFNs * for this area. It is locked by the pages->mutex. This simplifies the locking * as the pages code can rely on the storage_domain without having to get the * iopt->domains_rwsem. * * The io_pagetable::iova_rwsem protects node * The iopt_pages::mutex protects pages_node * iopt and iommu_prot are immutable * The pages::mutex protects num_accesses */ struct iopt_area { struct interval_tree_node node; struct interval_tree_node pages_node; struct io_pagetable *iopt; struct iopt_pages *pages; struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; unsigned int num_accesses; }; struct iopt_allowed { struct interval_tree_node node; }; struct iopt_reserved { struct interval_tree_node node; void *owner; }; int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages); void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages); int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain); void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain); void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain); static inline unsigned long iopt_area_index(struct iopt_area *area) { return area->pages_node.start; } static inline unsigned long iopt_area_last_index(struct iopt_area *area) { return area->pages_node.last; } static inline unsigned long iopt_area_iova(struct iopt_area *area) { return area->node.start; } static inline unsigned long iopt_area_last_iova(struct iopt_area *area) { return area->node.last; } static inline size_t iopt_area_length(struct iopt_area *area) { return (area->node.last - area->node.start) + 1; } /* * Number of bytes from the start of the iopt_pages that the iova begins. * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) { if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + iopt_area_index(area) * PAGE_SIZE; } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { return iopt_area_start_byte(area, iova) / PAGE_SIZE; } #define __make_iopt_iter(name) \ static inline struct iopt_##name *iopt_##name##_iter_first( \ struct io_pagetable *iopt, unsigned long start, \ unsigned long last) \ { \ struct interval_tree_node *node; \ \ lockdep_assert_held(&iopt->iova_rwsem); \ node = interval_tree_iter_first(&iopt->name##_itree, start, \ last); \ if (!node) \ return NULL; \ return container_of(node, struct iopt_##name, node); \ } \ static inline struct iopt_##name *iopt_##name##_iter_next( \ struct iopt_##name *last_node, unsigned long start, \ unsigned long last) \ { \ struct interval_tree_node *node; \ \ node = interval_tree_iter_next(&last_node->node, start, last); \ if (!node) \ return NULL; \ return container_of(node, struct iopt_##name, node); \ } __make_iopt_iter(area) __make_iopt_iter(allowed) __make_iopt_iter(reserved) struct iopt_area_contig_iter { unsigned long cur_iova; unsigned long last_iova; struct iopt_area *area; }; struct iopt_area *iopt_area_contig_init(struct iopt_area_contig_iter *iter, struct io_pagetable *iopt, unsigned long iova, unsigned long last_iova); struct iopt_area *iopt_area_contig_next(struct iopt_area_contig_iter *iter); static inline bool iopt_area_contig_done(struct iopt_area_contig_iter *iter) { return iter->area && iter->last_iova <= iopt_area_last_iova(iter->area); } /* * Iterate over a contiguous list of areas that span the iova,last_iova range. * The caller must check iopt_area_contig_done() after the loop to see if * contiguous areas existed. */ #define iopt_for_each_contig_area(iter, area, iopt, iova, last_iova) \ for (area = iopt_area_contig_init(iter, iopt, iova, last_iova); area; \ area = iopt_area_contig_next(iter)) enum { IOPT_PAGES_ACCOUNT_NONE = 0, IOPT_PAGES_ACCOUNT_USER = 1, IOPT_PAGES_ACCOUNT_MM = 2, IOPT_PAGES_ACCOUNT_MODE_NUM = 3, }; enum iopt_address_type { IOPT_ADDRESS_USER = 0, IOPT_ADDRESS_FILE = 1, }; /* * This holds a pinned page list for multiple areas of IO address space. The * pages always originate from a linear chunk of userspace VA. Multiple * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * * indexes in this structure are measured in PAGE_SIZE units, are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. */ struct iopt_pages { struct kref kref; struct mutex mutex; size_t npages; size_t npinned; size_t last_npinned; struct task_struct *source_task; struct mm_struct *source_mm; struct user_struct *source_user; enum iopt_address_type type; union { void __user *uptr; /* IOPT_ADDRESS_USER */ struct { /* IOPT_ADDRESS_FILE */ struct file *file; unsigned long start; }; }; bool writable:1; u8 account_mode; struct xarray pinned_pfns; /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, unsigned long length, bool writable); struct iopt_pages *iopt_alloc_file_pages(struct file *file, unsigned long start, unsigned long length, bool writable); void iopt_release_pages(struct kref *kref); static inline void iopt_put_pages(struct iopt_pages *pages) { kref_put(&pages->kref, iopt_release_pages); } void iopt_pages_fill_from_xarray(struct iopt_pages *pages, unsigned long start, unsigned long last, struct page **out_pages); int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start, unsigned long last, struct page **out_pages); void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, unsigned long last); int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, void *data, unsigned long length, unsigned int flags); /* * Each interval represents an active iopt_access_pages(), it acts as an * interval lock that keeps the PFNs pinned and stored in the xarray. */ struct iopt_pages_access { struct interval_tree_node node; unsigned int users; }; struct pfn_reader_user; int iopt_pages_update_pinned(struct iopt_pages *pages, unsigned long npages, bool inc, struct pfn_reader_user *user); #endif