1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2024, Google LLC. 4 * Pasha Tatashin <pasha.tatashin@soleen.com> 5 */ 6 7 #ifndef __IOMMU_PAGES_H 8 #define __IOMMU_PAGES_H 9 10 #include <linux/vmstat.h> 11 #include <linux/gfp.h> 12 #include <linux/mm.h> 13 14 /* 15 * All page allocations that should be reported to as "iommu-pagetables" to 16 * userspace must use one of the functions below. This includes allocations of 17 * page-tables and other per-iommu_domain configuration structures. 18 * 19 * This is necessary for the proper accounting as IOMMU state can be rather 20 * large, i.e. multiple gigabytes in size. 21 */ 22 23 /** 24 * __iommu_alloc_account - account for newly allocated page. 25 * @page: head struct page of the page. 26 * @order: order of the page 27 */ __iommu_alloc_account(struct page * page,int order)28 static inline void __iommu_alloc_account(struct page *page, int order) 29 { 30 const long pgcnt = 1l << order; 31 32 mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); 33 mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); 34 } 35 36 /** 37 * __iommu_free_account - account a page that is about to be freed. 38 * @page: head struct page of the page. 39 * @order: order of the page 40 */ __iommu_free_account(struct page * page,int order)41 static inline void __iommu_free_account(struct page *page, int order) 42 { 43 const long pgcnt = 1l << order; 44 45 mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); 46 mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); 47 } 48 49 /** 50 * __iommu_alloc_pages - allocate a zeroed page of a given order. 51 * @gfp: buddy allocator flags 52 * @order: page order 53 * 54 * returns the head struct page of the allocated page. 55 */ __iommu_alloc_pages(gfp_t gfp,int order)56 static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order) 57 { 58 struct page *page; 59 60 page = alloc_pages(gfp | __GFP_ZERO, order); 61 if (unlikely(!page)) 62 return NULL; 63 64 __iommu_alloc_account(page, order); 65 66 return page; 67 } 68 69 /** 70 * __iommu_free_pages - free page of a given order 71 * @page: head struct page of the page 72 * @order: page order 73 */ __iommu_free_pages(struct page * page,int order)74 static inline void __iommu_free_pages(struct page *page, int order) 75 { 76 if (!page) 77 return; 78 79 __iommu_free_account(page, order); 80 __free_pages(page, order); 81 } 82 83 /** 84 * iommu_alloc_pages_node - allocate a zeroed page of a given order from 85 * specific NUMA node. 86 * @nid: memory NUMA node id 87 * @gfp: buddy allocator flags 88 * @order: page order 89 * 90 * returns the virtual address of the allocated page 91 */ iommu_alloc_pages_node(int nid,gfp_t gfp,int order)92 static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) 93 { 94 struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order); 95 96 if (unlikely(!page)) 97 return NULL; 98 99 __iommu_alloc_account(page, order); 100 101 return page_address(page); 102 } 103 104 /** 105 * iommu_alloc_pages - allocate a zeroed page of a given order 106 * @gfp: buddy allocator flags 107 * @order: page order 108 * 109 * returns the virtual address of the allocated page 110 */ iommu_alloc_pages(gfp_t gfp,int order)111 static inline void *iommu_alloc_pages(gfp_t gfp, int order) 112 { 113 struct page *page = __iommu_alloc_pages(gfp, order); 114 115 if (unlikely(!page)) 116 return NULL; 117 118 return page_address(page); 119 } 120 121 /** 122 * iommu_alloc_page_node - allocate a zeroed page at specific NUMA node. 123 * @nid: memory NUMA node id 124 * @gfp: buddy allocator flags 125 * 126 * returns the virtual address of the allocated page 127 */ iommu_alloc_page_node(int nid,gfp_t gfp)128 static inline void *iommu_alloc_page_node(int nid, gfp_t gfp) 129 { 130 return iommu_alloc_pages_node(nid, gfp, 0); 131 } 132 133 /** 134 * iommu_alloc_page - allocate a zeroed page 135 * @gfp: buddy allocator flags 136 * 137 * returns the virtual address of the allocated page 138 */ iommu_alloc_page(gfp_t gfp)139 static inline void *iommu_alloc_page(gfp_t gfp) 140 { 141 return iommu_alloc_pages(gfp, 0); 142 } 143 144 /** 145 * iommu_free_pages - free page of a given order 146 * @virt: virtual address of the page to be freed. 147 * @order: page order 148 */ iommu_free_pages(void * virt,int order)149 static inline void iommu_free_pages(void *virt, int order) 150 { 151 if (!virt) 152 return; 153 154 __iommu_free_pages(virt_to_page(virt), order); 155 } 156 157 /** 158 * iommu_free_page - free page 159 * @virt: virtual address of the page to be freed. 160 */ iommu_free_page(void * virt)161 static inline void iommu_free_page(void *virt) 162 { 163 iommu_free_pages(virt, 0); 164 } 165 166 /** 167 * iommu_put_pages_list - free a list of pages. 168 * @page: the head of the lru list to be freed. 169 * 170 * There are no locking requirement for these pages, as they are going to be 171 * put on a free list as soon as refcount reaches 0. Pages are put on this LRU 172 * list once they are removed from the IOMMU page tables. However, they can 173 * still be access through debugfs. 174 */ iommu_put_pages_list(struct list_head * page)175 static inline void iommu_put_pages_list(struct list_head *page) 176 { 177 while (!list_empty(page)) { 178 struct page *p = list_entry(page->prev, struct page, lru); 179 180 list_del(&p->lru); 181 __iommu_free_account(p, 0); 182 put_page(p); 183 } 184 } 185 186 #endif /* __IOMMU_PAGES_H */ 187