1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2024, Google LLC. 4 * Pasha Tatashin <pasha.tatashin@soleen.com> 5 */ 6 7 #ifndef __IOMMU_PAGES_H 8 #define __IOMMU_PAGES_H 9 10 #include <linux/iommu.h> 11 12 /** 13 * struct ioptdesc - Memory descriptor for IOMMU page tables 14 * @iopt_freelist_elm: List element for a struct iommu_pages_list 15 * 16 * This struct overlays struct page for now. Do not modify without a good 17 * understanding of the issues. 18 */ 19 struct ioptdesc { 20 unsigned long __page_flags; 21 22 struct list_head iopt_freelist_elm; 23 unsigned long __page_mapping; 24 union { 25 u8 incoherent; 26 pgoff_t __index; 27 }; 28 void *_private; 29 30 unsigned int __page_type; 31 atomic_t __page_refcount; 32 #ifdef CONFIG_MEMCG 33 unsigned long memcg_data; 34 #endif 35 }; 36 37 static inline struct ioptdesc *folio_ioptdesc(struct folio *folio) 38 { 39 return (struct ioptdesc *)folio; 40 } 41 42 static inline struct folio *ioptdesc_folio(struct ioptdesc *iopt) 43 { 44 return (struct folio *)iopt; 45 } 46 47 static inline struct ioptdesc *virt_to_ioptdesc(void *virt) 48 { 49 return folio_ioptdesc(virt_to_folio(virt)); 50 } 51 52 void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size); 53 void iommu_free_pages(void *virt); 54 void iommu_put_pages_list(struct iommu_pages_list *list); 55 56 /** 57 * iommu_pages_list_add - add the page to a iommu_pages_list 58 * @list: List to add the page to 59 * @virt: Address returned from iommu_alloc_pages_node_sz() 60 */ 61 static inline void iommu_pages_list_add(struct iommu_pages_list *list, 62 void *virt) 63 { 64 list_add_tail(&virt_to_ioptdesc(virt)->iopt_freelist_elm, &list->pages); 65 } 66 67 /** 68 * iommu_pages_list_splice - Put all the pages in list from into list to 69 * @from: Source list of pages 70 * @to: Destination list of pages 71 * 72 * from must be re-initialized after calling this function if it is to be 73 * used again. 74 */ 75 static inline void iommu_pages_list_splice(struct iommu_pages_list *from, 76 struct iommu_pages_list *to) 77 { 78 list_splice(&from->pages, &to->pages); 79 } 80 81 /** 82 * iommu_pages_list_empty - True if the list is empty 83 * @list: List to check 84 */ 85 static inline bool iommu_pages_list_empty(struct iommu_pages_list *list) 86 { 87 return list_empty(&list->pages); 88 } 89 90 /** 91 * iommu_alloc_pages_sz - Allocate a zeroed page of a given size from 92 * specific NUMA node 93 * @nid: memory NUMA node id 94 * @gfp: buddy allocator flags 95 * @size: Memory size to allocate, this is rounded up to a power of 2 96 * 97 * Returns the virtual address of the allocated page. 98 */ 99 static inline void *iommu_alloc_pages_sz(gfp_t gfp, size_t size) 100 { 101 return iommu_alloc_pages_node_sz(NUMA_NO_NODE, gfp, size); 102 } 103 104 int iommu_pages_start_incoherent(void *virt, struct device *dma_dev); 105 int iommu_pages_start_incoherent_list(struct iommu_pages_list *list, 106 struct device *dma_dev); 107 108 #ifdef CONFIG_X86 109 #define IOMMU_PAGES_USE_DMA_API 0 110 #include <linux/cacheflush.h> 111 112 static inline void iommu_pages_flush_incoherent(struct device *dma_dev, 113 void *virt, size_t offset, 114 size_t len) 115 { 116 clflush_cache_range(virt + offset, len); 117 } 118 static inline void 119 iommu_pages_stop_incoherent_list(struct iommu_pages_list *list, 120 struct device *dma_dev) 121 { 122 /* 123 * For performance leave the incoherent flag alone which turns this into 124 * a NOP. For X86 the rest of the stop/free flow ignores the flag. 125 */ 126 } 127 static inline void iommu_pages_free_incoherent(void *virt, 128 struct device *dma_dev) 129 { 130 iommu_free_pages(virt); 131 } 132 #else 133 #define IOMMU_PAGES_USE_DMA_API 1 134 #include <linux/dma-mapping.h> 135 136 static inline void iommu_pages_flush_incoherent(struct device *dma_dev, 137 void *virt, size_t offset, 138 size_t len) 139 { 140 dma_sync_single_for_device(dma_dev, (uintptr_t)virt + offset, len, 141 DMA_TO_DEVICE); 142 } 143 void iommu_pages_stop_incoherent_list(struct iommu_pages_list *list, 144 struct device *dma_dev); 145 void iommu_pages_free_incoherent(void *virt, struct device *dma_dev); 146 #endif 147 148 #endif /* __IOMMU_PAGES_H */ 149