1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 François Tigeot 7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * $FreeBSD$ 32 */ 33 #ifndef _LINUXKPI_LINUX_MM_H_ 34 #define _LINUXKPI_LINUX_MM_H_ 35 36 #include <linux/spinlock.h> 37 #include <linux/gfp.h> 38 #include <linux/kernel.h> 39 #include <linux/mm_types.h> 40 #include <linux/pfn.h> 41 #include <linux/list.h> 42 #include <linux/mmap_lock.h> 43 #include <linux/shrinker.h> 44 45 #include <asm/pgtable.h> 46 47 #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) 48 49 /* 50 * Make sure our LinuxKPI defined virtual memory flags don't conflict 51 * with the ones defined by FreeBSD: 52 */ 53 CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0); 54 55 #define VM_READ VM_PROT_READ 56 #define VM_WRITE VM_PROT_WRITE 57 #define VM_EXEC VM_PROT_EXECUTE 58 59 #define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */ 60 #define VM_MIXEDMAP (1 << 9) 61 #define VM_NORESERVE (1 << 10) 62 #define VM_PFNMAP (1 << 11) 63 #define VM_IO (1 << 12) 64 #define VM_MAYWRITE (1 << 13) 65 #define VM_DONTCOPY (1 << 14) 66 #define VM_DONTEXPAND (1 << 15) 67 #define VM_DONTDUMP (1 << 16) 68 #define VM_SHARED (1 << 17) 69 70 #define VMA_MAX_PREFAULT_RECORD 1 71 72 #define FOLL_WRITE (1 << 0) 73 #define FOLL_FORCE (1 << 1) 74 75 #define VM_FAULT_OOM (1 << 0) 76 #define VM_FAULT_SIGBUS (1 << 1) 77 #define VM_FAULT_MAJOR (1 << 2) 78 #define VM_FAULT_WRITE (1 << 3) 79 #define VM_FAULT_HWPOISON (1 << 4) 80 #define VM_FAULT_HWPOISON_LARGE (1 << 5) 81 #define VM_FAULT_SIGSEGV (1 << 6) 82 #define VM_FAULT_NOPAGE (1 << 7) 83 #define VM_FAULT_LOCKED (1 << 8) 84 #define VM_FAULT_RETRY (1 << 9) 85 #define VM_FAULT_FALLBACK (1 << 10) 86 87 #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \ 88 VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) 89 90 #define FAULT_FLAG_WRITE (1 << 0) 91 #define FAULT_FLAG_MKWRITE (1 << 1) 92 #define FAULT_FLAG_ALLOW_RETRY (1 << 2) 93 #define FAULT_FLAG_RETRY_NOWAIT (1 << 3) 94 #define FAULT_FLAG_KILLABLE (1 << 4) 95 #define FAULT_FLAG_TRIED (1 << 5) 96 #define FAULT_FLAG_USER (1 << 6) 97 #define FAULT_FLAG_REMOTE (1 << 7) 98 #define FAULT_FLAG_INSTRUCTION (1 << 8) 99 100 #define fault_flag_allow_retry_first(flags) \ 101 (((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY) 102 103 typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data); 104 105 struct vm_area_struct { 106 vm_offset_t vm_start; 107 vm_offset_t vm_end; 108 vm_offset_t vm_pgoff; 109 pgprot_t vm_page_prot; 110 unsigned long vm_flags; 111 struct mm_struct *vm_mm; 112 void *vm_private_data; 113 const struct vm_operations_struct *vm_ops; 114 struct linux_file *vm_file; 115 116 /* internal operation */ 117 vm_paddr_t vm_pfn; /* PFN for memory map */ 118 vm_size_t vm_len; /* length for memory map */ 119 vm_pindex_t vm_pfn_first; 120 int vm_pfn_count; 121 int *vm_pfn_pcount; 122 vm_object_t vm_obj; 123 vm_map_t vm_cached_map; 124 TAILQ_ENTRY(vm_area_struct) vm_entry; 125 }; 126 127 struct vm_fault { 128 unsigned int flags; 129 pgoff_t pgoff; 130 union { 131 /* user-space address */ 132 void *virtual_address; /* < 4.11 */ 133 unsigned long address; /* >= 4.11 */ 134 }; 135 struct page *page; 136 struct vm_area_struct *vma; 137 }; 138 139 struct vm_operations_struct { 140 void (*open) (struct vm_area_struct *); 141 void (*close) (struct vm_area_struct *); 142 int (*fault) (struct vm_fault *); 143 int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); 144 }; 145 146 struct sysinfo { 147 uint64_t totalram; /* Total usable main memory size */ 148 uint64_t freeram; /* Available memory size */ 149 uint64_t totalhigh; /* Total high memory size */ 150 uint64_t freehigh; /* Available high memory size */ 151 uint32_t mem_unit; /* Memory unit size in bytes */ 152 }; 153 154 /* 155 * Compute log2 of the power of two rounded up count of pages 156 * needed for size bytes. 157 */ 158 static inline int 159 get_order(unsigned long size) 160 { 161 int order; 162 163 size = (size - 1) >> PAGE_SHIFT; 164 order = 0; 165 while (size) { 166 order++; 167 size >>= 1; 168 } 169 return (order); 170 } 171 172 static inline void * 173 lowmem_page_address(struct page *page) 174 { 175 return (page_address(page)); 176 } 177 178 /* 179 * This only works via memory map operations. 180 */ 181 static inline int 182 io_remap_pfn_range(struct vm_area_struct *vma, 183 unsigned long addr, unsigned long pfn, unsigned long size, 184 vm_memattr_t prot) 185 { 186 vma->vm_page_prot = prot; 187 vma->vm_pfn = pfn; 188 vma->vm_len = size; 189 190 return (0); 191 } 192 193 vm_fault_t 194 lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr, 195 unsigned long pfn, pgprot_t prot); 196 197 static inline vm_fault_t 198 vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 199 unsigned long pfn, pgprot_t prot) 200 { 201 vm_fault_t ret; 202 203 VM_OBJECT_WLOCK(vma->vm_obj); 204 ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot); 205 VM_OBJECT_WUNLOCK(vma->vm_obj); 206 207 return (ret); 208 } 209 #define vmf_insert_pfn_prot(...) \ 210 _Static_assert(false, \ 211 "This function is always called in a loop. Consider using the locked version") 212 213 static inline int 214 apply_to_page_range(struct mm_struct *mm, unsigned long address, 215 unsigned long size, pte_fn_t fn, void *data) 216 { 217 return (-ENOTSUP); 218 } 219 220 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 221 unsigned long size); 222 223 int lkpi_remap_pfn_range(struct vm_area_struct *vma, 224 unsigned long start_addr, unsigned long start_pfn, unsigned long size, 225 pgprot_t prot); 226 227 static inline int 228 remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 229 unsigned long pfn, unsigned long size, pgprot_t prot) 230 { 231 return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot)); 232 } 233 234 static inline unsigned long 235 vma_pages(struct vm_area_struct *vma) 236 { 237 return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); 238 } 239 240 #define offset_in_page(off) ((unsigned long)(off) & (PAGE_SIZE - 1)) 241 242 static inline void 243 set_page_dirty(struct vm_page *page) 244 { 245 vm_page_dirty(page); 246 } 247 248 static inline void 249 mark_page_accessed(struct vm_page *page) 250 { 251 vm_page_reference(page); 252 } 253 254 static inline void 255 get_page(struct vm_page *page) 256 { 257 vm_page_wire(page); 258 } 259 260 extern long 261 get_user_pages(unsigned long start, unsigned long nr_pages, 262 unsigned int gup_flags, struct page **, 263 struct vm_area_struct **); 264 265 static inline long 266 pin_user_pages(unsigned long start, unsigned long nr_pages, 267 unsigned int gup_flags, struct page **pages, 268 struct vm_area_struct **vmas) 269 { 270 return get_user_pages(start, nr_pages, gup_flags, pages, vmas); 271 } 272 273 extern int 274 __get_user_pages_fast(unsigned long start, int nr_pages, int write, 275 struct page **); 276 277 static inline int 278 pin_user_pages_fast(unsigned long start, int nr_pages, 279 unsigned int gup_flags, struct page **pages) 280 { 281 return __get_user_pages_fast( 282 start, nr_pages, !!(gup_flags & FOLL_WRITE), pages); 283 } 284 285 extern long 286 get_user_pages_remote(struct task_struct *, struct mm_struct *, 287 unsigned long start, unsigned long nr_pages, 288 unsigned int gup_flags, struct page **, 289 struct vm_area_struct **); 290 291 static inline long 292 pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm, 293 unsigned long start, unsigned long nr_pages, 294 unsigned int gup_flags, struct page **pages, 295 struct vm_area_struct **vmas) 296 { 297 return get_user_pages_remote( 298 task, mm, start, nr_pages, gup_flags, pages, vmas); 299 } 300 301 static inline void 302 put_page(struct vm_page *page) 303 { 304 vm_page_unwire(page, PQ_ACTIVE); 305 } 306 307 #define unpin_user_page(page) put_page(page) 308 #define unpin_user_pages(pages, npages) release_pages(pages, npages) 309 310 #define copy_highpage(to, from) pmap_copy_page(from, to) 311 312 static inline pgprot_t 313 vm_get_page_prot(unsigned long vm_flags) 314 { 315 return (vm_flags & VM_PROT_ALL); 316 } 317 318 static inline vm_page_t 319 vmalloc_to_page(const void *addr) 320 { 321 vm_paddr_t paddr; 322 323 paddr = pmap_kextract((vm_offset_t)addr); 324 return (PHYS_TO_VM_PAGE(paddr)); 325 } 326 327 static inline int 328 trylock_page(struct page *page) 329 { 330 return (vm_page_trylock(page)); 331 } 332 333 static inline void 334 unlock_page(struct page *page) 335 { 336 337 vm_page_unlock(page); 338 } 339 340 extern int is_vmalloc_addr(const void *addr); 341 void si_meminfo(struct sysinfo *si); 342 343 #define unmap_mapping_range(...) lkpi_unmap_mapping_range(__VA_ARGS__) 344 void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused, 345 loff_t const holelen, int even_cows __unused); 346 347 #define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE) 348 349 void vma_set_file(struct vm_area_struct *vma, struct linux_file *file); 350 351 static inline void 352 might_alloc(gfp_t gfp_mask __unused) 353 { 354 } 355 356 #define is_cow_mapping(flags) (false) 357 358 #endif /* _LINUXKPI_LINUX_MM_H_ */ 359