1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 François Tigeot 7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #ifndef _LINUXKPI_LINUX_MM_H_ 32 #define _LINUXKPI_LINUX_MM_H_ 33 34 #include <linux/spinlock.h> 35 #include <linux/gfp.h> 36 #include <linux/kernel.h> 37 #include <linux/mm_types.h> 38 #include <linux/mmzone.h> 39 #include <linux/pfn.h> 40 #include <linux/list.h> 41 #include <linux/mmap_lock.h> 42 #include <linux/overflow.h> 43 #include <linux/shrinker.h> 44 #include <linux/page.h> 45 46 #include <asm/pgtable.h> 47 48 #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) 49 50 /* 51 * Make sure our LinuxKPI defined virtual memory flags don't conflict 52 * with the ones defined by FreeBSD: 53 */ 54 CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0); 55 56 #define VM_READ VM_PROT_READ 57 #define VM_WRITE VM_PROT_WRITE 58 #define VM_EXEC VM_PROT_EXECUTE 59 60 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 61 62 #define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */ 63 #define VM_MIXEDMAP (1 << 9) 64 #define VM_NORESERVE (1 << 10) 65 #define VM_PFNMAP (1 << 11) 66 #define VM_IO (1 << 12) 67 #define VM_MAYWRITE (1 << 13) 68 #define VM_DONTCOPY (1 << 14) 69 #define VM_DONTEXPAND (1 << 15) 70 #define VM_DONTDUMP (1 << 16) 71 #define VM_SHARED (1 << 17) 72 73 #define VMA_MAX_PREFAULT_RECORD 1 74 75 #define FOLL_WRITE (1 << 0) 76 #define FOLL_FORCE (1 << 1) 77 78 #define VM_FAULT_OOM (1 << 0) 79 #define VM_FAULT_SIGBUS (1 << 1) 80 #define VM_FAULT_MAJOR (1 << 2) 81 #define VM_FAULT_WRITE (1 << 3) 82 #define VM_FAULT_HWPOISON (1 << 4) 83 #define VM_FAULT_HWPOISON_LARGE (1 << 5) 84 #define VM_FAULT_SIGSEGV (1 << 6) 85 #define VM_FAULT_NOPAGE (1 << 7) 86 #define VM_FAULT_LOCKED (1 << 8) 87 #define VM_FAULT_RETRY (1 << 9) 88 #define VM_FAULT_FALLBACK (1 << 10) 89 90 #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \ 91 VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) 92 93 #define FAULT_FLAG_WRITE (1 << 0) 94 #define FAULT_FLAG_MKWRITE (1 << 1) 95 #define FAULT_FLAG_ALLOW_RETRY (1 << 2) 96 #define FAULT_FLAG_RETRY_NOWAIT (1 << 3) 97 #define FAULT_FLAG_KILLABLE (1 << 4) 98 #define FAULT_FLAG_TRIED (1 << 5) 99 #define FAULT_FLAG_USER (1 << 6) 100 #define FAULT_FLAG_REMOTE (1 << 7) 101 #define FAULT_FLAG_INSTRUCTION (1 << 8) 102 103 #define fault_flag_allow_retry_first(flags) \ 104 (((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY) 105 106 typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data); 107 108 struct vm_area_struct { 109 vm_offset_t vm_start; 110 vm_offset_t vm_end; 111 vm_offset_t vm_pgoff; 112 pgprot_t vm_page_prot; 113 unsigned long vm_flags; 114 struct mm_struct *vm_mm; 115 void *vm_private_data; 116 const struct vm_operations_struct *vm_ops; 117 struct linux_file *vm_file; 118 119 /* internal operation */ 120 vm_paddr_t vm_pfn; /* PFN for memory map */ 121 vm_size_t vm_len; /* length for memory map */ 122 vm_pindex_t vm_pfn_first; 123 int vm_pfn_count; 124 int *vm_pfn_pcount; 125 vm_object_t vm_obj; 126 vm_map_t vm_cached_map; 127 TAILQ_ENTRY(vm_area_struct) vm_entry; 128 }; 129 130 struct vm_fault { 131 unsigned int flags; 132 pgoff_t pgoff; 133 union { 134 /* user-space address */ 135 void *virtual_address; /* < 4.11 */ 136 unsigned long address; /* >= 4.11 */ 137 }; 138 struct page *page; 139 struct vm_area_struct *vma; 140 }; 141 142 struct vm_operations_struct { 143 void (*open) (struct vm_area_struct *); 144 void (*close) (struct vm_area_struct *); 145 int (*fault) (struct vm_fault *); 146 int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); 147 }; 148 149 struct sysinfo { 150 uint64_t totalram; /* Total usable main memory size */ 151 uint64_t freeram; /* Available memory size */ 152 uint64_t totalhigh; /* Total high memory size */ 153 uint64_t freehigh; /* Available high memory size */ 154 uint32_t mem_unit; /* Memory unit size in bytes */ 155 }; 156 157 static inline struct page * 158 virt_to_head_page(const void *p) 159 { 160 161 return (virt_to_page(p)); 162 } 163 164 /* 165 * Compute log2 of the power of two rounded up count of pages 166 * needed for size bytes. 167 */ 168 static inline int 169 get_order(unsigned long size) 170 { 171 int order; 172 173 size = (size - 1) >> PAGE_SHIFT; 174 order = 0; 175 while (size) { 176 order++; 177 size >>= 1; 178 } 179 return (order); 180 } 181 182 /* 183 * Resolve a page into a virtual address: 184 * 185 * NOTE: This function only works for pages allocated by the kernel. 186 */ 187 void *linux_page_address(struct page *); 188 #define page_address(page) linux_page_address(page) 189 190 static inline void * 191 lowmem_page_address(struct page *page) 192 { 193 return (page_address(page)); 194 } 195 196 /* 197 * This only works via memory map operations. 198 */ 199 static inline int 200 io_remap_pfn_range(struct vm_area_struct *vma, 201 unsigned long addr, unsigned long pfn, unsigned long size, 202 vm_memattr_t prot) 203 { 204 vma->vm_page_prot = prot; 205 vma->vm_pfn = pfn; 206 vma->vm_len = size; 207 208 return (0); 209 } 210 211 vm_fault_t 212 lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr, 213 unsigned long pfn, pgprot_t prot); 214 215 static inline vm_fault_t 216 vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 217 unsigned long pfn, pgprot_t prot) 218 { 219 vm_fault_t ret; 220 221 VM_OBJECT_WLOCK(vma->vm_obj); 222 ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot); 223 VM_OBJECT_WUNLOCK(vma->vm_obj); 224 225 return (ret); 226 } 227 #define vmf_insert_pfn_prot(...) \ 228 _Static_assert(false, \ 229 "This function is always called in a loop. Consider using the locked version") 230 231 static inline int 232 apply_to_page_range(struct mm_struct *mm, unsigned long address, 233 unsigned long size, pte_fn_t fn, void *data) 234 { 235 return (-ENOTSUP); 236 } 237 238 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 239 unsigned long size); 240 241 int lkpi_remap_pfn_range(struct vm_area_struct *vma, 242 unsigned long start_addr, unsigned long start_pfn, unsigned long size, 243 pgprot_t prot); 244 245 static inline int 246 remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 247 unsigned long pfn, unsigned long size, pgprot_t prot) 248 { 249 return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot)); 250 } 251 252 static inline unsigned long 253 vma_pages(struct vm_area_struct *vma) 254 { 255 return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); 256 } 257 258 #define offset_in_page(off) ((unsigned long)(off) & (PAGE_SIZE - 1)) 259 260 static inline void 261 set_page_dirty(struct page *page) 262 { 263 vm_page_dirty(page); 264 } 265 266 static inline void 267 mark_page_accessed(struct page *page) 268 { 269 vm_page_reference(page); 270 } 271 272 static inline void 273 get_page(struct page *page) 274 { 275 vm_page_wire(page); 276 } 277 278 extern long 279 get_user_pages(unsigned long start, unsigned long nr_pages, 280 unsigned int gup_flags, struct page **, 281 struct vm_area_struct **); 282 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 60500 283 #define get_user_pages(start, nr_pages, gup_flags, pages) \ 284 get_user_pages(start, nr_pages, gup_flags, pages, NULL) 285 #endif 286 287 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 60500 288 static inline long 289 pin_user_pages(unsigned long start, unsigned long nr_pages, 290 unsigned int gup_flags, struct page **pages) 291 { 292 return (get_user_pages(start, nr_pages, gup_flags, pages)); 293 } 294 #else 295 static inline long 296 pin_user_pages(unsigned long start, unsigned long nr_pages, 297 unsigned int gup_flags, struct page **pages, 298 struct vm_area_struct **vmas) 299 { 300 return (get_user_pages(start, nr_pages, gup_flags, pages, vmas)); 301 } 302 #endif 303 304 extern int 305 __get_user_pages_fast(unsigned long start, int nr_pages, int write, 306 struct page **); 307 308 static inline int 309 pin_user_pages_fast(unsigned long start, int nr_pages, 310 unsigned int gup_flags, struct page **pages) 311 { 312 return __get_user_pages_fast( 313 start, nr_pages, !!(gup_flags & FOLL_WRITE), pages); 314 } 315 316 extern long 317 get_user_pages_remote(struct task_struct *, struct mm_struct *, 318 unsigned long start, unsigned long nr_pages, 319 unsigned int gup_flags, struct page **, 320 struct vm_area_struct **); 321 322 static inline long 323 pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm, 324 unsigned long start, unsigned long nr_pages, 325 unsigned int gup_flags, struct page **pages, 326 struct vm_area_struct **vmas) 327 { 328 return get_user_pages_remote( 329 task, mm, start, nr_pages, gup_flags, pages, vmas); 330 } 331 332 static inline void 333 put_page(struct page *page) 334 { 335 vm_page_unwire(page, PQ_ACTIVE); 336 } 337 338 #define unpin_user_page(page) put_page(page) 339 #define unpin_user_pages(pages, npages) release_pages(pages, npages) 340 341 #define copy_highpage(to, from) pmap_copy_page(from, to) 342 343 static inline pgprot_t 344 vm_get_page_prot(unsigned long vm_flags) 345 { 346 return (vm_flags & VM_PROT_ALL); 347 } 348 349 static inline void 350 vm_flags_set(struct vm_area_struct *vma, unsigned long flags) 351 { 352 vma->vm_flags |= flags; 353 } 354 355 static inline void 356 vm_flags_clear(struct vm_area_struct *vma, unsigned long flags) 357 { 358 vma->vm_flags &= ~flags; 359 } 360 361 static inline struct page * 362 vmalloc_to_page(const void *addr) 363 { 364 vm_paddr_t paddr; 365 366 paddr = pmap_kextract((vm_offset_t)addr); 367 return (PHYS_TO_VM_PAGE(paddr)); 368 } 369 370 static inline int 371 trylock_page(struct page *page) 372 { 373 return (vm_page_trylock(page)); 374 } 375 376 static inline void 377 unlock_page(struct page *page) 378 { 379 380 vm_page_unlock(page); 381 } 382 383 extern int is_vmalloc_addr(const void *addr); 384 void si_meminfo(struct sysinfo *si); 385 386 static inline unsigned long 387 totalram_pages(void) 388 { 389 return ((unsigned long)physmem); 390 } 391 392 #define unmap_mapping_range(...) lkpi_unmap_mapping_range(__VA_ARGS__) 393 void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused, 394 loff_t const holelen, int even_cows __unused); 395 396 #define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE) 397 398 void vma_set_file(struct vm_area_struct *vma, struct linux_file *file); 399 400 static inline void 401 might_alloc(gfp_t gfp_mask __unused) 402 { 403 } 404 405 #define is_cow_mapping(flags) (false) 406 407 static inline bool 408 want_init_on_free(void) 409 { 410 return (false); 411 } 412 413 #endif /* _LINUXKPI_LINUX_MM_H_ */ 414