1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * Copyright (c) 2015 François Tigeot 7 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #ifndef _LINUXKPI_LINUX_MM_H_ 32 #define _LINUXKPI_LINUX_MM_H_ 33 34 #include <linux/spinlock.h> 35 #include <linux/gfp.h> 36 #include <linux/kernel.h> 37 #include <linux/mm_types.h> 38 #include <linux/mmzone.h> 39 #include <linux/pfn.h> 40 #include <linux/list.h> 41 #include <linux/mmap_lock.h> 42 #include <linux/overflow.h> 43 #include <linux/shrinker.h> 44 #include <linux/page.h> 45 #include <linux/page-flags.h> 46 47 #include <asm/pgtable.h> 48 49 #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) 50 51 /* 52 * Make sure our LinuxKPI defined virtual memory flags don't conflict 53 * with the ones defined by FreeBSD: 54 */ 55 CTASSERT((VM_PROT_ALL & -(1 << 8)) == 0); 56 57 #define VM_READ VM_PROT_READ 58 #define VM_WRITE VM_PROT_WRITE 59 #define VM_EXEC VM_PROT_EXECUTE 60 61 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) 62 63 #define VM_PFNINTERNAL (1 << 8) /* FreeBSD private flag to vm_insert_pfn() */ 64 #define VM_MIXEDMAP (1 << 9) 65 #define VM_NORESERVE (1 << 10) 66 #define VM_PFNMAP (1 << 11) 67 #define VM_IO (1 << 12) 68 #define VM_MAYWRITE (1 << 13) 69 #define VM_DONTCOPY (1 << 14) 70 #define VM_DONTEXPAND (1 << 15) 71 #define VM_DONTDUMP (1 << 16) 72 #define VM_SHARED (1 << 17) 73 74 #define VMA_MAX_PREFAULT_RECORD 1 75 76 #define FOLL_WRITE (1 << 0) 77 #define FOLL_FORCE (1 << 1) 78 79 #define VM_FAULT_OOM (1 << 0) 80 #define VM_FAULT_SIGBUS (1 << 1) 81 #define VM_FAULT_MAJOR (1 << 2) 82 #define VM_FAULT_WRITE (1 << 3) 83 #define VM_FAULT_HWPOISON (1 << 4) 84 #define VM_FAULT_HWPOISON_LARGE (1 << 5) 85 #define VM_FAULT_SIGSEGV (1 << 6) 86 #define VM_FAULT_NOPAGE (1 << 7) 87 #define VM_FAULT_LOCKED (1 << 8) 88 #define VM_FAULT_RETRY (1 << 9) 89 #define VM_FAULT_FALLBACK (1 << 10) 90 91 #define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \ 92 VM_FAULT_HWPOISON |VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) 93 94 #define FAULT_FLAG_WRITE (1 << 0) 95 #define FAULT_FLAG_MKWRITE (1 << 1) 96 #define FAULT_FLAG_ALLOW_RETRY (1 << 2) 97 #define FAULT_FLAG_RETRY_NOWAIT (1 << 3) 98 #define FAULT_FLAG_KILLABLE (1 << 4) 99 #define FAULT_FLAG_TRIED (1 << 5) 100 #define FAULT_FLAG_USER (1 << 6) 101 #define FAULT_FLAG_REMOTE (1 << 7) 102 #define FAULT_FLAG_INSTRUCTION (1 << 8) 103 104 #define fault_flag_allow_retry_first(flags) \ 105 (((flags) & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) == FAULT_FLAG_ALLOW_RETRY) 106 107 typedef int (*pte_fn_t)(linux_pte_t *, unsigned long addr, void *data); 108 109 struct vm_area_struct { 110 vm_offset_t vm_start; 111 vm_offset_t vm_end; 112 vm_offset_t vm_pgoff; 113 pgprot_t vm_page_prot; 114 unsigned long vm_flags; 115 struct mm_struct *vm_mm; 116 void *vm_private_data; 117 const struct vm_operations_struct *vm_ops; 118 struct linux_file *vm_file; 119 120 /* internal operation */ 121 vm_paddr_t vm_pfn; /* PFN for memory map */ 122 vm_size_t vm_len; /* length for memory map */ 123 vm_pindex_t vm_pfn_first; 124 int vm_pfn_count; 125 int *vm_pfn_pcount; 126 vm_object_t vm_obj; 127 vm_map_t vm_cached_map; 128 TAILQ_ENTRY(vm_area_struct) vm_entry; 129 }; 130 131 struct vm_fault { 132 unsigned int flags; 133 pgoff_t pgoff; 134 union { 135 /* user-space address */ 136 void *virtual_address; /* < 4.11 */ 137 unsigned long address; /* >= 4.11 */ 138 }; 139 struct page *page; 140 struct vm_area_struct *vma; 141 }; 142 143 struct vm_operations_struct { 144 void (*open) (struct vm_area_struct *); 145 void (*close) (struct vm_area_struct *); 146 int (*fault) (struct vm_fault *); 147 int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); 148 }; 149 150 struct sysinfo { 151 uint64_t totalram; /* Total usable main memory size */ 152 uint64_t freeram; /* Available memory size */ 153 uint64_t totalhigh; /* Total high memory size */ 154 uint64_t freehigh; /* Available high memory size */ 155 uint32_t mem_unit; /* Memory unit size in bytes */ 156 }; 157 158 static inline struct page * 159 virt_to_head_page(const void *p) 160 { 161 162 return (virt_to_page(p)); 163 } 164 165 static inline struct folio * 166 virt_to_folio(const void *p) 167 { 168 struct page *page = virt_to_page(p); 169 170 return (page_folio(page)); 171 } 172 173 /* 174 * Compute log2 of the power of two rounded up count of pages 175 * needed for size bytes. 176 */ 177 static inline int 178 get_order(unsigned long size) 179 { 180 int order; 181 182 size = (size - 1) >> PAGE_SHIFT; 183 order = 0; 184 while (size) { 185 order++; 186 size >>= 1; 187 } 188 return (order); 189 } 190 191 /* 192 * Resolve a page into a virtual address: 193 * 194 * NOTE: This function only works for pages allocated by the kernel. 195 */ 196 void *linux_page_address(const struct page *); 197 #define page_address(page) linux_page_address(page) 198 199 static inline void * 200 lowmem_page_address(struct page *page) 201 { 202 return (page_address(page)); 203 } 204 205 /* 206 * This only works via memory map operations. 207 */ 208 static inline int 209 io_remap_pfn_range(struct vm_area_struct *vma, 210 unsigned long addr, unsigned long pfn, unsigned long size, 211 vm_memattr_t prot) 212 { 213 vma->vm_page_prot = prot; 214 vma->vm_pfn = pfn; 215 vma->vm_len = size; 216 217 return (0); 218 } 219 220 vm_fault_t 221 lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr, 222 unsigned long pfn, pgprot_t prot); 223 224 static inline vm_fault_t 225 vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 226 unsigned long pfn, pgprot_t prot) 227 { 228 vm_fault_t ret; 229 230 VM_OBJECT_WLOCK(vma->vm_obj); 231 ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot); 232 VM_OBJECT_WUNLOCK(vma->vm_obj); 233 234 return (ret); 235 } 236 #define vmf_insert_pfn_prot(...) \ 237 _Static_assert(false, \ 238 "This function is always called in a loop. Consider using the locked version") 239 240 static inline int 241 apply_to_page_range(struct mm_struct *mm, unsigned long address, 242 unsigned long size, pte_fn_t fn, void *data) 243 { 244 return (-ENOTSUP); 245 } 246 247 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 248 unsigned long size); 249 250 int lkpi_remap_pfn_range(struct vm_area_struct *vma, 251 unsigned long start_addr, unsigned long start_pfn, unsigned long size, 252 pgprot_t prot); 253 254 static inline int 255 remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 256 unsigned long pfn, unsigned long size, pgprot_t prot) 257 { 258 return (lkpi_remap_pfn_range(vma, addr, pfn, size, prot)); 259 } 260 261 static inline unsigned long 262 vma_pages(struct vm_area_struct *vma) 263 { 264 return ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); 265 } 266 267 #define offset_in_page(off) ((unsigned long)(off) & (PAGE_SIZE - 1)) 268 #define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1)) 269 270 static inline void 271 set_page_dirty(struct page *page) 272 { 273 vm_page_dirty(page); 274 } 275 276 static inline void 277 mark_page_accessed(struct page *page) 278 { 279 vm_page_reference(page); 280 } 281 282 static inline void 283 get_page(struct page *page) 284 { 285 vm_page_wire(page); 286 } 287 288 static inline void 289 put_page(struct page *page) 290 { 291 /* `__free_page()` takes care of the refcounting (unwire). */ 292 __free_page(page); 293 } 294 295 static inline void 296 folio_get(struct folio *folio) 297 { 298 get_page(&folio->page); 299 } 300 301 static inline void 302 folio_put(struct folio *folio) 303 { 304 put_page(&folio->page); 305 } 306 307 /* 308 * Linux uses the following "transparent" union so that `release_pages()` 309 * accepts both a list of `struct page` or a list of `struct folio`. This 310 * relies on the fact that a `struct folio` can be cast to a `struct page`. 311 */ 312 typedef union { 313 struct page **pages; 314 struct folio **folios; 315 } release_pages_arg __attribute__ ((__transparent_union__)); 316 317 void linux_release_pages(release_pages_arg arg, int nr); 318 #define release_pages(arg, nr) linux_release_pages((arg), (nr)) 319 320 extern long 321 lkpi_get_user_pages(unsigned long start, unsigned long nr_pages, 322 unsigned int gup_flags, struct page **); 323 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 60500 324 #define get_user_pages(start, nr_pages, gup_flags, pages) \ 325 lkpi_get_user_pages(start, nr_pages, gup_flags, pages) 326 #else 327 #define get_user_pages(start, nr_pages, gup_flags, pages, vmas) \ 328 lkpi_get_user_pages(start, nr_pages, gup_flags, pages) 329 #endif 330 331 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 60500 332 static inline long 333 pin_user_pages(unsigned long start, unsigned long nr_pages, 334 unsigned int gup_flags, struct page **pages) 335 { 336 return (get_user_pages(start, nr_pages, gup_flags, pages)); 337 } 338 #else 339 static inline long 340 pin_user_pages(unsigned long start, unsigned long nr_pages, 341 unsigned int gup_flags, struct page **pages, 342 struct vm_area_struct **vmas) 343 { 344 return (get_user_pages(start, nr_pages, gup_flags, pages, vmas)); 345 } 346 #endif 347 348 extern int 349 __get_user_pages_fast(unsigned long start, int nr_pages, int write, 350 struct page **); 351 352 static inline int 353 pin_user_pages_fast(unsigned long start, int nr_pages, 354 unsigned int gup_flags, struct page **pages) 355 { 356 return __get_user_pages_fast( 357 start, nr_pages, !!(gup_flags & FOLL_WRITE), pages); 358 } 359 360 extern long 361 get_user_pages_remote(struct task_struct *, struct mm_struct *, 362 unsigned long start, unsigned long nr_pages, 363 unsigned int gup_flags, struct page **, 364 struct vm_area_struct **); 365 366 static inline long 367 pin_user_pages_remote(struct task_struct *task, struct mm_struct *mm, 368 unsigned long start, unsigned long nr_pages, 369 unsigned int gup_flags, struct page **pages, 370 struct vm_area_struct **vmas) 371 { 372 return get_user_pages_remote( 373 task, mm, start, nr_pages, gup_flags, pages, vmas); 374 } 375 376 #define unpin_user_page(page) put_page(page) 377 #define unpin_user_pages(pages, npages) release_pages(pages, npages) 378 379 #define copy_highpage(to, from) pmap_copy_page(from, to) 380 381 static inline pgprot_t 382 vm_get_page_prot(unsigned long vm_flags) 383 { 384 return (vm_flags & VM_PROT_ALL); 385 } 386 387 static inline void 388 vm_flags_set(struct vm_area_struct *vma, unsigned long flags) 389 { 390 vma->vm_flags |= flags; 391 } 392 393 static inline void 394 vm_flags_clear(struct vm_area_struct *vma, unsigned long flags) 395 { 396 vma->vm_flags &= ~flags; 397 } 398 399 static inline struct page * 400 vmalloc_to_page(const void *addr) 401 { 402 vm_paddr_t paddr; 403 404 paddr = pmap_kextract((vm_offset_t)addr); 405 return (PHYS_TO_VM_PAGE(paddr)); 406 } 407 408 static inline int 409 trylock_page(struct page *page) 410 { 411 return (vm_page_tryxbusy(page)); 412 } 413 414 static inline void 415 unlock_page(struct page *page) 416 { 417 418 vm_page_xunbusy(page); 419 } 420 421 extern int is_vmalloc_addr(const void *addr); 422 void si_meminfo(struct sysinfo *si); 423 424 static inline unsigned long 425 totalram_pages(void) 426 { 427 return ((unsigned long)physmem); 428 } 429 430 #define unmap_mapping_range(...) lkpi_unmap_mapping_range(__VA_ARGS__) 431 void lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused, 432 loff_t const holelen, int even_cows __unused); 433 434 #define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE) 435 436 void vma_set_file(struct vm_area_struct *vma, struct linux_file *file); 437 438 static inline void 439 might_alloc(gfp_t gfp_mask __unused) 440 { 441 } 442 443 #define is_cow_mapping(flags) (false) 444 445 static inline bool 446 want_init_on_free(void) 447 { 448 return (false); 449 } 450 451 static inline unsigned long 452 folio_pfn(struct folio *folio) 453 { 454 return (page_to_pfn(&folio->page)); 455 } 456 457 static inline long 458 folio_nr_pages(struct folio *folio) 459 { 460 return (1); 461 } 462 463 static inline size_t 464 folio_size(struct folio *folio) 465 { 466 return (PAGE_SIZE); 467 } 468 469 static inline void 470 folio_mark_dirty(struct folio *folio) 471 { 472 set_page_dirty(&folio->page); 473 } 474 475 static inline void * 476 folio_address(const struct folio *folio) 477 { 478 return (page_address(&folio->page)); 479 } 480 481 #endif /* _LINUXKPI_LINUX_MM_H_ */ 482