1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org) 4 * Copyright (c) 2017 Mellanox Technologies, Ltd. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/malloc.h> 35 #include <sys/kernel.h> 36 #include <sys/sysctl.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/rwlock.h> 40 #include <sys/proc.h> 41 #include <sys/sched.h> 42 43 #include <machine/bus.h> 44 45 #include <linux/gfp.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_pageout.h> 50 51 void * 52 linux_page_address(struct page *page) 53 { 54 #ifdef __amd64__ 55 return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))); 56 #else 57 if (page->object != kmem_object && page->object != kernel_object) 58 return (NULL); 59 return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + 60 IDX_TO_OFF(page->pindex))); 61 #endif 62 } 63 64 vm_page_t 65 linux_alloc_pages(gfp_t flags, unsigned int order) 66 { 67 #ifdef __amd64__ 68 unsigned long npages = 1UL << order; 69 int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ | 70 VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL); 71 vm_page_t page; 72 73 if (order == 0 && (flags & GFP_DMA32) == 0) { 74 page = vm_page_alloc(NULL, 0, req); 75 if (page == NULL) 76 return (NULL); 77 } else { 78 vm_paddr_t pmax = (flags & GFP_DMA32) ? 79 BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR; 80 retry: 81 page = vm_page_alloc_contig(NULL, 0, req, 82 npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 83 84 if (page == NULL) { 85 if (flags & M_WAITOK) { 86 if (!vm_page_reclaim_contig(req, 87 npages, 0, pmax, PAGE_SIZE, 0)) { 88 VM_WAIT; 89 } 90 flags &= ~M_WAITOK; 91 goto retry; 92 } 93 return (NULL); 94 } 95 } 96 if (flags & M_ZERO) { 97 unsigned long x; 98 99 for (x = 0; x != npages; x++) { 100 vm_page_t pgo = page + x; 101 102 if ((pgo->flags & PG_ZERO) == 0) 103 pmap_zero_page(pgo); 104 } 105 } 106 #else 107 vm_offset_t vaddr; 108 vm_page_t page; 109 110 vaddr = linux_alloc_kmem(flags, order); 111 if (vaddr == 0) 112 return (NULL); 113 114 page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr)); 115 116 KASSERT(vaddr == (vm_offset_t)page_address(page), 117 ("Page address mismatch")); 118 #endif 119 return (page); 120 } 121 122 void 123 linux_free_pages(vm_page_t page, unsigned int order) 124 { 125 #ifdef __amd64__ 126 unsigned long npages = 1UL << order; 127 unsigned long x; 128 129 for (x = 0; x != npages; x++) { 130 vm_page_t pgo = page + x; 131 132 vm_page_lock(pgo); 133 vm_page_free(pgo); 134 vm_page_unlock(pgo); 135 } 136 #else 137 vm_offset_t vaddr; 138 139 vaddr = (vm_offset_t)page_address(page); 140 141 linux_free_kmem(vaddr, order); 142 #endif 143 } 144 145 vm_offset_t 146 linux_alloc_kmem(gfp_t flags, unsigned int order) 147 { 148 size_t size = ((size_t)PAGE_SIZE) << order; 149 vm_offset_t addr; 150 151 if ((flags & GFP_DMA32) == 0) { 152 addr = kmem_malloc(kmem_arena, size, flags & GFP_NATIVE_MASK); 153 } else { 154 addr = kmem_alloc_contig(kmem_arena, size, 155 flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT, 156 PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 157 } 158 return (addr); 159 } 160 161 void 162 linux_free_kmem(vm_offset_t addr, unsigned int order) 163 { 164 size_t size = ((size_t)PAGE_SIZE) << order; 165 166 kmem_free(kmem_arena, addr, size); 167 } 168