1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017, Jeffrey Roberson <jeff@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_vm.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bitset.h> 38 #include <sys/domainset.h> 39 #include <sys/proc.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/malloc.h> 43 #include <sys/vmmeter.h> 44 45 #include <vm/vm.h> 46 #include <vm/vm_param.h> 47 #include <vm/vm_domainset.h> 48 #include <vm/vm_object.h> 49 #include <vm/vm_page.h> 50 #include <vm/vm_phys.h> 51 52 #ifdef NUMA 53 /* 54 * Iterators are written such that the first nowait pass has as short a 55 * codepath as possible to eliminate bloat from the allocator. It is 56 * assumed that most allocations are successful. 57 */ 58 59 /* 60 * Determine which policy is to be used for this allocation. 61 */ 62 static void 63 vm_domainset_iter_domain(struct vm_domainset_iter *di, struct vm_object *obj) 64 { 65 struct domainset *domain; 66 67 /* 68 * object policy takes precedence over thread policy. The policies 69 * are immutable and unsynchronized. Updates can race but pointer 70 * loads are assumed to be atomic. 71 */ 72 if (obj != NULL && (domain = obj->domain.dr_policy) != NULL) { 73 di->di_domain = domain; 74 di->di_iter = &obj->domain.dr_iterator; 75 } else { 76 di->di_domain = curthread->td_domain.dr_policy; 77 di->di_iter = &curthread->td_domain.dr_iterator; 78 } 79 } 80 81 static void 82 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain) 83 { 84 int d; 85 86 d = *di->di_iter; 87 do { 88 d = (d + 1) % di->di_domain->ds_max; 89 } while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask)); 90 *di->di_iter = *domain = d; 91 } 92 93 static void 94 vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain) 95 { 96 int d; 97 98 d = *di->di_iter; 99 do { 100 d = (d + 1) % di->di_domain->ds_max; 101 } while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask) || 102 d == di->di_domain->ds_prefer); 103 *di->di_iter = *domain = d; 104 } 105 106 static void 107 vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain) 108 { 109 110 KASSERT(di->di_n > 0, 111 ("vm_domainset_iter_first: Invalid n %d", di->di_n)); 112 switch (di->di_domain->ds_policy) { 113 case DOMAINSET_POLICY_FIRSTTOUCH: 114 /* 115 * To prevent impossible allocations we convert an invalid 116 * first-touch to round-robin. 117 */ 118 /* FALLTHROUGH */ 119 case DOMAINSET_POLICY_ROUNDROBIN: 120 vm_domainset_iter_rr(di, domain); 121 break; 122 case DOMAINSET_POLICY_PREFER: 123 vm_domainset_iter_prefer(di, domain); 124 break; 125 default: 126 panic("vm_domainset_iter_first: Unknown policy %d", 127 di->di_domain->ds_policy); 128 } 129 KASSERT(*domain < vm_ndomains, 130 ("vm_domainset_iter_next: Invalid domain %d", *domain)); 131 } 132 133 static void 134 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain) 135 { 136 137 switch (di->di_domain->ds_policy) { 138 case DOMAINSET_POLICY_FIRSTTOUCH: 139 *domain = PCPU_GET(domain); 140 if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) { 141 di->di_n = 1; 142 break; 143 } 144 /* 145 * To prevent impossible allocations we convert an invalid 146 * first-touch to round-robin. 147 */ 148 /* FALLTHROUGH */ 149 case DOMAINSET_POLICY_ROUNDROBIN: 150 di->di_n = di->di_domain->ds_cnt; 151 vm_domainset_iter_rr(di, domain); 152 break; 153 case DOMAINSET_POLICY_PREFER: 154 *domain = di->di_domain->ds_prefer; 155 di->di_n = di->di_domain->ds_cnt; 156 break; 157 default: 158 panic("vm_domainset_iter_first: Unknown policy %d", 159 di->di_domain->ds_policy); 160 } 161 KASSERT(di->di_n > 0, 162 ("vm_domainset_iter_first: Invalid n %d", di->di_n)); 163 KASSERT(*domain < vm_ndomains, 164 ("vm_domainset_iter_first: Invalid domain %d", *domain)); 165 } 166 167 void 168 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, 169 int *domain, int *req) 170 { 171 172 vm_domainset_iter_domain(di, obj); 173 di->di_flags = *req; 174 *req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) | 175 VM_ALLOC_NOWAIT; 176 vm_domainset_iter_first(di, domain); 177 } 178 179 int 180 vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *req) 181 { 182 183 /* 184 * If we exhausted all options with NOWAIT and did a WAITFAIL it 185 * is time to return an error to the caller. 186 */ 187 if ((*req & VM_ALLOC_WAITFAIL) != 0) 188 return (ENOMEM); 189 190 /* If there are more domains to visit we run the iterator. */ 191 if (--di->di_n != 0) { 192 vm_domainset_iter_next(di, domain); 193 return (0); 194 } 195 196 /* If we visited all domains and this was a NOWAIT we return error. */ 197 if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0) 198 return (ENOMEM); 199 200 /* 201 * We have visited all domains with non-blocking allocations, try 202 * from the beginning with a blocking allocation. 203 */ 204 vm_domainset_iter_first(di, domain); 205 *req = di->di_flags; 206 207 return (0); 208 } 209 210 211 void 212 vm_domainset_iter_malloc_init(struct vm_domainset_iter *di, 213 struct vm_object *obj, int *domain, int *flags) 214 { 215 216 vm_domainset_iter_domain(di, obj); 217 di->di_flags = *flags; 218 *flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT; 219 vm_domainset_iter_first(di, domain); 220 } 221 222 int 223 vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags) 224 { 225 226 /* If there are more domains to visit we run the iterator. */ 227 if (--di->di_n != 0) { 228 vm_domainset_iter_next(di, domain); 229 return (0); 230 } 231 232 /* If we visited all domains and this was a NOWAIT we return error. */ 233 if ((di->di_flags & M_WAITOK) == 0) 234 return (ENOMEM); 235 236 /* 237 * We have visited all domains with non-blocking allocations, try 238 * from the beginning with a blocking allocation. 239 */ 240 vm_domainset_iter_first(di, domain); 241 *flags = di->di_flags; 242 243 return (0); 244 } 245 246 #else /* !NUMA */ 247 int 248 vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *flags) 249 { 250 251 return (EJUSTRETURN); 252 } 253 254 void 255 vm_domainset_iter_page_init(struct vm_domainset_iter *di, 256 struct vm_object *obj, int *domain, int *flags) 257 { 258 259 *domain = 0; 260 } 261 262 int 263 vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags) 264 { 265 266 return (EJUSTRETURN); 267 } 268 269 void 270 vm_domainset_iter_malloc_init(struct vm_domainset_iter *di, 271 struct vm_object *obj, int *domain, int *flags) 272 { 273 274 *domain = 0; 275 } 276 277 #endif 278