1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 5 * All rights reserved. 6 * 7 * Copyright (c) 2008 Nokia Corporation 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_ddb.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/sysctl.h> 41 #include <sys/ctype.h> 42 #include <sys/sysproto.h> 43 #include <sys/jail.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/mutex.h> 48 #include <sys/priv.h> 49 #include <sys/proc.h> 50 #include <sys/refcount.h> 51 #include <sys/sched.h> 52 #include <sys/smp.h> 53 #include <sys/syscallsubr.h> 54 #include <sys/capsicum.h> 55 #include <sys/cpuset.h> 56 #include <sys/domainset.h> 57 #include <sys/sx.h> 58 #include <sys/queue.h> 59 #include <sys/libkern.h> 60 #include <sys/limits.h> 61 #include <sys/bus.h> 62 #include <sys/interrupt.h> 63 #include <sys/vmmeter.h> 64 65 #include <vm/uma.h> 66 #include <vm/vm.h> 67 #include <vm/vm_object.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_pageout.h> 70 #include <vm/vm_extern.h> 71 #include <vm/vm_param.h> 72 #include <vm/vm_phys.h> 73 #include <vm/vm_pagequeue.h> 74 75 #ifdef DDB 76 #include <ddb/ddb.h> 77 #endif /* DDB */ 78 79 /* 80 * cpusets provide a mechanism for creating and manipulating sets of 81 * processors for the purpose of constraining the scheduling of threads to 82 * specific processors. 83 * 84 * Each process belongs to an identified set, by default this is set 1. Each 85 * thread may further restrict the cpus it may run on to a subset of this 86 * named set. This creates an anonymous set which other threads and processes 87 * may not join by number. 88 * 89 * The named set is referred to herein as the 'base' set to avoid ambiguity. 90 * This set is usually a child of a 'root' set while the anonymous set may 91 * simply be referred to as a mask. In the syscall api these are referred to 92 * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 93 * 94 * Threads inherit their set from their creator whether it be anonymous or 95 * not. This means that anonymous sets are immutable because they may be 96 * shared. To modify an anonymous set a new set is created with the desired 97 * mask and the same parent as the existing anonymous set. This gives the 98 * illusion of each thread having a private mask. 99 * 100 * Via the syscall apis a user may ask to retrieve or modify the root, base, 101 * or mask that is discovered via a pid, tid, or setid. Modifying a set 102 * modifies all numbered and anonymous child sets to comply with the new mask. 103 * Modifying a pid or tid's mask applies only to that tid but must still 104 * exist within the assigned parent set. 105 * 106 * A thread may not be assigned to a group separate from other threads in 107 * the process. This is to remove ambiguity when the setid is queried with 108 * a pid argument. There is no other technical limitation. 109 * 110 * This somewhat complex arrangement is intended to make it easy for 111 * applications to query available processors and bind their threads to 112 * specific processors while also allowing administrators to dynamically 113 * reprovision by changing sets which apply to groups of processes. 114 * 115 * A simple application should not concern itself with sets at all and 116 * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 117 * meaning 'curthread'. It may query available cpus for that tid with a 118 * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 119 */ 120 121 LIST_HEAD(domainlist, domainset); 122 struct domainset __read_mostly domainset_firsttouch; 123 struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; 124 struct domainset __read_mostly domainset_interleave; 125 struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; 126 struct domainset __read_mostly domainset_roundrobin; 127 128 static uma_zone_t cpuset_zone; 129 static uma_zone_t domainset_zone; 130 static struct mtx cpuset_lock; 131 static struct setlist cpuset_ids; 132 static struct domainlist cpuset_domains; 133 static struct unrhdr *cpuset_unr; 134 static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; 135 static struct domainset *domainset0, *domainset2; 136 137 /* Return the size of cpuset_t at the kernel level */ 138 SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, 139 SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); 140 141 cpuset_t *cpuset_root; 142 cpuset_t cpuset_domain[MAXMEMDOM]; 143 144 static int domainset_valid(const struct domainset *, const struct domainset *); 145 146 /* 147 * Find the first non-anonymous set starting from 'set'. 148 */ 149 static struct cpuset * 150 cpuset_getbase(struct cpuset *set) 151 { 152 153 if (set->cs_id == CPUSET_INVALID) 154 set = set->cs_parent; 155 return (set); 156 } 157 158 /* 159 * Walks up the tree from 'set' to find the root. 160 */ 161 static struct cpuset * 162 cpuset_getroot(struct cpuset *set) 163 { 164 165 while ((set->cs_flags & CPU_SET_ROOT) == 0 && set->cs_parent != NULL) 166 set = set->cs_parent; 167 return (set); 168 } 169 170 /* 171 * Acquire a reference to a cpuset, all pointers must be tracked with refs. 172 */ 173 struct cpuset * 174 cpuset_ref(struct cpuset *set) 175 { 176 177 refcount_acquire(&set->cs_ref); 178 return (set); 179 } 180 181 /* 182 * Walks up the tree from 'set' to find the root. Returns the root 183 * referenced. 184 */ 185 static struct cpuset * 186 cpuset_refroot(struct cpuset *set) 187 { 188 189 return (cpuset_ref(cpuset_getroot(set))); 190 } 191 192 /* 193 * Find the first non-anonymous set starting from 'set'. Returns this set 194 * referenced. May return the passed in set with an extra ref if it is 195 * not anonymous. 196 */ 197 static struct cpuset * 198 cpuset_refbase(struct cpuset *set) 199 { 200 201 return (cpuset_ref(cpuset_getbase(set))); 202 } 203 204 /* 205 * Release a reference in a context where it is safe to allocate. 206 */ 207 void 208 cpuset_rel(struct cpuset *set) 209 { 210 cpusetid_t id; 211 212 if (refcount_release_if_not_last(&set->cs_ref)) 213 return; 214 mtx_lock_spin(&cpuset_lock); 215 if (!refcount_release(&set->cs_ref)) { 216 mtx_unlock_spin(&cpuset_lock); 217 return; 218 } 219 LIST_REMOVE(set, cs_siblings); 220 id = set->cs_id; 221 if (id != CPUSET_INVALID) 222 LIST_REMOVE(set, cs_link); 223 mtx_unlock_spin(&cpuset_lock); 224 cpuset_rel(set->cs_parent); 225 uma_zfree(cpuset_zone, set); 226 if (id != CPUSET_INVALID) 227 free_unr(cpuset_unr, id); 228 } 229 230 /* 231 * Deferred release must be used when in a context that is not safe to 232 * allocate/free. This places any unreferenced sets on the list 'head'. 233 */ 234 static void 235 cpuset_rel_defer(struct setlist *head, struct cpuset *set) 236 { 237 238 if (refcount_release_if_not_last(&set->cs_ref)) 239 return; 240 mtx_lock_spin(&cpuset_lock); 241 if (!refcount_release(&set->cs_ref)) { 242 mtx_unlock_spin(&cpuset_lock); 243 return; 244 } 245 LIST_REMOVE(set, cs_siblings); 246 if (set->cs_id != CPUSET_INVALID) 247 LIST_REMOVE(set, cs_link); 248 LIST_INSERT_HEAD(head, set, cs_link); 249 mtx_unlock_spin(&cpuset_lock); 250 } 251 252 /* 253 * Complete a deferred release. Removes the set from the list provided to 254 * cpuset_rel_defer. 255 */ 256 static void 257 cpuset_rel_complete(struct cpuset *set) 258 { 259 cpusetid_t id; 260 261 id = set->cs_id; 262 LIST_REMOVE(set, cs_link); 263 cpuset_rel(set->cs_parent); 264 uma_zfree(cpuset_zone, set); 265 if (id != CPUSET_INVALID) 266 free_unr(cpuset_unr, id); 267 } 268 269 /* 270 * Find a set based on an id. Returns it with a ref. 271 */ 272 static struct cpuset * 273 cpuset_lookup(cpusetid_t setid, struct thread *td) 274 { 275 struct cpuset *set; 276 277 if (setid == CPUSET_INVALID) 278 return (NULL); 279 mtx_lock_spin(&cpuset_lock); 280 LIST_FOREACH(set, &cpuset_ids, cs_link) 281 if (set->cs_id == setid) 282 break; 283 if (set) 284 cpuset_ref(set); 285 mtx_unlock_spin(&cpuset_lock); 286 287 KASSERT(td != NULL, ("[%s:%d] td is NULL", __func__, __LINE__)); 288 if (set != NULL && jailed(td->td_ucred)) { 289 struct cpuset *jset, *tset; 290 291 jset = td->td_ucred->cr_prison->pr_cpuset; 292 for (tset = set; tset != NULL; tset = tset->cs_parent) 293 if (tset == jset) 294 break; 295 if (tset == NULL) { 296 cpuset_rel(set); 297 set = NULL; 298 } 299 } 300 301 return (set); 302 } 303 304 /* 305 * Initialize a set in the space provided in 'set' with the provided parameters. 306 * The set is returned with a single ref. May return EDEADLK if the set 307 * will have no valid cpu based on restrictions from the parent. 308 */ 309 static int 310 cpuset_init(struct cpuset *set, struct cpuset *parent, 311 const cpuset_t *mask, struct domainset *domain, cpusetid_t id) 312 { 313 314 if (domain == NULL) 315 domain = parent->cs_domain; 316 if (mask == NULL) 317 mask = &parent->cs_mask; 318 if (!CPU_OVERLAP(&parent->cs_mask, mask)) 319 return (EDEADLK); 320 /* The domain must be prepared ahead of time. */ 321 if (!domainset_valid(parent->cs_domain, domain)) 322 return (EDEADLK); 323 CPU_COPY(mask, &set->cs_mask); 324 LIST_INIT(&set->cs_children); 325 refcount_init(&set->cs_ref, 1); 326 set->cs_flags = 0; 327 mtx_lock_spin(&cpuset_lock); 328 set->cs_domain = domain; 329 CPU_AND(&set->cs_mask, &set->cs_mask, &parent->cs_mask); 330 set->cs_id = id; 331 set->cs_parent = cpuset_ref(parent); 332 LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 333 if (set->cs_id != CPUSET_INVALID) 334 LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 335 mtx_unlock_spin(&cpuset_lock); 336 337 return (0); 338 } 339 340 /* 341 * Create a new non-anonymous set with the requested parent and mask. May 342 * return failures if the mask is invalid or a new number can not be 343 * allocated. 344 * 345 * If *setp is not NULL, then it will be used as-is. The caller must take 346 * into account that *setp will be inserted at the head of cpuset_ids and 347 * plan any potentially conflicting cs_link usage accordingly. 348 */ 349 static int 350 cpuset_create(struct cpuset **setp, struct cpuset *parent, const cpuset_t *mask) 351 { 352 struct cpuset *set; 353 cpusetid_t id; 354 int error; 355 bool dofree; 356 357 id = alloc_unr(cpuset_unr); 358 if (id == -1) 359 return (ENFILE); 360 dofree = (*setp == NULL); 361 if (*setp != NULL) 362 set = *setp; 363 else 364 *setp = set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 365 error = cpuset_init(set, parent, mask, NULL, id); 366 if (error == 0) 367 return (0); 368 free_unr(cpuset_unr, id); 369 if (dofree) 370 uma_zfree(cpuset_zone, set); 371 372 return (error); 373 } 374 375 static void 376 cpuset_freelist_add(struct setlist *list, int count) 377 { 378 struct cpuset *set; 379 int i; 380 381 for (i = 0; i < count; i++) { 382 set = uma_zalloc(cpuset_zone, M_ZERO | M_WAITOK); 383 LIST_INSERT_HEAD(list, set, cs_link); 384 } 385 } 386 387 static void 388 cpuset_freelist_init(struct setlist *list, int count) 389 { 390 391 LIST_INIT(list); 392 cpuset_freelist_add(list, count); 393 } 394 395 static void 396 cpuset_freelist_free(struct setlist *list) 397 { 398 struct cpuset *set; 399 400 while ((set = LIST_FIRST(list)) != NULL) { 401 LIST_REMOVE(set, cs_link); 402 uma_zfree(cpuset_zone, set); 403 } 404 } 405 406 static void 407 domainset_freelist_add(struct domainlist *list, int count) 408 { 409 struct domainset *set; 410 int i; 411 412 for (i = 0; i < count; i++) { 413 set = uma_zalloc(domainset_zone, M_ZERO | M_WAITOK); 414 LIST_INSERT_HEAD(list, set, ds_link); 415 } 416 } 417 418 static void 419 domainset_freelist_init(struct domainlist *list, int count) 420 { 421 422 LIST_INIT(list); 423 domainset_freelist_add(list, count); 424 } 425 426 static void 427 domainset_freelist_free(struct domainlist *list) 428 { 429 struct domainset *set; 430 431 while ((set = LIST_FIRST(list)) != NULL) { 432 LIST_REMOVE(set, ds_link); 433 uma_zfree(domainset_zone, set); 434 } 435 } 436 437 /* Copy a domainset preserving mask and policy. */ 438 static void 439 domainset_copy(const struct domainset *from, struct domainset *to) 440 { 441 442 DOMAINSET_COPY(&from->ds_mask, &to->ds_mask); 443 to->ds_policy = from->ds_policy; 444 to->ds_prefer = from->ds_prefer; 445 } 446 447 /* Return 1 if mask and policy are equal, otherwise 0. */ 448 static int 449 domainset_equal(const struct domainset *one, const struct domainset *two) 450 { 451 452 return (DOMAINSET_CMP(&one->ds_mask, &two->ds_mask) == 0 && 453 one->ds_policy == two->ds_policy && 454 one->ds_prefer == two->ds_prefer); 455 } 456 457 /* Return 1 if child is a valid subset of parent. */ 458 static int 459 domainset_valid(const struct domainset *parent, const struct domainset *child) 460 { 461 if (child->ds_policy != DOMAINSET_POLICY_PREFER) 462 return (DOMAINSET_SUBSET(&parent->ds_mask, &child->ds_mask)); 463 return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 464 } 465 466 static int 467 domainset_restrict(const struct domainset *parent, 468 const struct domainset *child) 469 { 470 if (child->ds_policy != DOMAINSET_POLICY_PREFER) 471 return (DOMAINSET_OVERLAP(&parent->ds_mask, &child->ds_mask)); 472 return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 473 } 474 475 /* 476 * Lookup or create a domainset. The key is provided in ds_mask and 477 * ds_policy. If the domainset does not yet exist the storage in 478 * 'domain' is used to insert. Otherwise this storage is freed to the 479 * domainset_zone and the existing domainset is returned. 480 */ 481 static struct domainset * 482 _domainset_create(struct domainset *domain, struct domainlist *freelist) 483 { 484 struct domainset *ndomain; 485 int i, j; 486 487 KASSERT(domain->ds_cnt <= vm_ndomains, 488 ("invalid domain count in domainset %p", domain)); 489 KASSERT(domain->ds_policy != DOMAINSET_POLICY_PREFER || 490 domain->ds_prefer < vm_ndomains, 491 ("invalid preferred domain in domains %p", domain)); 492 493 mtx_lock_spin(&cpuset_lock); 494 LIST_FOREACH(ndomain, &cpuset_domains, ds_link) 495 if (domainset_equal(ndomain, domain)) 496 break; 497 /* 498 * If the domain does not yet exist we insert it and initialize 499 * various iteration helpers which are not part of the key. 500 */ 501 if (ndomain == NULL) { 502 LIST_INSERT_HEAD(&cpuset_domains, domain, ds_link); 503 domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 504 for (i = 0, j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 505 if (DOMAINSET_ISSET(i, &domain->ds_mask)) 506 domain->ds_order[j++] = i; 507 } 508 mtx_unlock_spin(&cpuset_lock); 509 if (ndomain == NULL) 510 return (domain); 511 if (freelist != NULL) 512 LIST_INSERT_HEAD(freelist, domain, ds_link); 513 else 514 uma_zfree(domainset_zone, domain); 515 return (ndomain); 516 517 } 518 519 /* 520 * Are any of the domains in the mask empty? If so, silently 521 * remove them and update the domainset accordingly. If only empty 522 * domains are present, we must return failure. 523 */ 524 static bool 525 domainset_empty_vm(struct domainset *domain) 526 { 527 domainset_t empty; 528 int i, j; 529 530 DOMAINSET_ZERO(&empty); 531 for (i = 0; i < vm_ndomains; i++) 532 if (VM_DOMAIN_EMPTY(i)) 533 DOMAINSET_SET(i, &empty); 534 if (DOMAINSET_SUBSET(&empty, &domain->ds_mask)) 535 return (true); 536 537 /* Remove empty domains from the set and recompute. */ 538 DOMAINSET_ANDNOT(&domain->ds_mask, &empty); 539 domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 540 for (i = j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 541 if (DOMAINSET_ISSET(i, &domain->ds_mask)) 542 domain->ds_order[j++] = i; 543 544 /* Convert a PREFER policy referencing an empty domain to RR. */ 545 if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 546 DOMAINSET_ISSET(domain->ds_prefer, &empty)) { 547 domain->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 548 domain->ds_prefer = -1; 549 } 550 551 return (false); 552 } 553 554 /* 555 * Create or lookup a domainset based on the key held in 'domain'. 556 */ 557 struct domainset * 558 domainset_create(const struct domainset *domain) 559 { 560 struct domainset *ndomain; 561 562 /* 563 * Validate the policy. It must specify a useable policy number with 564 * only valid domains. Preferred must include the preferred domain 565 * in the mask. 566 */ 567 if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || 568 domain->ds_policy > DOMAINSET_POLICY_MAX) 569 return (NULL); 570 if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 571 !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) 572 return (NULL); 573 if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) 574 return (NULL); 575 ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); 576 domainset_copy(domain, ndomain); 577 return _domainset_create(ndomain, NULL); 578 } 579 580 /* 581 * Update thread domainset pointers. 582 */ 583 static void 584 domainset_notify(void) 585 { 586 struct thread *td; 587 struct proc *p; 588 589 sx_slock(&allproc_lock); 590 FOREACH_PROC_IN_SYSTEM(p) { 591 PROC_LOCK(p); 592 if (p->p_state == PRS_NEW) { 593 PROC_UNLOCK(p); 594 continue; 595 } 596 FOREACH_THREAD_IN_PROC(p, td) { 597 thread_lock(td); 598 td->td_domain.dr_policy = td->td_cpuset->cs_domain; 599 thread_unlock(td); 600 } 601 PROC_UNLOCK(p); 602 } 603 sx_sunlock(&allproc_lock); 604 kernel_object->domain.dr_policy = cpuset_kernel->cs_domain; 605 } 606 607 /* 608 * Create a new set that is a subset of a parent. 609 */ 610 static struct domainset * 611 domainset_shadow(const struct domainset *pdomain, 612 const struct domainset *domain, struct domainlist *freelist) 613 { 614 struct domainset *ndomain; 615 616 ndomain = LIST_FIRST(freelist); 617 LIST_REMOVE(ndomain, ds_link); 618 619 /* 620 * Initialize the key from the request. 621 */ 622 domainset_copy(domain, ndomain); 623 624 /* 625 * Restrict the key by the parent. 626 */ 627 DOMAINSET_AND(&ndomain->ds_mask, &pdomain->ds_mask); 628 629 return _domainset_create(ndomain, freelist); 630 } 631 632 /* 633 * Recursively check for errors that would occur from applying mask to 634 * the tree of sets starting at 'set'. Checks for sets that would become 635 * empty as well as RDONLY flags. 636 */ 637 static int 638 cpuset_testupdate(struct cpuset *set, cpuset_t *mask, int augment_mask) 639 { 640 struct cpuset *nset; 641 cpuset_t newmask; 642 int error; 643 644 mtx_assert(&cpuset_lock, MA_OWNED); 645 if (set->cs_flags & CPU_SET_RDONLY) 646 return (EPERM); 647 if (augment_mask) { 648 CPU_AND(&newmask, &set->cs_mask, mask); 649 } else 650 CPU_COPY(mask, &newmask); 651 652 if (CPU_EMPTY(&newmask)) 653 return (EDEADLK); 654 error = 0; 655 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 656 if ((error = cpuset_testupdate(nset, &newmask, 1)) != 0) 657 break; 658 return (error); 659 } 660 661 /* 662 * Applies the mask 'mask' without checking for empty sets or permissions. 663 */ 664 static void 665 cpuset_update(struct cpuset *set, cpuset_t *mask) 666 { 667 struct cpuset *nset; 668 669 mtx_assert(&cpuset_lock, MA_OWNED); 670 CPU_AND(&set->cs_mask, &set->cs_mask, mask); 671 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 672 cpuset_update(nset, &set->cs_mask); 673 674 return; 675 } 676 677 /* 678 * Modify the set 'set' to use a copy of the mask provided. Apply this new 679 * mask to restrict all children in the tree. Checks for validity before 680 * applying the changes. 681 */ 682 static int 683 cpuset_modify(struct cpuset *set, cpuset_t *mask) 684 { 685 struct cpuset *root; 686 int error; 687 688 error = priv_check(curthread, PRIV_SCHED_CPUSET); 689 if (error) 690 return (error); 691 /* 692 * In case we are called from within the jail, 693 * we do not allow modifying the dedicated root 694 * cpuset of the jail but may still allow to 695 * change child sets, including subordinate jails' 696 * roots. 697 */ 698 if ((set->cs_flags & CPU_SET_ROOT) != 0 && 699 jailed(curthread->td_ucred) && 700 set == curthread->td_ucred->cr_prison->pr_cpuset) 701 return (EPERM); 702 /* 703 * Verify that we have access to this set of 704 * cpus. 705 */ 706 if ((set->cs_flags & (CPU_SET_ROOT | CPU_SET_RDONLY)) == CPU_SET_ROOT) { 707 KASSERT(set->cs_parent != NULL, 708 ("jail.cpuset=%d is not a proper child of parent jail's root.", 709 set->cs_id)); 710 711 /* 712 * cpuset_getroot() cannot work here due to how top-level jail 713 * roots are constructed. Top-level jails are parented to 714 * thread0's cpuset (i.e. cpuset 1) rather than the system root. 715 */ 716 root = set->cs_parent; 717 } else { 718 root = cpuset_getroot(set); 719 } 720 mtx_lock_spin(&cpuset_lock); 721 if (root && !CPU_SUBSET(&root->cs_mask, mask)) { 722 error = EINVAL; 723 goto out; 724 } 725 error = cpuset_testupdate(set, mask, 0); 726 if (error) 727 goto out; 728 CPU_COPY(mask, &set->cs_mask); 729 cpuset_update(set, mask); 730 out: 731 mtx_unlock_spin(&cpuset_lock); 732 733 return (error); 734 } 735 736 /* 737 * Recursively check for errors that would occur from applying mask to 738 * the tree of sets starting at 'set'. Checks for sets that would become 739 * empty as well as RDONLY flags. 740 */ 741 static int 742 cpuset_testupdate_domain(struct cpuset *set, struct domainset *dset, 743 struct domainset *orig, int *count, int augment_mask __unused) 744 { 745 struct cpuset *nset; 746 struct domainset *domain; 747 struct domainset newset; 748 int error; 749 750 mtx_assert(&cpuset_lock, MA_OWNED); 751 if (set->cs_flags & CPU_SET_RDONLY) 752 return (EPERM); 753 domain = set->cs_domain; 754 domainset_copy(domain, &newset); 755 if (!domainset_equal(domain, orig)) { 756 if (!domainset_restrict(domain, dset)) 757 return (EDEADLK); 758 DOMAINSET_AND(&newset.ds_mask, &dset->ds_mask); 759 /* Count the number of domains that are changing. */ 760 (*count)++; 761 } 762 error = 0; 763 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 764 if ((error = cpuset_testupdate_domain(nset, &newset, domain, 765 count, 1)) != 0) 766 break; 767 return (error); 768 } 769 770 /* 771 * Applies the mask 'mask' without checking for empty sets or permissions. 772 */ 773 static void 774 cpuset_update_domain(struct cpuset *set, struct domainset *domain, 775 struct domainset *orig, struct domainlist *domains) 776 { 777 struct cpuset *nset; 778 779 mtx_assert(&cpuset_lock, MA_OWNED); 780 /* 781 * If this domainset has changed from the parent we must calculate 782 * a new set. Otherwise it simply inherits from the parent. When 783 * we inherit from the parent we get a new mask and policy. If the 784 * set is modified from the parent we keep the policy and only 785 * update the mask. 786 */ 787 if (set->cs_domain != orig) { 788 orig = set->cs_domain; 789 set->cs_domain = domainset_shadow(domain, orig, domains); 790 } else 791 set->cs_domain = domain; 792 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 793 cpuset_update_domain(nset, set->cs_domain, orig, domains); 794 795 return; 796 } 797 798 /* 799 * Modify the set 'set' to use a copy the domainset provided. Apply this new 800 * mask to restrict all children in the tree. Checks for validity before 801 * applying the changes. 802 */ 803 static int 804 cpuset_modify_domain(struct cpuset *set, struct domainset *domain) 805 { 806 struct domainlist domains; 807 struct domainset temp; 808 struct domainset *dset; 809 struct cpuset *root; 810 int ndomains, needed; 811 int error; 812 813 error = priv_check(curthread, PRIV_SCHED_CPUSET); 814 if (error) 815 return (error); 816 /* 817 * In case we are called from within the jail 818 * we do not allow modifying the dedicated root 819 * cpuset of the jail but may still allow to 820 * change child sets. 821 */ 822 if (jailed(curthread->td_ucred) && 823 set->cs_flags & CPU_SET_ROOT) 824 return (EPERM); 825 domainset_freelist_init(&domains, 0); 826 domain = domainset_create(domain); 827 ndomains = 0; 828 829 mtx_lock_spin(&cpuset_lock); 830 for (;;) { 831 root = cpuset_getroot(set); 832 dset = root->cs_domain; 833 /* 834 * Verify that we have access to this set of domains. 835 */ 836 if (!domainset_valid(dset, domain)) { 837 error = EINVAL; 838 goto out; 839 } 840 /* 841 * If applying prefer we keep the current set as the fallback. 842 */ 843 if (domain->ds_policy == DOMAINSET_POLICY_PREFER) 844 DOMAINSET_COPY(&set->cs_domain->ds_mask, 845 &domain->ds_mask); 846 /* 847 * Determine whether we can apply this set of domains and 848 * how many new domain structures it will require. 849 */ 850 domainset_copy(domain, &temp); 851 needed = 0; 852 error = cpuset_testupdate_domain(set, &temp, set->cs_domain, 853 &needed, 0); 854 if (error) 855 goto out; 856 if (ndomains >= needed) 857 break; 858 859 /* Dropping the lock; we'll need to re-evaluate again. */ 860 mtx_unlock_spin(&cpuset_lock); 861 domainset_freelist_add(&domains, needed - ndomains); 862 ndomains = needed; 863 mtx_lock_spin(&cpuset_lock); 864 } 865 dset = set->cs_domain; 866 cpuset_update_domain(set, domain, dset, &domains); 867 out: 868 mtx_unlock_spin(&cpuset_lock); 869 domainset_freelist_free(&domains); 870 if (error == 0) 871 domainset_notify(); 872 873 return (error); 874 } 875 876 /* 877 * Resolve the 'which' parameter of several cpuset apis. 878 * 879 * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 880 * checks for permission via p_cansched(). 881 * 882 * For WHICH_SET returns a valid set with a new reference. 883 * 884 * -1 may be supplied for any argument to mean the current proc/thread or 885 * the base set of the current thread. May fail with ESRCH/EPERM. 886 */ 887 int 888 cpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 889 struct cpuset **setp) 890 { 891 struct cpuset *set; 892 struct thread *td; 893 struct proc *p; 894 int error; 895 896 *pp = p = NULL; 897 *tdp = td = NULL; 898 *setp = set = NULL; 899 switch (which) { 900 case CPU_WHICH_PID: 901 if (id == -1) { 902 PROC_LOCK(curproc); 903 p = curproc; 904 break; 905 } 906 if ((p = pfind(id)) == NULL) 907 return (ESRCH); 908 break; 909 case CPU_WHICH_TID: 910 if (id == -1) { 911 PROC_LOCK(curproc); 912 p = curproc; 913 td = curthread; 914 break; 915 } 916 td = tdfind(id, -1); 917 if (td == NULL) 918 return (ESRCH); 919 p = td->td_proc; 920 break; 921 case CPU_WHICH_CPUSET: 922 if (id == -1) { 923 thread_lock(curthread); 924 set = cpuset_refbase(curthread->td_cpuset); 925 thread_unlock(curthread); 926 } else 927 set = cpuset_lookup(id, curthread); 928 if (set) { 929 *setp = set; 930 return (0); 931 } 932 return (ESRCH); 933 case CPU_WHICH_JAIL: 934 { 935 /* Find `set' for prison with given id. */ 936 struct prison *pr; 937 938 sx_slock(&allprison_lock); 939 pr = prison_find_child(curthread->td_ucred->cr_prison, id); 940 sx_sunlock(&allprison_lock); 941 if (pr == NULL) 942 return (ESRCH); 943 cpuset_ref(pr->pr_cpuset); 944 *setp = pr->pr_cpuset; 945 mtx_unlock(&pr->pr_mtx); 946 return (0); 947 } 948 case CPU_WHICH_IRQ: 949 case CPU_WHICH_DOMAIN: 950 return (0); 951 default: 952 return (EINVAL); 953 } 954 error = p_cansched(curthread, p); 955 if (error) { 956 PROC_UNLOCK(p); 957 return (error); 958 } 959 if (td == NULL) 960 td = FIRST_THREAD_IN_PROC(p); 961 *pp = p; 962 *tdp = td; 963 return (0); 964 } 965 966 static int 967 cpuset_testshadow(struct cpuset *set, const cpuset_t *mask, 968 const struct domainset *domain) 969 { 970 struct cpuset *parent; 971 struct domainset *dset; 972 973 parent = cpuset_getbase(set); 974 /* 975 * If we are restricting a cpu mask it must be a subset of the 976 * parent or invalid CPUs have been specified. 977 */ 978 if (mask != NULL && !CPU_SUBSET(&parent->cs_mask, mask)) 979 return (EINVAL); 980 981 /* 982 * If we are restricting a domain mask it must be a subset of the 983 * parent or invalid domains have been specified. 984 */ 985 dset = parent->cs_domain; 986 if (domain != NULL && !domainset_valid(dset, domain)) 987 return (EINVAL); 988 989 return (0); 990 } 991 992 /* 993 * Create an anonymous set with the provided mask in the space provided by 994 * 'nset'. If the passed in set is anonymous we use its parent otherwise 995 * the new set is a child of 'set'. 996 */ 997 static int 998 cpuset_shadow(struct cpuset *set, struct cpuset **nsetp, 999 const cpuset_t *mask, const struct domainset *domain, 1000 struct setlist *cpusets, struct domainlist *domains) 1001 { 1002 struct cpuset *parent; 1003 struct cpuset *nset; 1004 struct domainset *dset; 1005 struct domainset *d; 1006 int error; 1007 1008 error = cpuset_testshadow(set, mask, domain); 1009 if (error) 1010 return (error); 1011 1012 parent = cpuset_getbase(set); 1013 dset = parent->cs_domain; 1014 if (mask == NULL) 1015 mask = &set->cs_mask; 1016 if (domain != NULL) 1017 d = domainset_shadow(dset, domain, domains); 1018 else 1019 d = set->cs_domain; 1020 nset = LIST_FIRST(cpusets); 1021 error = cpuset_init(nset, parent, mask, d, CPUSET_INVALID); 1022 if (error == 0) { 1023 LIST_REMOVE(nset, cs_link); 1024 *nsetp = nset; 1025 } 1026 return (error); 1027 } 1028 1029 static struct cpuset * 1030 cpuset_update_thread(struct thread *td, struct cpuset *nset) 1031 { 1032 struct cpuset *tdset; 1033 1034 tdset = td->td_cpuset; 1035 td->td_cpuset = nset; 1036 td->td_domain.dr_policy = nset->cs_domain; 1037 sched_affinity(td); 1038 1039 return (tdset); 1040 } 1041 1042 static int 1043 cpuset_setproc_test_maskthread(struct cpuset *tdset, cpuset_t *mask, 1044 struct domainset *domain) 1045 { 1046 struct cpuset *parent; 1047 1048 parent = cpuset_getbase(tdset); 1049 if (mask == NULL) 1050 mask = &tdset->cs_mask; 1051 if (domain == NULL) 1052 domain = tdset->cs_domain; 1053 return cpuset_testshadow(parent, mask, domain); 1054 } 1055 1056 static int 1057 cpuset_setproc_maskthread(struct cpuset *tdset, cpuset_t *mask, 1058 struct domainset *domain, struct cpuset **nsetp, 1059 struct setlist *freelist, struct domainlist *domainlist) 1060 { 1061 struct cpuset *parent; 1062 1063 parent = cpuset_getbase(tdset); 1064 if (mask == NULL) 1065 mask = &tdset->cs_mask; 1066 if (domain == NULL) 1067 domain = tdset->cs_domain; 1068 return cpuset_shadow(parent, nsetp, mask, domain, freelist, 1069 domainlist); 1070 } 1071 1072 static int 1073 cpuset_setproc_setthread_mask(struct cpuset *tdset, struct cpuset *set, 1074 cpuset_t *mask, struct domainset *domain) 1075 { 1076 struct cpuset *parent; 1077 1078 parent = cpuset_getbase(tdset); 1079 1080 /* 1081 * If the thread restricted its mask then apply that same 1082 * restriction to the new set, otherwise take it wholesale. 1083 */ 1084 if (CPU_CMP(&tdset->cs_mask, &parent->cs_mask) != 0) { 1085 CPU_AND(mask, &tdset->cs_mask, &set->cs_mask); 1086 } else 1087 CPU_COPY(&set->cs_mask, mask); 1088 1089 /* 1090 * If the thread restricted the domain then we apply the 1091 * restriction to the new set but retain the policy. 1092 */ 1093 if (tdset->cs_domain != parent->cs_domain) { 1094 domainset_copy(tdset->cs_domain, domain); 1095 DOMAINSET_AND(&domain->ds_mask, &set->cs_domain->ds_mask); 1096 } else 1097 domainset_copy(set->cs_domain, domain); 1098 1099 if (CPU_EMPTY(mask) || DOMAINSET_EMPTY(&domain->ds_mask)) 1100 return (EDEADLK); 1101 1102 return (0); 1103 } 1104 1105 static int 1106 cpuset_setproc_test_setthread(struct cpuset *tdset, struct cpuset *set) 1107 { 1108 struct domainset domain; 1109 cpuset_t mask; 1110 1111 if (tdset->cs_id != CPUSET_INVALID) 1112 return (0); 1113 return cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 1114 } 1115 1116 static int 1117 cpuset_setproc_setthread(struct cpuset *tdset, struct cpuset *set, 1118 struct cpuset **nsetp, struct setlist *freelist, 1119 struct domainlist *domainlist) 1120 { 1121 struct domainset domain; 1122 cpuset_t mask; 1123 int error; 1124 1125 /* 1126 * If we're replacing on a thread that has not constrained the 1127 * original set we can simply accept the new set. 1128 */ 1129 if (tdset->cs_id != CPUSET_INVALID) { 1130 *nsetp = cpuset_ref(set); 1131 return (0); 1132 } 1133 error = cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 1134 if (error) 1135 return (error); 1136 1137 return cpuset_shadow(set, nsetp, &mask, &domain, freelist, 1138 domainlist); 1139 } 1140 1141 static int 1142 cpuset_setproc_newbase(struct thread *td, struct cpuset *set, 1143 struct cpuset *nroot, struct cpuset **nsetp, 1144 struct setlist *cpusets, struct domainlist *domainlist) 1145 { 1146 struct domainset ndomain; 1147 cpuset_t nmask; 1148 struct cpuset *pbase; 1149 int error; 1150 1151 pbase = cpuset_getbase(td->td_cpuset); 1152 1153 /* Copy process mask, then further apply the new root mask. */ 1154 CPU_AND(&nmask, &pbase->cs_mask, &nroot->cs_mask); 1155 1156 domainset_copy(pbase->cs_domain, &ndomain); 1157 DOMAINSET_AND(&ndomain.ds_mask, &set->cs_domain->ds_mask); 1158 1159 /* Policy is too restrictive, will not work. */ 1160 if (CPU_EMPTY(&nmask) || DOMAINSET_EMPTY(&ndomain.ds_mask)) 1161 return (EDEADLK); 1162 1163 /* 1164 * Remove pbase from the freelist in advance, it'll be pushed to 1165 * cpuset_ids on success. We assume here that cpuset_create() will not 1166 * touch pbase on failure, and we just enqueue it back to the freelist 1167 * to remain in a consistent state. 1168 */ 1169 pbase = LIST_FIRST(cpusets); 1170 LIST_REMOVE(pbase, cs_link); 1171 error = cpuset_create(&pbase, set, &nmask); 1172 if (error != 0) { 1173 LIST_INSERT_HEAD(cpusets, pbase, cs_link); 1174 return (error); 1175 } 1176 1177 /* Duplicates some work from above... oh well. */ 1178 pbase->cs_domain = domainset_shadow(set->cs_domain, &ndomain, 1179 domainlist); 1180 *nsetp = pbase; 1181 return (0); 1182 } 1183 1184 /* 1185 * Handle four cases for updating an entire process. 1186 * 1187 * 1) Set is non-null and the process is not rebasing onto a new root. This 1188 * reparents all anonymous sets to the provided set and replaces all 1189 * non-anonymous td_cpusets with the provided set. 1190 * 2) Set is non-null and the process is rebasing onto a new root. This 1191 * creates a new base set if the process previously had its own base set, 1192 * then reparents all anonymous sets either to that set or the provided set 1193 * if one was not created. Non-anonymous sets are similarly replaced. 1194 * 3) Mask is non-null. This replaces or creates anonymous sets for every 1195 * thread with the existing base as a parent. 1196 * 4) domain is non-null. This creates anonymous sets for every thread 1197 * and replaces the domain set. 1198 * 1199 * This is overly complicated because we can't allocate while holding a 1200 * spinlock and spinlocks must be held while changing and examining thread 1201 * state. 1202 */ 1203 static int 1204 cpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask, 1205 struct domainset *domain, bool rebase) 1206 { 1207 struct setlist freelist; 1208 struct setlist droplist; 1209 struct domainlist domainlist; 1210 struct cpuset *base, *nset, *nroot, *tdroot; 1211 struct thread *td; 1212 struct proc *p; 1213 int needed; 1214 int nfree; 1215 int error; 1216 1217 /* 1218 * The algorithm requires two passes due to locking considerations. 1219 * 1220 * 1) Lookup the process and acquire the locks in the required order. 1221 * 2) If enough cpusets have not been allocated release the locks and 1222 * allocate them. Loop. 1223 */ 1224 cpuset_freelist_init(&freelist, 1); 1225 domainset_freelist_init(&domainlist, 1); 1226 nfree = 1; 1227 LIST_INIT(&droplist); 1228 nfree = 0; 1229 base = set; 1230 nroot = NULL; 1231 if (set != NULL) 1232 nroot = cpuset_getroot(set); 1233 for (;;) { 1234 error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 1235 if (error) 1236 goto out; 1237 tdroot = cpuset_getroot(td->td_cpuset); 1238 needed = p->p_numthreads; 1239 if (set != NULL && rebase && tdroot != nroot) 1240 needed++; 1241 if (nfree >= needed) 1242 break; 1243 PROC_UNLOCK(p); 1244 if (nfree < needed) { 1245 cpuset_freelist_add(&freelist, needed - nfree); 1246 domainset_freelist_add(&domainlist, needed - nfree); 1247 nfree = needed; 1248 } 1249 } 1250 PROC_LOCK_ASSERT(p, MA_OWNED); 1251 1252 /* 1253 * If we're changing roots and the root set is what has been specified 1254 * as the parent, then we'll check if the process was previously using 1255 * the root set and, if it wasn't, create a new base with the process's 1256 * mask applied to it. 1257 * 1258 * If the new root is incompatible with the existing mask, then we allow 1259 * the process to take on the new root if and only if they have 1260 * privilege to widen their mask anyways. Unprivileged processes get 1261 * rejected with EDEADLK. 1262 */ 1263 if (set != NULL && rebase && nroot != tdroot) { 1264 cpusetid_t base_id, root_id; 1265 1266 root_id = td->td_ucred->cr_prison->pr_cpuset->cs_id; 1267 base_id = cpuset_getbase(td->td_cpuset)->cs_id; 1268 1269 if (base_id != root_id) { 1270 error = cpuset_setproc_newbase(td, set, nroot, &base, 1271 &freelist, &domainlist); 1272 if (error == EDEADLK && 1273 priv_check(td, PRIV_SCHED_CPUSET) == 0) 1274 error = 0; 1275 if (error != 0) 1276 goto unlock_out; 1277 } 1278 } 1279 1280 /* 1281 * Now that the appropriate locks are held and we have enough cpusets, 1282 * make sure the operation will succeed before applying changes. The 1283 * proc lock prevents td_cpuset from changing between calls. 1284 */ 1285 error = 0; 1286 FOREACH_THREAD_IN_PROC(p, td) { 1287 thread_lock(td); 1288 if (set != NULL) 1289 error = cpuset_setproc_test_setthread(td->td_cpuset, 1290 base); 1291 else 1292 error = cpuset_setproc_test_maskthread(td->td_cpuset, 1293 mask, domain); 1294 thread_unlock(td); 1295 if (error) 1296 goto unlock_out; 1297 } 1298 /* 1299 * Replace each thread's cpuset while using deferred release. We 1300 * must do this because the thread lock must be held while operating 1301 * on the thread and this limits the type of operations allowed. 1302 */ 1303 FOREACH_THREAD_IN_PROC(p, td) { 1304 thread_lock(td); 1305 if (set != NULL) 1306 error = cpuset_setproc_setthread(td->td_cpuset, base, 1307 &nset, &freelist, &domainlist); 1308 else 1309 error = cpuset_setproc_maskthread(td->td_cpuset, mask, 1310 domain, &nset, &freelist, &domainlist); 1311 if (error) { 1312 thread_unlock(td); 1313 break; 1314 } 1315 cpuset_rel_defer(&droplist, cpuset_update_thread(td, nset)); 1316 thread_unlock(td); 1317 } 1318 unlock_out: 1319 PROC_UNLOCK(p); 1320 out: 1321 if (base != NULL && base != set) 1322 cpuset_rel(base); 1323 while ((nset = LIST_FIRST(&droplist)) != NULL) 1324 cpuset_rel_complete(nset); 1325 cpuset_freelist_free(&freelist); 1326 domainset_freelist_free(&domainlist); 1327 return (error); 1328 } 1329 1330 static int 1331 bitset_strprint(char *buf, size_t bufsiz, const struct bitset *set, int setlen) 1332 { 1333 size_t bytes; 1334 int i, once; 1335 char *p; 1336 1337 once = 0; 1338 p = buf; 1339 for (i = 0; i < __bitset_words(setlen); i++) { 1340 if (once != 0) { 1341 if (bufsiz < 1) 1342 return (0); 1343 *p = ','; 1344 p++; 1345 bufsiz--; 1346 } else 1347 once = 1; 1348 if (bufsiz < sizeof(__STRING(ULONG_MAX))) 1349 return (0); 1350 bytes = snprintf(p, bufsiz, "%lx", set->__bits[i]); 1351 p += bytes; 1352 bufsiz -= bytes; 1353 } 1354 return (p - buf); 1355 } 1356 1357 static int 1358 bitset_strscan(struct bitset *set, int setlen, const char *buf) 1359 { 1360 int i, ret; 1361 const char *p; 1362 1363 BIT_ZERO(setlen, set); 1364 p = buf; 1365 for (i = 0; i < __bitset_words(setlen); i++) { 1366 if (*p == ',') { 1367 p++; 1368 continue; 1369 } 1370 ret = sscanf(p, "%lx", &set->__bits[i]); 1371 if (ret == 0 || ret == -1) 1372 break; 1373 while (isxdigit(*p)) 1374 p++; 1375 } 1376 return (p - buf); 1377 } 1378 1379 /* 1380 * Return a string representing a valid layout for a cpuset_t object. 1381 * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1382 */ 1383 char * 1384 cpusetobj_strprint(char *buf, const cpuset_t *set) 1385 { 1386 1387 bitset_strprint(buf, CPUSETBUFSIZ, (const struct bitset *)set, 1388 CPU_SETSIZE); 1389 return (buf); 1390 } 1391 1392 /* 1393 * Build a valid cpuset_t object from a string representation. 1394 * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1395 */ 1396 int 1397 cpusetobj_strscan(cpuset_t *set, const char *buf) 1398 { 1399 char p; 1400 1401 if (strlen(buf) > CPUSETBUFSIZ - 1) 1402 return (-1); 1403 1404 p = buf[bitset_strscan((struct bitset *)set, CPU_SETSIZE, buf)]; 1405 if (p != '\0') 1406 return (-1); 1407 1408 return (0); 1409 } 1410 1411 /* 1412 * Handle a domainset specifier in the sysctl tree. A poiner to a pointer to 1413 * a domainset is in arg1. If the user specifies a valid domainset the 1414 * pointer is updated. 1415 * 1416 * Format is: 1417 * hex mask word 0,hex mask word 1,...:decimal policy:decimal preferred 1418 */ 1419 int 1420 sysctl_handle_domainset(SYSCTL_HANDLER_ARGS) 1421 { 1422 char buf[DOMAINSETBUFSIZ]; 1423 struct domainset *dset; 1424 struct domainset key; 1425 int policy, prefer, error; 1426 char *p; 1427 1428 dset = *(struct domainset **)arg1; 1429 error = 0; 1430 1431 if (dset != NULL) { 1432 p = buf + bitset_strprint(buf, DOMAINSETBUFSIZ, 1433 (const struct bitset *)&dset->ds_mask, DOMAINSET_SETSIZE); 1434 sprintf(p, ":%d:%d", dset->ds_policy, dset->ds_prefer); 1435 } else 1436 sprintf(buf, "<NULL>"); 1437 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 1438 if (error != 0 || req->newptr == NULL) 1439 return (error); 1440 1441 /* 1442 * Read in and validate the string. 1443 */ 1444 memset(&key, 0, sizeof(key)); 1445 p = &buf[bitset_strscan((struct bitset *)&key.ds_mask, 1446 DOMAINSET_SETSIZE, buf)]; 1447 if (p == buf) 1448 return (EINVAL); 1449 if (sscanf(p, ":%d:%d", &policy, &prefer) != 2) 1450 return (EINVAL); 1451 key.ds_policy = policy; 1452 key.ds_prefer = prefer; 1453 1454 /* Domainset_create() validates the policy.*/ 1455 dset = domainset_create(&key); 1456 if (dset == NULL) 1457 return (EINVAL); 1458 *(struct domainset **)arg1 = dset; 1459 1460 return (error); 1461 } 1462 1463 /* 1464 * Apply an anonymous mask or a domain to a single thread. 1465 */ 1466 static int 1467 _cpuset_setthread(lwpid_t id, cpuset_t *mask, struct domainset *domain) 1468 { 1469 struct setlist cpusets; 1470 struct domainlist domainlist; 1471 struct cpuset *nset; 1472 struct cpuset *set; 1473 struct thread *td; 1474 struct proc *p; 1475 int error; 1476 1477 cpuset_freelist_init(&cpusets, 1); 1478 domainset_freelist_init(&domainlist, domain != NULL); 1479 error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &set); 1480 if (error) 1481 goto out; 1482 set = NULL; 1483 thread_lock(td); 1484 error = cpuset_shadow(td->td_cpuset, &nset, mask, domain, 1485 &cpusets, &domainlist); 1486 if (error == 0) 1487 set = cpuset_update_thread(td, nset); 1488 thread_unlock(td); 1489 PROC_UNLOCK(p); 1490 if (set) 1491 cpuset_rel(set); 1492 out: 1493 cpuset_freelist_free(&cpusets); 1494 domainset_freelist_free(&domainlist); 1495 return (error); 1496 } 1497 1498 /* 1499 * Apply an anonymous mask to a single thread. 1500 */ 1501 int 1502 cpuset_setthread(lwpid_t id, cpuset_t *mask) 1503 { 1504 1505 return _cpuset_setthread(id, mask, NULL); 1506 } 1507 1508 /* 1509 * Apply new cpumask to the ithread. 1510 */ 1511 int 1512 cpuset_setithread(lwpid_t id, int cpu) 1513 { 1514 cpuset_t mask; 1515 1516 CPU_ZERO(&mask); 1517 if (cpu == NOCPU) 1518 CPU_COPY(cpuset_root, &mask); 1519 else 1520 CPU_SET(cpu, &mask); 1521 return _cpuset_setthread(id, &mask, NULL); 1522 } 1523 1524 /* 1525 * Initialize static domainsets after NUMA information is available. This is 1526 * called before memory allocators are initialized. 1527 */ 1528 void 1529 domainset_init(void) 1530 { 1531 struct domainset *dset; 1532 int i; 1533 1534 dset = &domainset_firsttouch; 1535 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1536 dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; 1537 dset->ds_prefer = -1; 1538 _domainset_create(dset, NULL); 1539 1540 dset = &domainset_interleave; 1541 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1542 dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; 1543 dset->ds_prefer = -1; 1544 _domainset_create(dset, NULL); 1545 1546 dset = &domainset_roundrobin; 1547 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1548 dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 1549 dset->ds_prefer = -1; 1550 _domainset_create(dset, NULL); 1551 1552 for (i = 0; i < vm_ndomains; i++) { 1553 dset = &domainset_fixed[i]; 1554 DOMAINSET_ZERO(&dset->ds_mask); 1555 DOMAINSET_SET(i, &dset->ds_mask); 1556 dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 1557 _domainset_create(dset, NULL); 1558 1559 dset = &domainset_prefer[i]; 1560 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1561 dset->ds_policy = DOMAINSET_POLICY_PREFER; 1562 dset->ds_prefer = i; 1563 _domainset_create(dset, NULL); 1564 } 1565 } 1566 1567 /* 1568 * Define the domainsets for cpuset 0, 1 and cpuset 2. 1569 */ 1570 void 1571 domainset_zero(void) 1572 { 1573 struct domainset *dset, *tmp; 1574 1575 mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 1576 1577 domainset0 = &domainset_firsttouch; 1578 curthread->td_domain.dr_policy = domainset0; 1579 1580 domainset2 = &domainset_interleave; 1581 kernel_object->domain.dr_policy = domainset2; 1582 1583 /* Remove empty domains from the global policies. */ 1584 LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) 1585 if (domainset_empty_vm(dset)) 1586 LIST_REMOVE(dset, ds_link); 1587 } 1588 1589 /* 1590 * Creates system-wide cpusets and the cpuset for thread0 including three 1591 * sets: 1592 * 1593 * 0 - The root set which should represent all valid processors in the 1594 * system. This set is immutable. 1595 * 1 - The default set which all processes are a member of until changed. 1596 * This allows an administrator to move all threads off of given cpus to 1597 * dedicate them to high priority tasks or save power etc. 1598 * 2 - The kernel set which allows restriction and policy to be applied only 1599 * to kernel threads and the kernel_object. 1600 */ 1601 struct cpuset * 1602 cpuset_thread0(void) 1603 { 1604 struct cpuset *set; 1605 int i; 1606 int error __unused; 1607 1608 cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 1609 NULL, NULL, UMA_ALIGN_CACHE, 0); 1610 domainset_zone = uma_zcreate("domainset", sizeof(struct domainset), 1611 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 1612 1613 /* 1614 * Create the root system set (0) for the whole machine. Doesn't use 1615 * cpuset_create() due to NULL parent. 1616 */ 1617 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1618 CPU_COPY(&all_cpus, &set->cs_mask); 1619 LIST_INIT(&set->cs_children); 1620 LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 1621 refcount_init(&set->cs_ref, 1); 1622 set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; 1623 set->cs_domain = domainset0; 1624 cpuset_zero = set; 1625 cpuset_root = &set->cs_mask; 1626 1627 /* 1628 * Now derive a default (1), modifiable set from that to give out. 1629 */ 1630 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1631 error = cpuset_init(set, cpuset_zero, NULL, NULL, 1); 1632 KASSERT(error == 0, ("Error creating default set: %d\n", error)); 1633 cpuset_default = set; 1634 /* 1635 * Create the kernel set (2). 1636 */ 1637 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1638 error = cpuset_init(set, cpuset_zero, NULL, NULL, 2); 1639 KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); 1640 set->cs_domain = domainset2; 1641 cpuset_kernel = set; 1642 1643 /* 1644 * Initialize the unit allocator. 0 and 1 are allocated above. 1645 */ 1646 cpuset_unr = new_unrhdr(3, INT_MAX, NULL); 1647 1648 /* 1649 * If MD code has not initialized per-domain cpusets, place all 1650 * CPUs in domain 0. 1651 */ 1652 for (i = 0; i < MAXMEMDOM; i++) 1653 if (!CPU_EMPTY(&cpuset_domain[i])) 1654 goto domains_set; 1655 CPU_COPY(&all_cpus, &cpuset_domain[0]); 1656 domains_set: 1657 1658 return (cpuset_default); 1659 } 1660 1661 void 1662 cpuset_kernthread(struct thread *td) 1663 { 1664 struct cpuset *set; 1665 1666 thread_lock(td); 1667 set = td->td_cpuset; 1668 td->td_cpuset = cpuset_ref(cpuset_kernel); 1669 thread_unlock(td); 1670 cpuset_rel(set); 1671 } 1672 1673 /* 1674 * Create a cpuset, which would be cpuset_create() but 1675 * mark the new 'set' as root. 1676 * 1677 * We are not going to reparent the td to it. Use cpuset_setproc_update_set() 1678 * for that. 1679 * 1680 * In case of no error, returns the set in *setp locked with a reference. 1681 */ 1682 int 1683 cpuset_create_root(struct prison *pr, struct cpuset **setp) 1684 { 1685 struct cpuset *set; 1686 int error; 1687 1688 KASSERT(pr != NULL, ("[%s:%d] invalid pr", __func__, __LINE__)); 1689 KASSERT(setp != NULL, ("[%s:%d] invalid setp", __func__, __LINE__)); 1690 1691 set = NULL; 1692 error = cpuset_create(&set, pr->pr_cpuset, &pr->pr_cpuset->cs_mask); 1693 if (error) 1694 return (error); 1695 1696 KASSERT(set != NULL, ("[%s:%d] cpuset_create returned invalid data", 1697 __func__, __LINE__)); 1698 1699 /* Mark the set as root. */ 1700 set->cs_flags |= CPU_SET_ROOT; 1701 *setp = set; 1702 1703 return (0); 1704 } 1705 1706 int 1707 cpuset_setproc_update_set(struct proc *p, struct cpuset *set) 1708 { 1709 int error; 1710 1711 KASSERT(p != NULL, ("[%s:%d] invalid proc", __func__, __LINE__)); 1712 KASSERT(set != NULL, ("[%s:%d] invalid set", __func__, __LINE__)); 1713 1714 cpuset_ref(set); 1715 error = cpuset_setproc(p->p_pid, set, NULL, NULL, true); 1716 if (error) 1717 return (error); 1718 cpuset_rel(set); 1719 return (0); 1720 } 1721 1722 /* 1723 * In Capability mode, the only accesses that are permitted are to the current 1724 * thread and process' CPU and domain sets. 1725 */ 1726 static int 1727 cpuset_check_capabilities(struct thread *td, cpulevel_t level, cpuwhich_t which, 1728 id_t id) 1729 { 1730 if (IN_CAPABILITY_MODE(td)) { 1731 if (level != CPU_LEVEL_WHICH) 1732 return (ECAPMODE); 1733 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 1734 return (ECAPMODE); 1735 if (id != -1 && 1736 !(which == CPU_WHICH_TID && id == td->td_tid) && 1737 !(which == CPU_WHICH_PID && id == td->td_proc->p_pid)) 1738 return (ECAPMODE); 1739 } 1740 return (0); 1741 } 1742 1743 #ifndef _SYS_SYSPROTO_H_ 1744 struct cpuset_args { 1745 cpusetid_t *setid; 1746 }; 1747 #endif 1748 int 1749 sys_cpuset(struct thread *td, struct cpuset_args *uap) 1750 { 1751 struct cpuset *root; 1752 struct cpuset *set; 1753 int error; 1754 1755 thread_lock(td); 1756 root = cpuset_refroot(td->td_cpuset); 1757 thread_unlock(td); 1758 set = NULL; 1759 error = cpuset_create(&set, root, &root->cs_mask); 1760 cpuset_rel(root); 1761 if (error) 1762 return (error); 1763 error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 1764 if (error == 0) 1765 error = cpuset_setproc(-1, set, NULL, NULL, false); 1766 cpuset_rel(set); 1767 return (error); 1768 } 1769 1770 #ifndef _SYS_SYSPROTO_H_ 1771 struct cpuset_setid_args { 1772 cpuwhich_t which; 1773 id_t id; 1774 cpusetid_t setid; 1775 }; 1776 #endif 1777 int 1778 sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 1779 { 1780 1781 return (kern_cpuset_setid(td, uap->which, uap->id, uap->setid)); 1782 } 1783 1784 int 1785 kern_cpuset_setid(struct thread *td, cpuwhich_t which, 1786 id_t id, cpusetid_t setid) 1787 { 1788 struct cpuset *set; 1789 int error; 1790 1791 /* 1792 * Presently we only support per-process sets. 1793 */ 1794 if (which != CPU_WHICH_PID) 1795 return (EINVAL); 1796 set = cpuset_lookup(setid, td); 1797 if (set == NULL) 1798 return (ESRCH); 1799 error = cpuset_setproc(id, set, NULL, NULL, false); 1800 cpuset_rel(set); 1801 return (error); 1802 } 1803 1804 #ifndef _SYS_SYSPROTO_H_ 1805 struct cpuset_getid_args { 1806 cpulevel_t level; 1807 cpuwhich_t which; 1808 id_t id; 1809 cpusetid_t *setid; 1810 }; 1811 #endif 1812 int 1813 sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 1814 { 1815 1816 return (kern_cpuset_getid(td, uap->level, uap->which, uap->id, 1817 uap->setid)); 1818 } 1819 1820 int 1821 kern_cpuset_getid(struct thread *td, cpulevel_t level, cpuwhich_t which, 1822 id_t id, cpusetid_t *setid) 1823 { 1824 struct cpuset *nset; 1825 struct cpuset *set; 1826 struct thread *ttd; 1827 struct proc *p; 1828 cpusetid_t tmpid; 1829 int error; 1830 1831 if (level == CPU_LEVEL_WHICH && which != CPU_WHICH_CPUSET) 1832 return (EINVAL); 1833 error = cpuset_which(which, id, &p, &ttd, &set); 1834 if (error) 1835 return (error); 1836 switch (which) { 1837 case CPU_WHICH_TID: 1838 case CPU_WHICH_PID: 1839 thread_lock(ttd); 1840 set = cpuset_refbase(ttd->td_cpuset); 1841 thread_unlock(ttd); 1842 PROC_UNLOCK(p); 1843 break; 1844 case CPU_WHICH_CPUSET: 1845 case CPU_WHICH_JAIL: 1846 break; 1847 case CPU_WHICH_IRQ: 1848 case CPU_WHICH_DOMAIN: 1849 return (EINVAL); 1850 } 1851 switch (level) { 1852 case CPU_LEVEL_ROOT: 1853 nset = cpuset_refroot(set); 1854 cpuset_rel(set); 1855 set = nset; 1856 break; 1857 case CPU_LEVEL_CPUSET: 1858 break; 1859 case CPU_LEVEL_WHICH: 1860 break; 1861 } 1862 tmpid = set->cs_id; 1863 cpuset_rel(set); 1864 if (error == 0) 1865 error = copyout(&tmpid, setid, sizeof(tmpid)); 1866 1867 return (error); 1868 } 1869 1870 #ifndef _SYS_SYSPROTO_H_ 1871 struct cpuset_getaffinity_args { 1872 cpulevel_t level; 1873 cpuwhich_t which; 1874 id_t id; 1875 size_t cpusetsize; 1876 cpuset_t *mask; 1877 }; 1878 #endif 1879 int 1880 sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 1881 { 1882 1883 return (kern_cpuset_getaffinity(td, uap->level, uap->which, 1884 uap->id, uap->cpusetsize, uap->mask)); 1885 } 1886 1887 int 1888 kern_cpuset_getaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 1889 id_t id, size_t cpusetsize, cpuset_t *maskp) 1890 { 1891 struct thread *ttd; 1892 struct cpuset *nset; 1893 struct cpuset *set; 1894 struct proc *p; 1895 cpuset_t *mask; 1896 int error; 1897 size_t size; 1898 1899 if (cpusetsize < sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY) 1900 return (ERANGE); 1901 error = cpuset_check_capabilities(td, level, which, id); 1902 if (error != 0) 1903 return (error); 1904 size = cpusetsize; 1905 mask = malloc(size, M_TEMP, M_WAITOK | M_ZERO); 1906 error = cpuset_which(which, id, &p, &ttd, &set); 1907 if (error) 1908 goto out; 1909 switch (level) { 1910 case CPU_LEVEL_ROOT: 1911 case CPU_LEVEL_CPUSET: 1912 switch (which) { 1913 case CPU_WHICH_TID: 1914 case CPU_WHICH_PID: 1915 thread_lock(ttd); 1916 set = cpuset_ref(ttd->td_cpuset); 1917 thread_unlock(ttd); 1918 break; 1919 case CPU_WHICH_CPUSET: 1920 case CPU_WHICH_JAIL: 1921 break; 1922 case CPU_WHICH_IRQ: 1923 case CPU_WHICH_INTRHANDLER: 1924 case CPU_WHICH_ITHREAD: 1925 case CPU_WHICH_DOMAIN: 1926 error = EINVAL; 1927 goto out; 1928 } 1929 if (level == CPU_LEVEL_ROOT) 1930 nset = cpuset_refroot(set); 1931 else 1932 nset = cpuset_refbase(set); 1933 CPU_COPY(&nset->cs_mask, mask); 1934 cpuset_rel(nset); 1935 break; 1936 case CPU_LEVEL_WHICH: 1937 switch (which) { 1938 case CPU_WHICH_TID: 1939 thread_lock(ttd); 1940 CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 1941 thread_unlock(ttd); 1942 break; 1943 case CPU_WHICH_PID: 1944 FOREACH_THREAD_IN_PROC(p, ttd) { 1945 thread_lock(ttd); 1946 CPU_OR(mask, mask, &ttd->td_cpuset->cs_mask); 1947 thread_unlock(ttd); 1948 } 1949 break; 1950 case CPU_WHICH_CPUSET: 1951 case CPU_WHICH_JAIL: 1952 CPU_COPY(&set->cs_mask, mask); 1953 break; 1954 case CPU_WHICH_IRQ: 1955 case CPU_WHICH_INTRHANDLER: 1956 case CPU_WHICH_ITHREAD: 1957 error = intr_getaffinity(id, which, mask); 1958 break; 1959 case CPU_WHICH_DOMAIN: 1960 if (id < 0 || id >= MAXMEMDOM) 1961 error = ESRCH; 1962 else 1963 CPU_COPY(&cpuset_domain[id], mask); 1964 break; 1965 } 1966 break; 1967 default: 1968 error = EINVAL; 1969 break; 1970 } 1971 if (set) 1972 cpuset_rel(set); 1973 if (p) 1974 PROC_UNLOCK(p); 1975 if (error == 0) 1976 error = copyout(mask, maskp, size); 1977 out: 1978 free(mask, M_TEMP); 1979 return (error); 1980 } 1981 1982 #ifndef _SYS_SYSPROTO_H_ 1983 struct cpuset_setaffinity_args { 1984 cpulevel_t level; 1985 cpuwhich_t which; 1986 id_t id; 1987 size_t cpusetsize; 1988 const cpuset_t *mask; 1989 }; 1990 #endif 1991 int 1992 sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 1993 { 1994 1995 return (kern_cpuset_setaffinity(td, uap->level, uap->which, 1996 uap->id, uap->cpusetsize, uap->mask)); 1997 } 1998 1999 int 2000 kern_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 2001 id_t id, size_t cpusetsize, const cpuset_t *maskp) 2002 { 2003 struct cpuset *nset; 2004 struct cpuset *set; 2005 struct thread *ttd; 2006 struct proc *p; 2007 cpuset_t *mask; 2008 int error; 2009 2010 if (cpusetsize < sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY) 2011 return (ERANGE); 2012 error = cpuset_check_capabilities(td, level, which, id); 2013 if (error != 0) 2014 return (error); 2015 mask = malloc(cpusetsize, M_TEMP, M_WAITOK | M_ZERO); 2016 error = copyin(maskp, mask, cpusetsize); 2017 if (error) 2018 goto out; 2019 /* 2020 * Verify that no high bits are set. 2021 */ 2022 if (cpusetsize > sizeof(cpuset_t)) { 2023 char *end; 2024 char *cp; 2025 2026 end = cp = (char *)&mask->__bits; 2027 end += cpusetsize; 2028 cp += sizeof(cpuset_t); 2029 while (cp != end) 2030 if (*cp++ != 0) { 2031 error = EINVAL; 2032 goto out; 2033 } 2034 } 2035 if (CPU_EMPTY(mask)) { 2036 error = EDEADLK; 2037 goto out; 2038 } 2039 switch (level) { 2040 case CPU_LEVEL_ROOT: 2041 case CPU_LEVEL_CPUSET: 2042 error = cpuset_which(which, id, &p, &ttd, &set); 2043 if (error) 2044 break; 2045 switch (which) { 2046 case CPU_WHICH_TID: 2047 case CPU_WHICH_PID: 2048 thread_lock(ttd); 2049 set = cpuset_ref(ttd->td_cpuset); 2050 thread_unlock(ttd); 2051 PROC_UNLOCK(p); 2052 break; 2053 case CPU_WHICH_CPUSET: 2054 case CPU_WHICH_JAIL: 2055 break; 2056 case CPU_WHICH_IRQ: 2057 case CPU_WHICH_INTRHANDLER: 2058 case CPU_WHICH_ITHREAD: 2059 case CPU_WHICH_DOMAIN: 2060 error = EINVAL; 2061 goto out; 2062 } 2063 if (level == CPU_LEVEL_ROOT) 2064 nset = cpuset_refroot(set); 2065 else 2066 nset = cpuset_refbase(set); 2067 error = cpuset_modify(nset, mask); 2068 cpuset_rel(nset); 2069 cpuset_rel(set); 2070 break; 2071 case CPU_LEVEL_WHICH: 2072 switch (which) { 2073 case CPU_WHICH_TID: 2074 error = cpuset_setthread(id, mask); 2075 break; 2076 case CPU_WHICH_PID: 2077 error = cpuset_setproc(id, NULL, mask, NULL, false); 2078 break; 2079 case CPU_WHICH_CPUSET: 2080 case CPU_WHICH_JAIL: 2081 error = cpuset_which(which, id, &p, &ttd, &set); 2082 if (error == 0) { 2083 error = cpuset_modify(set, mask); 2084 cpuset_rel(set); 2085 } 2086 break; 2087 case CPU_WHICH_IRQ: 2088 case CPU_WHICH_INTRHANDLER: 2089 case CPU_WHICH_ITHREAD: 2090 error = intr_setaffinity(id, which, mask); 2091 break; 2092 default: 2093 error = EINVAL; 2094 break; 2095 } 2096 break; 2097 default: 2098 error = EINVAL; 2099 break; 2100 } 2101 out: 2102 free(mask, M_TEMP); 2103 return (error); 2104 } 2105 2106 #ifndef _SYS_SYSPROTO_H_ 2107 struct cpuset_getdomain_args { 2108 cpulevel_t level; 2109 cpuwhich_t which; 2110 id_t id; 2111 size_t domainsetsize; 2112 domainset_t *mask; 2113 int *policy; 2114 }; 2115 #endif 2116 int 2117 sys_cpuset_getdomain(struct thread *td, struct cpuset_getdomain_args *uap) 2118 { 2119 2120 return (kern_cpuset_getdomain(td, uap->level, uap->which, 2121 uap->id, uap->domainsetsize, uap->mask, uap->policy)); 2122 } 2123 2124 int 2125 kern_cpuset_getdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 2126 id_t id, size_t domainsetsize, domainset_t *maskp, int *policyp) 2127 { 2128 struct domainset outset; 2129 struct thread *ttd; 2130 struct cpuset *nset; 2131 struct cpuset *set; 2132 struct domainset *dset; 2133 struct proc *p; 2134 domainset_t *mask; 2135 int error; 2136 2137 if (domainsetsize < sizeof(domainset_t) || 2138 domainsetsize > DOMAINSET_MAXSIZE / NBBY) 2139 return (ERANGE); 2140 error = cpuset_check_capabilities(td, level, which, id); 2141 if (error != 0) 2142 return (error); 2143 mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 2144 bzero(&outset, sizeof(outset)); 2145 error = cpuset_which(which, id, &p, &ttd, &set); 2146 if (error) 2147 goto out; 2148 switch (level) { 2149 case CPU_LEVEL_ROOT: 2150 case CPU_LEVEL_CPUSET: 2151 switch (which) { 2152 case CPU_WHICH_TID: 2153 case CPU_WHICH_PID: 2154 thread_lock(ttd); 2155 set = cpuset_ref(ttd->td_cpuset); 2156 thread_unlock(ttd); 2157 break; 2158 case CPU_WHICH_CPUSET: 2159 case CPU_WHICH_JAIL: 2160 break; 2161 case CPU_WHICH_IRQ: 2162 case CPU_WHICH_INTRHANDLER: 2163 case CPU_WHICH_ITHREAD: 2164 case CPU_WHICH_DOMAIN: 2165 error = EINVAL; 2166 goto out; 2167 } 2168 if (level == CPU_LEVEL_ROOT) 2169 nset = cpuset_refroot(set); 2170 else 2171 nset = cpuset_refbase(set); 2172 domainset_copy(nset->cs_domain, &outset); 2173 cpuset_rel(nset); 2174 break; 2175 case CPU_LEVEL_WHICH: 2176 switch (which) { 2177 case CPU_WHICH_TID: 2178 thread_lock(ttd); 2179 domainset_copy(ttd->td_cpuset->cs_domain, &outset); 2180 thread_unlock(ttd); 2181 break; 2182 case CPU_WHICH_PID: 2183 FOREACH_THREAD_IN_PROC(p, ttd) { 2184 thread_lock(ttd); 2185 dset = ttd->td_cpuset->cs_domain; 2186 /* Show all domains in the proc. */ 2187 DOMAINSET_OR(&outset.ds_mask, &dset->ds_mask); 2188 /* Last policy wins. */ 2189 outset.ds_policy = dset->ds_policy; 2190 outset.ds_prefer = dset->ds_prefer; 2191 thread_unlock(ttd); 2192 } 2193 break; 2194 case CPU_WHICH_CPUSET: 2195 case CPU_WHICH_JAIL: 2196 domainset_copy(set->cs_domain, &outset); 2197 break; 2198 case CPU_WHICH_IRQ: 2199 case CPU_WHICH_INTRHANDLER: 2200 case CPU_WHICH_ITHREAD: 2201 case CPU_WHICH_DOMAIN: 2202 error = EINVAL; 2203 break; 2204 } 2205 break; 2206 default: 2207 error = EINVAL; 2208 break; 2209 } 2210 if (set) 2211 cpuset_rel(set); 2212 if (p) 2213 PROC_UNLOCK(p); 2214 /* 2215 * Translate prefer into a set containing only the preferred domain, 2216 * not the entire fallback set. 2217 */ 2218 if (outset.ds_policy == DOMAINSET_POLICY_PREFER) { 2219 DOMAINSET_ZERO(&outset.ds_mask); 2220 DOMAINSET_SET(outset.ds_prefer, &outset.ds_mask); 2221 } 2222 DOMAINSET_COPY(&outset.ds_mask, mask); 2223 if (error == 0) 2224 error = copyout(mask, maskp, domainsetsize); 2225 if (error == 0) 2226 if (suword32(policyp, outset.ds_policy) != 0) 2227 error = EFAULT; 2228 out: 2229 free(mask, M_TEMP); 2230 return (error); 2231 } 2232 2233 #ifndef _SYS_SYSPROTO_H_ 2234 struct cpuset_setdomain_args { 2235 cpulevel_t level; 2236 cpuwhich_t which; 2237 id_t id; 2238 size_t domainsetsize; 2239 domainset_t *mask; 2240 int policy; 2241 }; 2242 #endif 2243 int 2244 sys_cpuset_setdomain(struct thread *td, struct cpuset_setdomain_args *uap) 2245 { 2246 2247 return (kern_cpuset_setdomain(td, uap->level, uap->which, 2248 uap->id, uap->domainsetsize, uap->mask, uap->policy)); 2249 } 2250 2251 int 2252 kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 2253 id_t id, size_t domainsetsize, const domainset_t *maskp, int policy) 2254 { 2255 struct cpuset *nset; 2256 struct cpuset *set; 2257 struct thread *ttd; 2258 struct proc *p; 2259 struct domainset domain; 2260 domainset_t *mask; 2261 int error; 2262 2263 if (domainsetsize < sizeof(domainset_t) || 2264 domainsetsize > DOMAINSET_MAXSIZE / NBBY) 2265 return (ERANGE); 2266 if (policy <= DOMAINSET_POLICY_INVALID || 2267 policy > DOMAINSET_POLICY_MAX) 2268 return (EINVAL); 2269 error = cpuset_check_capabilities(td, level, which, id); 2270 if (error != 0) 2271 return (error); 2272 memset(&domain, 0, sizeof(domain)); 2273 mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 2274 error = copyin(maskp, mask, domainsetsize); 2275 if (error) 2276 goto out; 2277 /* 2278 * Verify that no high bits are set. 2279 */ 2280 if (domainsetsize > sizeof(domainset_t)) { 2281 char *end; 2282 char *cp; 2283 2284 end = cp = (char *)&mask->__bits; 2285 end += domainsetsize; 2286 cp += sizeof(domainset_t); 2287 while (cp != end) 2288 if (*cp++ != 0) { 2289 error = EINVAL; 2290 goto out; 2291 } 2292 } 2293 if (DOMAINSET_EMPTY(mask)) { 2294 error = EDEADLK; 2295 goto out; 2296 } 2297 DOMAINSET_COPY(mask, &domain.ds_mask); 2298 domain.ds_policy = policy; 2299 2300 /* 2301 * Sanitize the provided mask. 2302 */ 2303 if (!DOMAINSET_SUBSET(&all_domains, &domain.ds_mask)) { 2304 error = EINVAL; 2305 goto out; 2306 } 2307 2308 /* Translate preferred policy into a mask and fallback. */ 2309 if (policy == DOMAINSET_POLICY_PREFER) { 2310 /* Only support a single preferred domain. */ 2311 if (DOMAINSET_COUNT(&domain.ds_mask) != 1) { 2312 error = EINVAL; 2313 goto out; 2314 } 2315 domain.ds_prefer = DOMAINSET_FFS(&domain.ds_mask) - 1; 2316 /* This will be constrained by domainset_shadow(). */ 2317 DOMAINSET_COPY(&all_domains, &domain.ds_mask); 2318 } 2319 2320 /* 2321 * When given an impossible policy, fall back to interleaving 2322 * across all domains. 2323 */ 2324 if (domainset_empty_vm(&domain)) 2325 domainset_copy(domainset2, &domain); 2326 2327 switch (level) { 2328 case CPU_LEVEL_ROOT: 2329 case CPU_LEVEL_CPUSET: 2330 error = cpuset_which(which, id, &p, &ttd, &set); 2331 if (error) 2332 break; 2333 switch (which) { 2334 case CPU_WHICH_TID: 2335 case CPU_WHICH_PID: 2336 thread_lock(ttd); 2337 set = cpuset_ref(ttd->td_cpuset); 2338 thread_unlock(ttd); 2339 PROC_UNLOCK(p); 2340 break; 2341 case CPU_WHICH_CPUSET: 2342 case CPU_WHICH_JAIL: 2343 break; 2344 case CPU_WHICH_IRQ: 2345 case CPU_WHICH_INTRHANDLER: 2346 case CPU_WHICH_ITHREAD: 2347 case CPU_WHICH_DOMAIN: 2348 error = EINVAL; 2349 goto out; 2350 } 2351 if (level == CPU_LEVEL_ROOT) 2352 nset = cpuset_refroot(set); 2353 else 2354 nset = cpuset_refbase(set); 2355 error = cpuset_modify_domain(nset, &domain); 2356 cpuset_rel(nset); 2357 cpuset_rel(set); 2358 break; 2359 case CPU_LEVEL_WHICH: 2360 switch (which) { 2361 case CPU_WHICH_TID: 2362 error = _cpuset_setthread(id, NULL, &domain); 2363 break; 2364 case CPU_WHICH_PID: 2365 error = cpuset_setproc(id, NULL, NULL, &domain, false); 2366 break; 2367 case CPU_WHICH_CPUSET: 2368 case CPU_WHICH_JAIL: 2369 error = cpuset_which(which, id, &p, &ttd, &set); 2370 if (error == 0) { 2371 error = cpuset_modify_domain(set, &domain); 2372 cpuset_rel(set); 2373 } 2374 break; 2375 case CPU_WHICH_IRQ: 2376 case CPU_WHICH_INTRHANDLER: 2377 case CPU_WHICH_ITHREAD: 2378 default: 2379 error = EINVAL; 2380 break; 2381 } 2382 break; 2383 default: 2384 error = EINVAL; 2385 break; 2386 } 2387 out: 2388 free(mask, M_TEMP); 2389 return (error); 2390 } 2391 2392 #ifdef DDB 2393 2394 static void 2395 ddb_display_bitset(const struct bitset *set, int size) 2396 { 2397 int bit, once; 2398 2399 for (once = 0, bit = 0; bit < size; bit++) { 2400 if (CPU_ISSET(bit, set)) { 2401 if (once == 0) { 2402 db_printf("%d", bit); 2403 once = 1; 2404 } else 2405 db_printf(",%d", bit); 2406 } 2407 } 2408 if (once == 0) 2409 db_printf("<none>"); 2410 } 2411 2412 void 2413 ddb_display_cpuset(const cpuset_t *set) 2414 { 2415 ddb_display_bitset((const struct bitset *)set, CPU_SETSIZE); 2416 } 2417 2418 static void 2419 ddb_display_domainset(const domainset_t *set) 2420 { 2421 ddb_display_bitset((const struct bitset *)set, DOMAINSET_SETSIZE); 2422 } 2423 2424 DB_SHOW_COMMAND(cpusets, db_show_cpusets) 2425 { 2426 struct cpuset *set; 2427 2428 LIST_FOREACH(set, &cpuset_ids, cs_link) { 2429 db_printf("set=%p id=%-6u ref=%-6d flags=0x%04x parent id=%d\n", 2430 set, set->cs_id, refcount_load(&set->cs_ref), set->cs_flags, 2431 (set->cs_parent != NULL) ? set->cs_parent->cs_id : 0); 2432 db_printf(" cpu mask="); 2433 ddb_display_cpuset(&set->cs_mask); 2434 db_printf("\n"); 2435 db_printf(" domain policy %d prefer %d mask=", 2436 set->cs_domain->ds_policy, set->cs_domain->ds_prefer); 2437 ddb_display_domainset(&set->cs_domain->ds_mask); 2438 db_printf("\n"); 2439 if (db_pager_quit) 2440 break; 2441 } 2442 } 2443 2444 DB_SHOW_COMMAND(domainsets, db_show_domainsets) 2445 { 2446 struct domainset *set; 2447 2448 LIST_FOREACH(set, &cpuset_domains, ds_link) { 2449 db_printf("set=%p policy %d prefer %d cnt %d\n", 2450 set, set->ds_policy, set->ds_prefer, set->ds_cnt); 2451 db_printf(" mask ="); 2452 ddb_display_domainset(&set->ds_mask); 2453 db_printf("\n"); 2454 } 2455 } 2456 #endif /* DDB */ 2457