1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 5 * All rights reserved. 6 * 7 * Copyright (c) 2008 Nokia Corporation 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_ddb.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/sysctl.h> 41 #include <sys/ctype.h> 42 #include <sys/sysproto.h> 43 #include <sys/jail.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/mutex.h> 48 #include <sys/priv.h> 49 #include <sys/proc.h> 50 #include <sys/refcount.h> 51 #include <sys/sched.h> 52 #include <sys/smp.h> 53 #include <sys/syscallsubr.h> 54 #include <sys/capsicum.h> 55 #include <sys/cpuset.h> 56 #include <sys/domainset.h> 57 #include <sys/sx.h> 58 #include <sys/queue.h> 59 #include <sys/libkern.h> 60 #include <sys/limits.h> 61 #include <sys/bus.h> 62 #include <sys/interrupt.h> 63 #include <sys/vmmeter.h> 64 65 #include <vm/uma.h> 66 #include <vm/vm.h> 67 #include <vm/vm_object.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_pageout.h> 70 #include <vm/vm_extern.h> 71 #include <vm/vm_param.h> 72 #include <vm/vm_phys.h> 73 #include <vm/vm_pagequeue.h> 74 75 #ifdef DDB 76 #include <ddb/ddb.h> 77 #endif /* DDB */ 78 79 /* 80 * cpusets provide a mechanism for creating and manipulating sets of 81 * processors for the purpose of constraining the scheduling of threads to 82 * specific processors. 83 * 84 * Each process belongs to an identified set, by default this is set 1. Each 85 * thread may further restrict the cpus it may run on to a subset of this 86 * named set. This creates an anonymous set which other threads and processes 87 * may not join by number. 88 * 89 * The named set is referred to herein as the 'base' set to avoid ambiguity. 90 * This set is usually a child of a 'root' set while the anonymous set may 91 * simply be referred to as a mask. In the syscall api these are referred to 92 * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 93 * 94 * Threads inherit their set from their creator whether it be anonymous or 95 * not. This means that anonymous sets are immutable because they may be 96 * shared. To modify an anonymous set a new set is created with the desired 97 * mask and the same parent as the existing anonymous set. This gives the 98 * illusion of each thread having a private mask. 99 * 100 * Via the syscall apis a user may ask to retrieve or modify the root, base, 101 * or mask that is discovered via a pid, tid, or setid. Modifying a set 102 * modifies all numbered and anonymous child sets to comply with the new mask. 103 * Modifying a pid or tid's mask applies only to that tid but must still 104 * exist within the assigned parent set. 105 * 106 * A thread may not be assigned to a group separate from other threads in 107 * the process. This is to remove ambiguity when the setid is queried with 108 * a pid argument. There is no other technical limitation. 109 * 110 * This somewhat complex arrangement is intended to make it easy for 111 * applications to query available processors and bind their threads to 112 * specific processors while also allowing administrators to dynamically 113 * reprovision by changing sets which apply to groups of processes. 114 * 115 * A simple application should not concern itself with sets at all and 116 * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 117 * meaning 'curthread'. It may query available cpus for that tid with a 118 * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 119 */ 120 121 LIST_HEAD(domainlist, domainset); 122 struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; 123 struct domainset __read_mostly domainset_roundrobin; 124 125 static uma_zone_t cpuset_zone; 126 static uma_zone_t domainset_zone; 127 static struct mtx cpuset_lock; 128 static struct setlist cpuset_ids; 129 static struct domainlist cpuset_domains; 130 static struct unrhdr *cpuset_unr; 131 static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; 132 static struct domainset domainset0, domainset2; 133 134 /* Return the size of cpuset_t at the kernel level */ 135 SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, 136 SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); 137 138 cpuset_t *cpuset_root; 139 cpuset_t cpuset_domain[MAXMEMDOM]; 140 141 static int domainset_valid(const struct domainset *, const struct domainset *); 142 143 /* 144 * Find the first non-anonymous set starting from 'set'. 145 */ 146 static struct cpuset * 147 cpuset_getbase(struct cpuset *set) 148 { 149 150 if (set->cs_id == CPUSET_INVALID) 151 set = set->cs_parent; 152 return (set); 153 } 154 155 /* 156 * Walks up the tree from 'set' to find the root. 157 */ 158 static struct cpuset * 159 cpuset_getroot(struct cpuset *set) 160 { 161 162 while ((set->cs_flags & CPU_SET_ROOT) == 0 && set->cs_parent != NULL) 163 set = set->cs_parent; 164 return (set); 165 } 166 167 /* 168 * Acquire a reference to a cpuset, all pointers must be tracked with refs. 169 */ 170 struct cpuset * 171 cpuset_ref(struct cpuset *set) 172 { 173 174 refcount_acquire(&set->cs_ref); 175 return (set); 176 } 177 178 /* 179 * Walks up the tree from 'set' to find the root. Returns the root 180 * referenced. 181 */ 182 static struct cpuset * 183 cpuset_refroot(struct cpuset *set) 184 { 185 186 return (cpuset_ref(cpuset_getroot(set))); 187 } 188 189 /* 190 * Find the first non-anonymous set starting from 'set'. Returns this set 191 * referenced. May return the passed in set with an extra ref if it is 192 * not anonymous. 193 */ 194 static struct cpuset * 195 cpuset_refbase(struct cpuset *set) 196 { 197 198 return (cpuset_ref(cpuset_getbase(set))); 199 } 200 201 /* 202 * Release a reference in a context where it is safe to allocate. 203 */ 204 void 205 cpuset_rel(struct cpuset *set) 206 { 207 cpusetid_t id; 208 209 if (refcount_release(&set->cs_ref) == 0) 210 return; 211 mtx_lock_spin(&cpuset_lock); 212 LIST_REMOVE(set, cs_siblings); 213 id = set->cs_id; 214 if (id != CPUSET_INVALID) 215 LIST_REMOVE(set, cs_link); 216 mtx_unlock_spin(&cpuset_lock); 217 cpuset_rel(set->cs_parent); 218 uma_zfree(cpuset_zone, set); 219 if (id != CPUSET_INVALID) 220 free_unr(cpuset_unr, id); 221 } 222 223 /* 224 * Deferred release must be used when in a context that is not safe to 225 * allocate/free. This places any unreferenced sets on the list 'head'. 226 */ 227 static void 228 cpuset_rel_defer(struct setlist *head, struct cpuset *set) 229 { 230 231 if (refcount_release(&set->cs_ref) == 0) 232 return; 233 mtx_lock_spin(&cpuset_lock); 234 LIST_REMOVE(set, cs_siblings); 235 if (set->cs_id != CPUSET_INVALID) 236 LIST_REMOVE(set, cs_link); 237 LIST_INSERT_HEAD(head, set, cs_link); 238 mtx_unlock_spin(&cpuset_lock); 239 } 240 241 /* 242 * Complete a deferred release. Removes the set from the list provided to 243 * cpuset_rel_defer. 244 */ 245 static void 246 cpuset_rel_complete(struct cpuset *set) 247 { 248 LIST_REMOVE(set, cs_link); 249 cpuset_rel(set->cs_parent); 250 uma_zfree(cpuset_zone, set); 251 } 252 253 /* 254 * Find a set based on an id. Returns it with a ref. 255 */ 256 static struct cpuset * 257 cpuset_lookup(cpusetid_t setid, struct thread *td) 258 { 259 struct cpuset *set; 260 261 if (setid == CPUSET_INVALID) 262 return (NULL); 263 mtx_lock_spin(&cpuset_lock); 264 LIST_FOREACH(set, &cpuset_ids, cs_link) 265 if (set->cs_id == setid) 266 break; 267 if (set) 268 cpuset_ref(set); 269 mtx_unlock_spin(&cpuset_lock); 270 271 KASSERT(td != NULL, ("[%s:%d] td is NULL", __func__, __LINE__)); 272 if (set != NULL && jailed(td->td_ucred)) { 273 struct cpuset *jset, *tset; 274 275 jset = td->td_ucred->cr_prison->pr_cpuset; 276 for (tset = set; tset != NULL; tset = tset->cs_parent) 277 if (tset == jset) 278 break; 279 if (tset == NULL) { 280 cpuset_rel(set); 281 set = NULL; 282 } 283 } 284 285 return (set); 286 } 287 288 /* 289 * Create a set in the space provided in 'set' with the provided parameters. 290 * The set is returned with a single ref. May return EDEADLK if the set 291 * will have no valid cpu based on restrictions from the parent. 292 */ 293 static int 294 _cpuset_create(struct cpuset *set, struct cpuset *parent, 295 const cpuset_t *mask, struct domainset *domain, cpusetid_t id) 296 { 297 298 if (domain == NULL) 299 domain = parent->cs_domain; 300 if (mask == NULL) 301 mask = &parent->cs_mask; 302 if (!CPU_OVERLAP(&parent->cs_mask, mask)) 303 return (EDEADLK); 304 /* The domain must be prepared ahead of time. */ 305 if (!domainset_valid(parent->cs_domain, domain)) 306 return (EDEADLK); 307 CPU_COPY(mask, &set->cs_mask); 308 LIST_INIT(&set->cs_children); 309 refcount_init(&set->cs_ref, 1); 310 set->cs_flags = 0; 311 mtx_lock_spin(&cpuset_lock); 312 set->cs_domain = domain; 313 CPU_AND(&set->cs_mask, &parent->cs_mask); 314 set->cs_id = id; 315 set->cs_parent = cpuset_ref(parent); 316 LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 317 if (set->cs_id != CPUSET_INVALID) 318 LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 319 mtx_unlock_spin(&cpuset_lock); 320 321 return (0); 322 } 323 324 /* 325 * Create a new non-anonymous set with the requested parent and mask. May 326 * return failures if the mask is invalid or a new number can not be 327 * allocated. 328 */ 329 static int 330 cpuset_create(struct cpuset **setp, struct cpuset *parent, const cpuset_t *mask) 331 { 332 struct cpuset *set; 333 cpusetid_t id; 334 int error; 335 336 id = alloc_unr(cpuset_unr); 337 if (id == -1) 338 return (ENFILE); 339 *setp = set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 340 error = _cpuset_create(set, parent, mask, NULL, id); 341 if (error == 0) 342 return (0); 343 free_unr(cpuset_unr, id); 344 uma_zfree(cpuset_zone, set); 345 346 return (error); 347 } 348 349 static void 350 cpuset_freelist_add(struct setlist *list, int count) 351 { 352 struct cpuset *set; 353 int i; 354 355 for (i = 0; i < count; i++) { 356 set = uma_zalloc(cpuset_zone, M_ZERO | M_WAITOK); 357 LIST_INSERT_HEAD(list, set, cs_link); 358 } 359 } 360 361 static void 362 cpuset_freelist_init(struct setlist *list, int count) 363 { 364 365 LIST_INIT(list); 366 cpuset_freelist_add(list, count); 367 } 368 369 static void 370 cpuset_freelist_free(struct setlist *list) 371 { 372 struct cpuset *set; 373 374 while ((set = LIST_FIRST(list)) != NULL) { 375 LIST_REMOVE(set, cs_link); 376 uma_zfree(cpuset_zone, set); 377 } 378 } 379 380 static void 381 domainset_freelist_add(struct domainlist *list, int count) 382 { 383 struct domainset *set; 384 int i; 385 386 for (i = 0; i < count; i++) { 387 set = uma_zalloc(domainset_zone, M_ZERO | M_WAITOK); 388 LIST_INSERT_HEAD(list, set, ds_link); 389 } 390 } 391 392 static void 393 domainset_freelist_init(struct domainlist *list, int count) 394 { 395 396 LIST_INIT(list); 397 domainset_freelist_add(list, count); 398 } 399 400 static void 401 domainset_freelist_free(struct domainlist *list) 402 { 403 struct domainset *set; 404 405 while ((set = LIST_FIRST(list)) != NULL) { 406 LIST_REMOVE(set, ds_link); 407 uma_zfree(domainset_zone, set); 408 } 409 } 410 411 /* Copy a domainset preserving mask and policy. */ 412 static void 413 domainset_copy(const struct domainset *from, struct domainset *to) 414 { 415 416 DOMAINSET_COPY(&from->ds_mask, &to->ds_mask); 417 to->ds_policy = from->ds_policy; 418 to->ds_prefer = from->ds_prefer; 419 } 420 421 /* Return 1 if mask and policy are equal, otherwise 0. */ 422 static int 423 domainset_equal(const struct domainset *one, const struct domainset *two) 424 { 425 426 return (DOMAINSET_CMP(&one->ds_mask, &two->ds_mask) == 0 && 427 one->ds_policy == two->ds_policy && 428 one->ds_prefer == two->ds_prefer); 429 } 430 431 /* Return 1 if child is a valid subset of parent. */ 432 static int 433 domainset_valid(const struct domainset *parent, const struct domainset *child) 434 { 435 if (child->ds_policy != DOMAINSET_POLICY_PREFER) 436 return (DOMAINSET_SUBSET(&parent->ds_mask, &child->ds_mask)); 437 return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 438 } 439 440 static int 441 domainset_restrict(const struct domainset *parent, 442 const struct domainset *child) 443 { 444 if (child->ds_policy != DOMAINSET_POLICY_PREFER) 445 return (DOMAINSET_OVERLAP(&parent->ds_mask, &child->ds_mask)); 446 return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 447 } 448 449 /* 450 * Lookup or create a domainset. The key is provided in ds_mask and 451 * ds_policy. If the domainset does not yet exist the storage in 452 * 'domain' is used to insert. Otherwise this storage is freed to the 453 * domainset_zone and the existing domainset is returned. 454 */ 455 static struct domainset * 456 _domainset_create(struct domainset *domain, struct domainlist *freelist) 457 { 458 struct domainset *ndomain; 459 int i, j, max; 460 461 KASSERT(domain->ds_cnt <= vm_ndomains, 462 ("invalid domain count in domainset %p", domain)); 463 KASSERT(domain->ds_policy != DOMAINSET_POLICY_PREFER || 464 domain->ds_prefer < vm_ndomains, 465 ("invalid preferred domain in domains %p", domain)); 466 467 mtx_lock_spin(&cpuset_lock); 468 LIST_FOREACH(ndomain, &cpuset_domains, ds_link) 469 if (domainset_equal(ndomain, domain)) 470 break; 471 /* 472 * If the domain does not yet exist we insert it and initialize 473 * various iteration helpers which are not part of the key. 474 */ 475 if (ndomain == NULL) { 476 LIST_INSERT_HEAD(&cpuset_domains, domain, ds_link); 477 domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 478 max = DOMAINSET_FLS(&domain->ds_mask) + 1; 479 for (i = 0, j = 0; i < max; i++) 480 if (DOMAINSET_ISSET(i, &domain->ds_mask)) 481 domain->ds_order[j++] = i; 482 } 483 mtx_unlock_spin(&cpuset_lock); 484 if (ndomain == NULL) 485 return (domain); 486 if (freelist != NULL) 487 LIST_INSERT_HEAD(freelist, domain, ds_link); 488 else 489 uma_zfree(domainset_zone, domain); 490 return (ndomain); 491 492 } 493 494 /* 495 * Are any of the domains in the mask empty? If so, silently 496 * remove them. If only empty domains are present, we must 497 * return failure. 498 */ 499 static bool 500 domainset_empty_vm(struct domainset *domain) 501 { 502 int i, max; 503 504 max = DOMAINSET_FLS(&domain->ds_mask) + 1; 505 for (i = 0; i < max; i++) { 506 if (DOMAINSET_ISSET(i, &domain->ds_mask) && 507 VM_DOMAIN_EMPTY(i)) 508 DOMAINSET_CLR(i, &domain->ds_mask); 509 } 510 511 return (DOMAINSET_EMPTY(&domain->ds_mask)); 512 } 513 514 /* 515 * Create or lookup a domainset based on the key held in 'domain'. 516 */ 517 struct domainset * 518 domainset_create(const struct domainset *domain) 519 { 520 struct domainset *ndomain; 521 522 /* 523 * Validate the policy. It must specify a useable policy number with 524 * only valid domains. Preferred must include the preferred domain 525 * in the mask. 526 */ 527 if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || 528 domain->ds_policy > DOMAINSET_POLICY_MAX) 529 return (NULL); 530 if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 531 !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) 532 return (NULL); 533 if (!DOMAINSET_SUBSET(&domainset0.ds_mask, &domain->ds_mask)) 534 return (NULL); 535 ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); 536 domainset_copy(domain, ndomain); 537 return _domainset_create(ndomain, NULL); 538 } 539 540 /* 541 * Update thread domainset pointers. 542 */ 543 static void 544 domainset_notify(void) 545 { 546 struct thread *td; 547 struct proc *p; 548 549 sx_slock(&allproc_lock); 550 FOREACH_PROC_IN_SYSTEM(p) { 551 PROC_LOCK(p); 552 if (p->p_state == PRS_NEW) { 553 PROC_UNLOCK(p); 554 continue; 555 } 556 FOREACH_THREAD_IN_PROC(p, td) { 557 thread_lock(td); 558 td->td_domain.dr_policy = td->td_cpuset->cs_domain; 559 thread_unlock(td); 560 } 561 PROC_UNLOCK(p); 562 } 563 sx_sunlock(&allproc_lock); 564 kernel_object->domain.dr_policy = cpuset_kernel->cs_domain; 565 } 566 567 /* 568 * Create a new set that is a subset of a parent. 569 */ 570 static struct domainset * 571 domainset_shadow(const struct domainset *pdomain, 572 const struct domainset *domain, struct domainlist *freelist) 573 { 574 struct domainset *ndomain; 575 576 ndomain = LIST_FIRST(freelist); 577 LIST_REMOVE(ndomain, ds_link); 578 579 /* 580 * Initialize the key from the request. 581 */ 582 domainset_copy(domain, ndomain); 583 584 /* 585 * Restrict the key by the parent. 586 */ 587 DOMAINSET_AND(&ndomain->ds_mask, &pdomain->ds_mask); 588 589 return _domainset_create(ndomain, freelist); 590 } 591 592 /* 593 * Recursively check for errors that would occur from applying mask to 594 * the tree of sets starting at 'set'. Checks for sets that would become 595 * empty as well as RDONLY flags. 596 */ 597 static int 598 cpuset_testupdate(struct cpuset *set, cpuset_t *mask, int check_mask) 599 { 600 struct cpuset *nset; 601 cpuset_t newmask; 602 int error; 603 604 mtx_assert(&cpuset_lock, MA_OWNED); 605 if (set->cs_flags & CPU_SET_RDONLY) 606 return (EPERM); 607 if (check_mask) { 608 if (!CPU_OVERLAP(&set->cs_mask, mask)) 609 return (EDEADLK); 610 CPU_COPY(&set->cs_mask, &newmask); 611 CPU_AND(&newmask, mask); 612 } else 613 CPU_COPY(mask, &newmask); 614 error = 0; 615 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 616 if ((error = cpuset_testupdate(nset, &newmask, 1)) != 0) 617 break; 618 return (error); 619 } 620 621 /* 622 * Applies the mask 'mask' without checking for empty sets or permissions. 623 */ 624 static void 625 cpuset_update(struct cpuset *set, cpuset_t *mask) 626 { 627 struct cpuset *nset; 628 629 mtx_assert(&cpuset_lock, MA_OWNED); 630 CPU_AND(&set->cs_mask, mask); 631 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 632 cpuset_update(nset, &set->cs_mask); 633 634 return; 635 } 636 637 /* 638 * Modify the set 'set' to use a copy of the mask provided. Apply this new 639 * mask to restrict all children in the tree. Checks for validity before 640 * applying the changes. 641 */ 642 static int 643 cpuset_modify(struct cpuset *set, cpuset_t *mask) 644 { 645 struct cpuset *root; 646 int error; 647 648 error = priv_check(curthread, PRIV_SCHED_CPUSET); 649 if (error) 650 return (error); 651 /* 652 * In case we are called from within the jail 653 * we do not allow modifying the dedicated root 654 * cpuset of the jail but may still allow to 655 * change child sets. 656 */ 657 if (jailed(curthread->td_ucred) && 658 set->cs_flags & CPU_SET_ROOT) 659 return (EPERM); 660 /* 661 * Verify that we have access to this set of 662 * cpus. 663 */ 664 root = cpuset_getroot(set); 665 mtx_lock_spin(&cpuset_lock); 666 if (root && !CPU_SUBSET(&root->cs_mask, mask)) { 667 error = EINVAL; 668 goto out; 669 } 670 error = cpuset_testupdate(set, mask, 0); 671 if (error) 672 goto out; 673 CPU_COPY(mask, &set->cs_mask); 674 cpuset_update(set, mask); 675 out: 676 mtx_unlock_spin(&cpuset_lock); 677 678 return (error); 679 } 680 681 /* 682 * Recursively check for errors that would occur from applying mask to 683 * the tree of sets starting at 'set'. Checks for sets that would become 684 * empty as well as RDONLY flags. 685 */ 686 static int 687 cpuset_testupdate_domain(struct cpuset *set, struct domainset *dset, 688 struct domainset *orig, int *count, int check_mask) 689 { 690 struct cpuset *nset; 691 struct domainset *domain; 692 struct domainset newset; 693 int error; 694 695 mtx_assert(&cpuset_lock, MA_OWNED); 696 if (set->cs_flags & CPU_SET_RDONLY) 697 return (EPERM); 698 domain = set->cs_domain; 699 domainset_copy(domain, &newset); 700 if (!domainset_equal(domain, orig)) { 701 if (!domainset_restrict(domain, dset)) 702 return (EDEADLK); 703 DOMAINSET_AND(&newset.ds_mask, &dset->ds_mask); 704 /* Count the number of domains that are changing. */ 705 (*count)++; 706 } 707 error = 0; 708 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 709 if ((error = cpuset_testupdate_domain(nset, &newset, domain, 710 count, 1)) != 0) 711 break; 712 return (error); 713 } 714 715 /* 716 * Applies the mask 'mask' without checking for empty sets or permissions. 717 */ 718 static void 719 cpuset_update_domain(struct cpuset *set, struct domainset *domain, 720 struct domainset *orig, struct domainlist *domains) 721 { 722 struct cpuset *nset; 723 724 mtx_assert(&cpuset_lock, MA_OWNED); 725 /* 726 * If this domainset has changed from the parent we must calculate 727 * a new set. Otherwise it simply inherits from the parent. When 728 * we inherit from the parent we get a new mask and policy. If the 729 * set is modified from the parent we keep the policy and only 730 * update the mask. 731 */ 732 if (set->cs_domain != orig) { 733 orig = set->cs_domain; 734 set->cs_domain = domainset_shadow(domain, orig, domains); 735 } else 736 set->cs_domain = domain; 737 LIST_FOREACH(nset, &set->cs_children, cs_siblings) 738 cpuset_update_domain(nset, set->cs_domain, orig, domains); 739 740 return; 741 } 742 743 /* 744 * Modify the set 'set' to use a copy the domainset provided. Apply this new 745 * mask to restrict all children in the tree. Checks for validity before 746 * applying the changes. 747 */ 748 static int 749 cpuset_modify_domain(struct cpuset *set, struct domainset *domain) 750 { 751 struct domainlist domains; 752 struct domainset temp; 753 struct domainset *dset; 754 struct cpuset *root; 755 int ndomains, needed; 756 int error; 757 758 error = priv_check(curthread, PRIV_SCHED_CPUSET); 759 if (error) 760 return (error); 761 /* 762 * In case we are called from within the jail 763 * we do not allow modifying the dedicated root 764 * cpuset of the jail but may still allow to 765 * change child sets. 766 */ 767 if (jailed(curthread->td_ucred) && 768 set->cs_flags & CPU_SET_ROOT) 769 return (EPERM); 770 domainset_freelist_init(&domains, 0); 771 domain = domainset_create(domain); 772 ndomains = needed = 0; 773 do { 774 if (ndomains < needed) { 775 domainset_freelist_add(&domains, needed - ndomains); 776 ndomains = needed; 777 } 778 root = cpuset_getroot(set); 779 mtx_lock_spin(&cpuset_lock); 780 dset = root->cs_domain; 781 /* 782 * Verify that we have access to this set of domains. 783 */ 784 if (root && !domainset_valid(dset, domain)) { 785 error = EINVAL; 786 goto out; 787 } 788 /* 789 * If applying prefer we keep the current set as the fallback. 790 */ 791 if (domain->ds_policy == DOMAINSET_POLICY_PREFER) 792 DOMAINSET_COPY(&set->cs_domain->ds_mask, 793 &domain->ds_mask); 794 /* 795 * Determine whether we can apply this set of domains and 796 * how many new domain structures it will require. 797 */ 798 domainset_copy(domain, &temp); 799 needed = 0; 800 error = cpuset_testupdate_domain(set, &temp, set->cs_domain, 801 &needed, 0); 802 if (error) 803 goto out; 804 } while (ndomains < needed); 805 dset = set->cs_domain; 806 cpuset_update_domain(set, domain, dset, &domains); 807 out: 808 mtx_unlock_spin(&cpuset_lock); 809 domainset_freelist_free(&domains); 810 if (error == 0) 811 domainset_notify(); 812 813 return (error); 814 } 815 816 /* 817 * Resolve the 'which' parameter of several cpuset apis. 818 * 819 * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 820 * checks for permission via p_cansched(). 821 * 822 * For WHICH_SET returns a valid set with a new reference. 823 * 824 * -1 may be supplied for any argument to mean the current proc/thread or 825 * the base set of the current thread. May fail with ESRCH/EPERM. 826 */ 827 int 828 cpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 829 struct cpuset **setp) 830 { 831 struct cpuset *set; 832 struct thread *td; 833 struct proc *p; 834 int error; 835 836 *pp = p = NULL; 837 *tdp = td = NULL; 838 *setp = set = NULL; 839 switch (which) { 840 case CPU_WHICH_PID: 841 if (id == -1) { 842 PROC_LOCK(curproc); 843 p = curproc; 844 break; 845 } 846 if ((p = pfind(id)) == NULL) 847 return (ESRCH); 848 break; 849 case CPU_WHICH_TID: 850 if (id == -1) { 851 PROC_LOCK(curproc); 852 p = curproc; 853 td = curthread; 854 break; 855 } 856 td = tdfind(id, -1); 857 if (td == NULL) 858 return (ESRCH); 859 p = td->td_proc; 860 break; 861 case CPU_WHICH_CPUSET: 862 if (id == -1) { 863 thread_lock(curthread); 864 set = cpuset_refbase(curthread->td_cpuset); 865 thread_unlock(curthread); 866 } else 867 set = cpuset_lookup(id, curthread); 868 if (set) { 869 *setp = set; 870 return (0); 871 } 872 return (ESRCH); 873 case CPU_WHICH_JAIL: 874 { 875 /* Find `set' for prison with given id. */ 876 struct prison *pr; 877 878 sx_slock(&allprison_lock); 879 pr = prison_find_child(curthread->td_ucred->cr_prison, id); 880 sx_sunlock(&allprison_lock); 881 if (pr == NULL) 882 return (ESRCH); 883 cpuset_ref(pr->pr_cpuset); 884 *setp = pr->pr_cpuset; 885 mtx_unlock(&pr->pr_mtx); 886 return (0); 887 } 888 case CPU_WHICH_IRQ: 889 case CPU_WHICH_DOMAIN: 890 return (0); 891 default: 892 return (EINVAL); 893 } 894 error = p_cansched(curthread, p); 895 if (error) { 896 PROC_UNLOCK(p); 897 return (error); 898 } 899 if (td == NULL) 900 td = FIRST_THREAD_IN_PROC(p); 901 *pp = p; 902 *tdp = td; 903 return (0); 904 } 905 906 static int 907 cpuset_testshadow(struct cpuset *set, const cpuset_t *mask, 908 const struct domainset *domain) 909 { 910 struct cpuset *parent; 911 struct domainset *dset; 912 913 parent = cpuset_getbase(set); 914 /* 915 * If we are restricting a cpu mask it must be a subset of the 916 * parent or invalid CPUs have been specified. 917 */ 918 if (mask != NULL && !CPU_SUBSET(&parent->cs_mask, mask)) 919 return (EINVAL); 920 921 /* 922 * If we are restricting a domain mask it must be a subset of the 923 * parent or invalid domains have been specified. 924 */ 925 dset = parent->cs_domain; 926 if (domain != NULL && !domainset_valid(dset, domain)) 927 return (EINVAL); 928 929 return (0); 930 } 931 932 /* 933 * Create an anonymous set with the provided mask in the space provided by 934 * 'nset'. If the passed in set is anonymous we use its parent otherwise 935 * the new set is a child of 'set'. 936 */ 937 static int 938 cpuset_shadow(struct cpuset *set, struct cpuset **nsetp, 939 const cpuset_t *mask, const struct domainset *domain, 940 struct setlist *cpusets, struct domainlist *domains) 941 { 942 struct cpuset *parent; 943 struct cpuset *nset; 944 struct domainset *dset; 945 struct domainset *d; 946 int error; 947 948 error = cpuset_testshadow(set, mask, domain); 949 if (error) 950 return (error); 951 952 parent = cpuset_getbase(set); 953 dset = parent->cs_domain; 954 if (mask == NULL) 955 mask = &set->cs_mask; 956 if (domain != NULL) 957 d = domainset_shadow(dset, domain, domains); 958 else 959 d = set->cs_domain; 960 nset = LIST_FIRST(cpusets); 961 error = _cpuset_create(nset, parent, mask, d, CPUSET_INVALID); 962 if (error == 0) { 963 LIST_REMOVE(nset, cs_link); 964 *nsetp = nset; 965 } 966 return (error); 967 } 968 969 static struct cpuset * 970 cpuset_update_thread(struct thread *td, struct cpuset *nset) 971 { 972 struct cpuset *tdset; 973 974 tdset = td->td_cpuset; 975 td->td_cpuset = nset; 976 td->td_domain.dr_policy = nset->cs_domain; 977 sched_affinity(td); 978 979 return (tdset); 980 } 981 982 static int 983 cpuset_setproc_test_maskthread(struct cpuset *tdset, cpuset_t *mask, 984 struct domainset *domain) 985 { 986 struct cpuset *parent; 987 988 parent = cpuset_getbase(tdset); 989 if (mask == NULL) 990 mask = &tdset->cs_mask; 991 if (domain == NULL) 992 domain = tdset->cs_domain; 993 return cpuset_testshadow(parent, mask, domain); 994 } 995 996 static int 997 cpuset_setproc_maskthread(struct cpuset *tdset, cpuset_t *mask, 998 struct domainset *domain, struct cpuset **nsetp, 999 struct setlist *freelist, struct domainlist *domainlist) 1000 { 1001 struct cpuset *parent; 1002 1003 parent = cpuset_getbase(tdset); 1004 if (mask == NULL) 1005 mask = &tdset->cs_mask; 1006 if (domain == NULL) 1007 domain = tdset->cs_domain; 1008 return cpuset_shadow(parent, nsetp, mask, domain, freelist, 1009 domainlist); 1010 } 1011 1012 static int 1013 cpuset_setproc_setthread_mask(struct cpuset *tdset, struct cpuset *set, 1014 cpuset_t *mask, struct domainset *domain) 1015 { 1016 struct cpuset *parent; 1017 1018 parent = cpuset_getbase(tdset); 1019 1020 /* 1021 * If the thread restricted its mask then apply that same 1022 * restriction to the new set, otherwise take it wholesale. 1023 */ 1024 if (CPU_CMP(&tdset->cs_mask, &parent->cs_mask) != 0) { 1025 CPU_COPY(&tdset->cs_mask, mask); 1026 CPU_AND(mask, &set->cs_mask); 1027 } else 1028 CPU_COPY(&set->cs_mask, mask); 1029 1030 /* 1031 * If the thread restricted the domain then we apply the 1032 * restriction to the new set but retain the policy. 1033 */ 1034 if (tdset->cs_domain != parent->cs_domain) { 1035 domainset_copy(tdset->cs_domain, domain); 1036 DOMAINSET_AND(&domain->ds_mask, &set->cs_domain->ds_mask); 1037 } else 1038 domainset_copy(set->cs_domain, domain); 1039 1040 if (CPU_EMPTY(mask) || DOMAINSET_EMPTY(&domain->ds_mask)) 1041 return (EDEADLK); 1042 1043 return (0); 1044 } 1045 1046 static int 1047 cpuset_setproc_test_setthread(struct cpuset *tdset, struct cpuset *set) 1048 { 1049 struct domainset domain; 1050 cpuset_t mask; 1051 1052 if (tdset->cs_id != CPUSET_INVALID) 1053 return (0); 1054 return cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 1055 } 1056 1057 static int 1058 cpuset_setproc_setthread(struct cpuset *tdset, struct cpuset *set, 1059 struct cpuset **nsetp, struct setlist *freelist, 1060 struct domainlist *domainlist) 1061 { 1062 struct domainset domain; 1063 cpuset_t mask; 1064 int error; 1065 1066 /* 1067 * If we're replacing on a thread that has not constrained the 1068 * original set we can simply accept the new set. 1069 */ 1070 if (tdset->cs_id != CPUSET_INVALID) { 1071 *nsetp = cpuset_ref(set); 1072 return (0); 1073 } 1074 error = cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 1075 if (error) 1076 return (error); 1077 1078 return cpuset_shadow(tdset, nsetp, &mask, &domain, freelist, 1079 domainlist); 1080 } 1081 1082 /* 1083 * Handle three cases for updating an entire process. 1084 * 1085 * 1) Set is non-null. This reparents all anonymous sets to the provided 1086 * set and replaces all non-anonymous td_cpusets with the provided set. 1087 * 2) Mask is non-null. This replaces or creates anonymous sets for every 1088 * thread with the existing base as a parent. 1089 * 3) domain is non-null. This creates anonymous sets for every thread 1090 * and replaces the domain set. 1091 * 1092 * This is overly complicated because we can't allocate while holding a 1093 * spinlock and spinlocks must be held while changing and examining thread 1094 * state. 1095 */ 1096 static int 1097 cpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask, 1098 struct domainset *domain) 1099 { 1100 struct setlist freelist; 1101 struct setlist droplist; 1102 struct domainlist domainlist; 1103 struct cpuset *nset; 1104 struct thread *td; 1105 struct proc *p; 1106 int threads; 1107 int nfree; 1108 int error; 1109 1110 /* 1111 * The algorithm requires two passes due to locking considerations. 1112 * 1113 * 1) Lookup the process and acquire the locks in the required order. 1114 * 2) If enough cpusets have not been allocated release the locks and 1115 * allocate them. Loop. 1116 */ 1117 cpuset_freelist_init(&freelist, 1); 1118 domainset_freelist_init(&domainlist, 1); 1119 nfree = 1; 1120 LIST_INIT(&droplist); 1121 nfree = 0; 1122 for (;;) { 1123 error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 1124 if (error) 1125 goto out; 1126 if (nfree >= p->p_numthreads) 1127 break; 1128 threads = p->p_numthreads; 1129 PROC_UNLOCK(p); 1130 if (nfree < threads) { 1131 cpuset_freelist_add(&freelist, threads - nfree); 1132 domainset_freelist_add(&domainlist, threads - nfree); 1133 nfree = threads; 1134 } 1135 } 1136 PROC_LOCK_ASSERT(p, MA_OWNED); 1137 /* 1138 * Now that the appropriate locks are held and we have enough cpusets, 1139 * make sure the operation will succeed before applying changes. The 1140 * proc lock prevents td_cpuset from changing between calls. 1141 */ 1142 error = 0; 1143 FOREACH_THREAD_IN_PROC(p, td) { 1144 thread_lock(td); 1145 if (set != NULL) 1146 error = cpuset_setproc_test_setthread(td->td_cpuset, 1147 set); 1148 else 1149 error = cpuset_setproc_test_maskthread(td->td_cpuset, 1150 mask, domain); 1151 thread_unlock(td); 1152 if (error) 1153 goto unlock_out; 1154 } 1155 /* 1156 * Replace each thread's cpuset while using deferred release. We 1157 * must do this because the thread lock must be held while operating 1158 * on the thread and this limits the type of operations allowed. 1159 */ 1160 FOREACH_THREAD_IN_PROC(p, td) { 1161 thread_lock(td); 1162 if (set != NULL) 1163 error = cpuset_setproc_setthread(td->td_cpuset, set, 1164 &nset, &freelist, &domainlist); 1165 else 1166 error = cpuset_setproc_maskthread(td->td_cpuset, mask, 1167 domain, &nset, &freelist, &domainlist); 1168 if (error) { 1169 thread_unlock(td); 1170 break; 1171 } 1172 cpuset_rel_defer(&droplist, cpuset_update_thread(td, nset)); 1173 thread_unlock(td); 1174 } 1175 unlock_out: 1176 PROC_UNLOCK(p); 1177 out: 1178 while ((nset = LIST_FIRST(&droplist)) != NULL) 1179 cpuset_rel_complete(nset); 1180 cpuset_freelist_free(&freelist); 1181 domainset_freelist_free(&domainlist); 1182 return (error); 1183 } 1184 1185 static int 1186 bitset_strprint(char *buf, size_t bufsiz, const struct bitset *set, int setlen) 1187 { 1188 size_t bytes; 1189 int i, once; 1190 char *p; 1191 1192 once = 0; 1193 p = buf; 1194 for (i = 0; i < __bitset_words(setlen); i++) { 1195 if (once != 0) { 1196 if (bufsiz < 1) 1197 return (0); 1198 *p = ','; 1199 p++; 1200 bufsiz--; 1201 } else 1202 once = 1; 1203 if (bufsiz < sizeof(__STRING(ULONG_MAX))) 1204 return (0); 1205 bytes = snprintf(p, bufsiz, "%lx", set->__bits[i]); 1206 p += bytes; 1207 bufsiz -= bytes; 1208 } 1209 return (p - buf); 1210 } 1211 1212 static int 1213 bitset_strscan(struct bitset *set, int setlen, const char *buf) 1214 { 1215 int i, ret; 1216 const char *p; 1217 1218 BIT_ZERO(setlen, set); 1219 p = buf; 1220 for (i = 0; i < __bitset_words(setlen); i++) { 1221 if (*p == ',') { 1222 p++; 1223 continue; 1224 } 1225 ret = sscanf(p, "%lx", &set->__bits[i]); 1226 if (ret == 0 || ret == -1) 1227 break; 1228 while (isxdigit(*p)) 1229 p++; 1230 } 1231 return (p - buf); 1232 } 1233 1234 /* 1235 * Return a string representing a valid layout for a cpuset_t object. 1236 * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1237 */ 1238 char * 1239 cpusetobj_strprint(char *buf, const cpuset_t *set) 1240 { 1241 1242 bitset_strprint(buf, CPUSETBUFSIZ, (const struct bitset *)set, 1243 CPU_SETSIZE); 1244 return (buf); 1245 } 1246 1247 /* 1248 * Build a valid cpuset_t object from a string representation. 1249 * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1250 */ 1251 int 1252 cpusetobj_strscan(cpuset_t *set, const char *buf) 1253 { 1254 char p; 1255 1256 if (strlen(buf) > CPUSETBUFSIZ - 1) 1257 return (-1); 1258 1259 p = buf[bitset_strscan((struct bitset *)set, CPU_SETSIZE, buf)]; 1260 if (p != '\0') 1261 return (-1); 1262 1263 return (0); 1264 } 1265 1266 /* 1267 * Handle a domainset specifier in the sysctl tree. A poiner to a pointer to 1268 * a domainset is in arg1. If the user specifies a valid domainset the 1269 * pointer is updated. 1270 * 1271 * Format is: 1272 * hex mask word 0,hex mask word 1,...:decimal policy:decimal preferred 1273 */ 1274 int 1275 sysctl_handle_domainset(SYSCTL_HANDLER_ARGS) 1276 { 1277 char buf[DOMAINSETBUFSIZ]; 1278 struct domainset *dset; 1279 struct domainset key; 1280 int policy, prefer, error; 1281 char *p; 1282 1283 dset = *(struct domainset **)arg1; 1284 error = 0; 1285 1286 if (dset != NULL) { 1287 p = buf + bitset_strprint(buf, DOMAINSETBUFSIZ, 1288 (const struct bitset *)&dset->ds_mask, DOMAINSET_SETSIZE); 1289 sprintf(p, ":%d:%d", dset->ds_policy, dset->ds_prefer); 1290 } else 1291 sprintf(buf, "<NULL>"); 1292 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 1293 if (error != 0 || req->newptr == NULL) 1294 return (error); 1295 1296 /* 1297 * Read in and validate the string. 1298 */ 1299 memset(&key, 0, sizeof(key)); 1300 p = &buf[bitset_strscan((struct bitset *)&key.ds_mask, 1301 DOMAINSET_SETSIZE, buf)]; 1302 if (p == buf) 1303 return (EINVAL); 1304 if (sscanf(p, ":%d:%d", &policy, &prefer) != 2) 1305 return (EINVAL); 1306 key.ds_policy = policy; 1307 key.ds_prefer = prefer; 1308 1309 /* Domainset_create() validates the policy.*/ 1310 dset = domainset_create(&key); 1311 if (dset == NULL) 1312 return (EINVAL); 1313 *(struct domainset **)arg1 = dset; 1314 1315 return (error); 1316 } 1317 1318 /* 1319 * Apply an anonymous mask or a domain to a single thread. 1320 */ 1321 static int 1322 _cpuset_setthread(lwpid_t id, cpuset_t *mask, struct domainset *domain) 1323 { 1324 struct setlist cpusets; 1325 struct domainlist domainlist; 1326 struct cpuset *nset; 1327 struct cpuset *set; 1328 struct thread *td; 1329 struct proc *p; 1330 int error; 1331 1332 cpuset_freelist_init(&cpusets, 1); 1333 domainset_freelist_init(&domainlist, domain != NULL); 1334 error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &set); 1335 if (error) 1336 goto out; 1337 set = NULL; 1338 thread_lock(td); 1339 error = cpuset_shadow(td->td_cpuset, &nset, mask, domain, 1340 &cpusets, &domainlist); 1341 if (error == 0) 1342 set = cpuset_update_thread(td, nset); 1343 thread_unlock(td); 1344 PROC_UNLOCK(p); 1345 if (set) 1346 cpuset_rel(set); 1347 out: 1348 cpuset_freelist_free(&cpusets); 1349 domainset_freelist_free(&domainlist); 1350 return (error); 1351 } 1352 1353 /* 1354 * Apply an anonymous mask to a single thread. 1355 */ 1356 int 1357 cpuset_setthread(lwpid_t id, cpuset_t *mask) 1358 { 1359 1360 return _cpuset_setthread(id, mask, NULL); 1361 } 1362 1363 /* 1364 * Apply new cpumask to the ithread. 1365 */ 1366 int 1367 cpuset_setithread(lwpid_t id, int cpu) 1368 { 1369 cpuset_t mask; 1370 1371 CPU_ZERO(&mask); 1372 if (cpu == NOCPU) 1373 CPU_COPY(cpuset_root, &mask); 1374 else 1375 CPU_SET(cpu, &mask); 1376 return _cpuset_setthread(id, &mask, NULL); 1377 } 1378 1379 /* 1380 * Initialize static domainsets after NUMA information is available. This is 1381 * called very early during boot. 1382 */ 1383 void 1384 domainset_init(void) 1385 { 1386 struct domainset *dset; 1387 int i; 1388 1389 dset = &domainset_roundrobin; 1390 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1391 dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 1392 dset->ds_prefer = -1; 1393 _domainset_create(dset, NULL); 1394 1395 for (i = 0; i < vm_ndomains; i++) { 1396 dset = &domainset_prefer[i]; 1397 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1398 dset->ds_policy = DOMAINSET_POLICY_PREFER; 1399 dset->ds_prefer = i; 1400 _domainset_create(dset, NULL); 1401 } 1402 } 1403 1404 /* 1405 * Create the domainset for cpuset 0, 1 and cpuset 2. 1406 */ 1407 void 1408 domainset_zero(void) 1409 { 1410 struct domainset *dset; 1411 1412 mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 1413 1414 dset = &domainset0; 1415 DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1416 dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; 1417 dset->ds_prefer = -1; 1418 curthread->td_domain.dr_policy = _domainset_create(dset, NULL); 1419 1420 domainset_copy(dset, &domainset2); 1421 domainset2.ds_policy = DOMAINSET_POLICY_INTERLEAVE; 1422 kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL); 1423 1424 /* Remove empty domains from the global policies. */ 1425 LIST_FOREACH(dset, &cpuset_domains, ds_link) 1426 (void)domainset_empty_vm(dset); 1427 } 1428 1429 /* 1430 * Creates system-wide cpusets and the cpuset for thread0 including three 1431 * sets: 1432 * 1433 * 0 - The root set which should represent all valid processors in the 1434 * system. It is initially created with a mask of all processors 1435 * because we don't know what processors are valid until cpuset_init() 1436 * runs. This set is immutable. 1437 * 1 - The default set which all processes are a member of until changed. 1438 * This allows an administrator to move all threads off of given cpus to 1439 * dedicate them to high priority tasks or save power etc. 1440 * 2 - The kernel set which allows restriction and policy to be applied only 1441 * to kernel threads and the kernel_object. 1442 */ 1443 struct cpuset * 1444 cpuset_thread0(void) 1445 { 1446 struct cpuset *set; 1447 int i; 1448 int error __unused; 1449 1450 cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 1451 NULL, NULL, UMA_ALIGN_CACHE, 0); 1452 domainset_zone = uma_zcreate("domainset", sizeof(struct domainset), 1453 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 1454 1455 /* 1456 * Create the root system set (0) for the whole machine. Doesn't use 1457 * cpuset_create() due to NULL parent. 1458 */ 1459 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1460 CPU_COPY(&all_cpus, &set->cs_mask); 1461 LIST_INIT(&set->cs_children); 1462 LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 1463 set->cs_ref = 1; 1464 set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; 1465 set->cs_domain = &domainset0; 1466 cpuset_zero = set; 1467 cpuset_root = &set->cs_mask; 1468 1469 /* 1470 * Now derive a default (1), modifiable set from that to give out. 1471 */ 1472 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1473 error = _cpuset_create(set, cpuset_zero, NULL, NULL, 1); 1474 KASSERT(error == 0, ("Error creating default set: %d\n", error)); 1475 cpuset_default = set; 1476 /* 1477 * Create the kernel set (2). 1478 */ 1479 set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 1480 error = _cpuset_create(set, cpuset_zero, NULL, NULL, 2); 1481 KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); 1482 set->cs_domain = &domainset2; 1483 cpuset_kernel = set; 1484 1485 /* 1486 * Initialize the unit allocator. 0 and 1 are allocated above. 1487 */ 1488 cpuset_unr = new_unrhdr(2, INT_MAX, NULL); 1489 1490 /* 1491 * If MD code has not initialized per-domain cpusets, place all 1492 * CPUs in domain 0. 1493 */ 1494 for (i = 0; i < MAXMEMDOM; i++) 1495 if (!CPU_EMPTY(&cpuset_domain[i])) 1496 goto domains_set; 1497 CPU_COPY(&all_cpus, &cpuset_domain[0]); 1498 domains_set: 1499 1500 return (cpuset_default); 1501 } 1502 1503 void 1504 cpuset_kernthread(struct thread *td) 1505 { 1506 struct cpuset *set; 1507 1508 thread_lock(td); 1509 set = td->td_cpuset; 1510 td->td_cpuset = cpuset_ref(cpuset_kernel); 1511 thread_unlock(td); 1512 cpuset_rel(set); 1513 } 1514 1515 /* 1516 * Create a cpuset, which would be cpuset_create() but 1517 * mark the new 'set' as root. 1518 * 1519 * We are not going to reparent the td to it. Use cpuset_setproc_update_set() 1520 * for that. 1521 * 1522 * In case of no error, returns the set in *setp locked with a reference. 1523 */ 1524 int 1525 cpuset_create_root(struct prison *pr, struct cpuset **setp) 1526 { 1527 struct cpuset *set; 1528 int error; 1529 1530 KASSERT(pr != NULL, ("[%s:%d] invalid pr", __func__, __LINE__)); 1531 KASSERT(setp != NULL, ("[%s:%d] invalid setp", __func__, __LINE__)); 1532 1533 error = cpuset_create(setp, pr->pr_cpuset, &pr->pr_cpuset->cs_mask); 1534 if (error) 1535 return (error); 1536 1537 KASSERT(*setp != NULL, ("[%s:%d] cpuset_create returned invalid data", 1538 __func__, __LINE__)); 1539 1540 /* Mark the set as root. */ 1541 set = *setp; 1542 set->cs_flags |= CPU_SET_ROOT; 1543 1544 return (0); 1545 } 1546 1547 int 1548 cpuset_setproc_update_set(struct proc *p, struct cpuset *set) 1549 { 1550 int error; 1551 1552 KASSERT(p != NULL, ("[%s:%d] invalid proc", __func__, __LINE__)); 1553 KASSERT(set != NULL, ("[%s:%d] invalid set", __func__, __LINE__)); 1554 1555 cpuset_ref(set); 1556 error = cpuset_setproc(p->p_pid, set, NULL, NULL); 1557 if (error) 1558 return (error); 1559 cpuset_rel(set); 1560 return (0); 1561 } 1562 1563 #ifndef _SYS_SYSPROTO_H_ 1564 struct cpuset_args { 1565 cpusetid_t *setid; 1566 }; 1567 #endif 1568 int 1569 sys_cpuset(struct thread *td, struct cpuset_args *uap) 1570 { 1571 struct cpuset *root; 1572 struct cpuset *set; 1573 int error; 1574 1575 thread_lock(td); 1576 root = cpuset_refroot(td->td_cpuset); 1577 thread_unlock(td); 1578 error = cpuset_create(&set, root, &root->cs_mask); 1579 cpuset_rel(root); 1580 if (error) 1581 return (error); 1582 error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 1583 if (error == 0) 1584 error = cpuset_setproc(-1, set, NULL, NULL); 1585 cpuset_rel(set); 1586 return (error); 1587 } 1588 1589 #ifndef _SYS_SYSPROTO_H_ 1590 struct cpuset_setid_args { 1591 cpuwhich_t which; 1592 id_t id; 1593 cpusetid_t setid; 1594 }; 1595 #endif 1596 int 1597 sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 1598 { 1599 1600 return (kern_cpuset_setid(td, uap->which, uap->id, uap->setid)); 1601 } 1602 1603 int 1604 kern_cpuset_setid(struct thread *td, cpuwhich_t which, 1605 id_t id, cpusetid_t setid) 1606 { 1607 struct cpuset *set; 1608 int error; 1609 1610 /* 1611 * Presently we only support per-process sets. 1612 */ 1613 if (which != CPU_WHICH_PID) 1614 return (EINVAL); 1615 set = cpuset_lookup(setid, td); 1616 if (set == NULL) 1617 return (ESRCH); 1618 error = cpuset_setproc(id, set, NULL, NULL); 1619 cpuset_rel(set); 1620 return (error); 1621 } 1622 1623 #ifndef _SYS_SYSPROTO_H_ 1624 struct cpuset_getid_args { 1625 cpulevel_t level; 1626 cpuwhich_t which; 1627 id_t id; 1628 cpusetid_t *setid; 1629 }; 1630 #endif 1631 int 1632 sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 1633 { 1634 1635 return (kern_cpuset_getid(td, uap->level, uap->which, uap->id, 1636 uap->setid)); 1637 } 1638 1639 int 1640 kern_cpuset_getid(struct thread *td, cpulevel_t level, cpuwhich_t which, 1641 id_t id, cpusetid_t *setid) 1642 { 1643 struct cpuset *nset; 1644 struct cpuset *set; 1645 struct thread *ttd; 1646 struct proc *p; 1647 cpusetid_t tmpid; 1648 int error; 1649 1650 if (level == CPU_LEVEL_WHICH && which != CPU_WHICH_CPUSET) 1651 return (EINVAL); 1652 error = cpuset_which(which, id, &p, &ttd, &set); 1653 if (error) 1654 return (error); 1655 switch (which) { 1656 case CPU_WHICH_TID: 1657 case CPU_WHICH_PID: 1658 thread_lock(ttd); 1659 set = cpuset_refbase(ttd->td_cpuset); 1660 thread_unlock(ttd); 1661 PROC_UNLOCK(p); 1662 break; 1663 case CPU_WHICH_CPUSET: 1664 case CPU_WHICH_JAIL: 1665 break; 1666 case CPU_WHICH_IRQ: 1667 case CPU_WHICH_DOMAIN: 1668 return (EINVAL); 1669 } 1670 switch (level) { 1671 case CPU_LEVEL_ROOT: 1672 nset = cpuset_refroot(set); 1673 cpuset_rel(set); 1674 set = nset; 1675 break; 1676 case CPU_LEVEL_CPUSET: 1677 break; 1678 case CPU_LEVEL_WHICH: 1679 break; 1680 } 1681 tmpid = set->cs_id; 1682 cpuset_rel(set); 1683 if (error == 0) 1684 error = copyout(&tmpid, setid, sizeof(tmpid)); 1685 1686 return (error); 1687 } 1688 1689 #ifndef _SYS_SYSPROTO_H_ 1690 struct cpuset_getaffinity_args { 1691 cpulevel_t level; 1692 cpuwhich_t which; 1693 id_t id; 1694 size_t cpusetsize; 1695 cpuset_t *mask; 1696 }; 1697 #endif 1698 int 1699 sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 1700 { 1701 1702 return (kern_cpuset_getaffinity(td, uap->level, uap->which, 1703 uap->id, uap->cpusetsize, uap->mask)); 1704 } 1705 1706 int 1707 kern_cpuset_getaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 1708 id_t id, size_t cpusetsize, cpuset_t *maskp) 1709 { 1710 struct thread *ttd; 1711 struct cpuset *nset; 1712 struct cpuset *set; 1713 struct proc *p; 1714 cpuset_t *mask; 1715 int error; 1716 size_t size; 1717 1718 if (cpusetsize < sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY) 1719 return (ERANGE); 1720 /* In Capability mode, you can only get your own CPU set. */ 1721 if (IN_CAPABILITY_MODE(td)) { 1722 if (level != CPU_LEVEL_WHICH) 1723 return (ECAPMODE); 1724 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 1725 return (ECAPMODE); 1726 if (id != -1) 1727 return (ECAPMODE); 1728 } 1729 size = cpusetsize; 1730 mask = malloc(size, M_TEMP, M_WAITOK | M_ZERO); 1731 error = cpuset_which(which, id, &p, &ttd, &set); 1732 if (error) 1733 goto out; 1734 switch (level) { 1735 case CPU_LEVEL_ROOT: 1736 case CPU_LEVEL_CPUSET: 1737 switch (which) { 1738 case CPU_WHICH_TID: 1739 case CPU_WHICH_PID: 1740 thread_lock(ttd); 1741 set = cpuset_ref(ttd->td_cpuset); 1742 thread_unlock(ttd); 1743 break; 1744 case CPU_WHICH_CPUSET: 1745 case CPU_WHICH_JAIL: 1746 break; 1747 case CPU_WHICH_IRQ: 1748 case CPU_WHICH_INTRHANDLER: 1749 case CPU_WHICH_ITHREAD: 1750 case CPU_WHICH_DOMAIN: 1751 error = EINVAL; 1752 goto out; 1753 } 1754 if (level == CPU_LEVEL_ROOT) 1755 nset = cpuset_refroot(set); 1756 else 1757 nset = cpuset_refbase(set); 1758 CPU_COPY(&nset->cs_mask, mask); 1759 cpuset_rel(nset); 1760 break; 1761 case CPU_LEVEL_WHICH: 1762 switch (which) { 1763 case CPU_WHICH_TID: 1764 thread_lock(ttd); 1765 CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 1766 thread_unlock(ttd); 1767 break; 1768 case CPU_WHICH_PID: 1769 FOREACH_THREAD_IN_PROC(p, ttd) { 1770 thread_lock(ttd); 1771 CPU_OR(mask, &ttd->td_cpuset->cs_mask); 1772 thread_unlock(ttd); 1773 } 1774 break; 1775 case CPU_WHICH_CPUSET: 1776 case CPU_WHICH_JAIL: 1777 CPU_COPY(&set->cs_mask, mask); 1778 break; 1779 case CPU_WHICH_IRQ: 1780 case CPU_WHICH_INTRHANDLER: 1781 case CPU_WHICH_ITHREAD: 1782 error = intr_getaffinity(id, which, mask); 1783 break; 1784 case CPU_WHICH_DOMAIN: 1785 if (id < 0 || id >= MAXMEMDOM) 1786 error = ESRCH; 1787 else 1788 CPU_COPY(&cpuset_domain[id], mask); 1789 break; 1790 } 1791 break; 1792 default: 1793 error = EINVAL; 1794 break; 1795 } 1796 if (set) 1797 cpuset_rel(set); 1798 if (p) 1799 PROC_UNLOCK(p); 1800 if (error == 0) 1801 error = copyout(mask, maskp, size); 1802 out: 1803 free(mask, M_TEMP); 1804 return (error); 1805 } 1806 1807 #ifndef _SYS_SYSPROTO_H_ 1808 struct cpuset_setaffinity_args { 1809 cpulevel_t level; 1810 cpuwhich_t which; 1811 id_t id; 1812 size_t cpusetsize; 1813 const cpuset_t *mask; 1814 }; 1815 #endif 1816 int 1817 sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 1818 { 1819 1820 return (kern_cpuset_setaffinity(td, uap->level, uap->which, 1821 uap->id, uap->cpusetsize, uap->mask)); 1822 } 1823 1824 int 1825 kern_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 1826 id_t id, size_t cpusetsize, const cpuset_t *maskp) 1827 { 1828 struct cpuset *nset; 1829 struct cpuset *set; 1830 struct thread *ttd; 1831 struct proc *p; 1832 cpuset_t *mask; 1833 int error; 1834 1835 if (cpusetsize < sizeof(cpuset_t) || cpusetsize > CPU_MAXSIZE / NBBY) 1836 return (ERANGE); 1837 /* In Capability mode, you can only set your own CPU set. */ 1838 if (IN_CAPABILITY_MODE(td)) { 1839 if (level != CPU_LEVEL_WHICH) 1840 return (ECAPMODE); 1841 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 1842 return (ECAPMODE); 1843 if (id != -1) 1844 return (ECAPMODE); 1845 } 1846 mask = malloc(cpusetsize, M_TEMP, M_WAITOK | M_ZERO); 1847 error = copyin(maskp, mask, cpusetsize); 1848 if (error) 1849 goto out; 1850 /* 1851 * Verify that no high bits are set. 1852 */ 1853 if (cpusetsize > sizeof(cpuset_t)) { 1854 char *end; 1855 char *cp; 1856 1857 end = cp = (char *)&mask->__bits; 1858 end += cpusetsize; 1859 cp += sizeof(cpuset_t); 1860 while (cp != end) 1861 if (*cp++ != 0) { 1862 error = EINVAL; 1863 goto out; 1864 } 1865 1866 } 1867 switch (level) { 1868 case CPU_LEVEL_ROOT: 1869 case CPU_LEVEL_CPUSET: 1870 error = cpuset_which(which, id, &p, &ttd, &set); 1871 if (error) 1872 break; 1873 switch (which) { 1874 case CPU_WHICH_TID: 1875 case CPU_WHICH_PID: 1876 thread_lock(ttd); 1877 set = cpuset_ref(ttd->td_cpuset); 1878 thread_unlock(ttd); 1879 PROC_UNLOCK(p); 1880 break; 1881 case CPU_WHICH_CPUSET: 1882 case CPU_WHICH_JAIL: 1883 break; 1884 case CPU_WHICH_IRQ: 1885 case CPU_WHICH_INTRHANDLER: 1886 case CPU_WHICH_ITHREAD: 1887 case CPU_WHICH_DOMAIN: 1888 error = EINVAL; 1889 goto out; 1890 } 1891 if (level == CPU_LEVEL_ROOT) 1892 nset = cpuset_refroot(set); 1893 else 1894 nset = cpuset_refbase(set); 1895 error = cpuset_modify(nset, mask); 1896 cpuset_rel(nset); 1897 cpuset_rel(set); 1898 break; 1899 case CPU_LEVEL_WHICH: 1900 switch (which) { 1901 case CPU_WHICH_TID: 1902 error = cpuset_setthread(id, mask); 1903 break; 1904 case CPU_WHICH_PID: 1905 error = cpuset_setproc(id, NULL, mask, NULL); 1906 break; 1907 case CPU_WHICH_CPUSET: 1908 case CPU_WHICH_JAIL: 1909 error = cpuset_which(which, id, &p, &ttd, &set); 1910 if (error == 0) { 1911 error = cpuset_modify(set, mask); 1912 cpuset_rel(set); 1913 } 1914 break; 1915 case CPU_WHICH_IRQ: 1916 case CPU_WHICH_INTRHANDLER: 1917 case CPU_WHICH_ITHREAD: 1918 error = intr_setaffinity(id, which, mask); 1919 break; 1920 default: 1921 error = EINVAL; 1922 break; 1923 } 1924 break; 1925 default: 1926 error = EINVAL; 1927 break; 1928 } 1929 out: 1930 free(mask, M_TEMP); 1931 return (error); 1932 } 1933 1934 #ifndef _SYS_SYSPROTO_H_ 1935 struct cpuset_getdomain_args { 1936 cpulevel_t level; 1937 cpuwhich_t which; 1938 id_t id; 1939 size_t domainsetsize; 1940 domainset_t *mask; 1941 int *policy; 1942 }; 1943 #endif 1944 int 1945 sys_cpuset_getdomain(struct thread *td, struct cpuset_getdomain_args *uap) 1946 { 1947 1948 return (kern_cpuset_getdomain(td, uap->level, uap->which, 1949 uap->id, uap->domainsetsize, uap->mask, uap->policy)); 1950 } 1951 1952 int 1953 kern_cpuset_getdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 1954 id_t id, size_t domainsetsize, domainset_t *maskp, int *policyp) 1955 { 1956 struct domainset outset; 1957 struct thread *ttd; 1958 struct cpuset *nset; 1959 struct cpuset *set; 1960 struct domainset *dset; 1961 struct proc *p; 1962 domainset_t *mask; 1963 int error; 1964 1965 if (domainsetsize < sizeof(domainset_t) || 1966 domainsetsize > DOMAINSET_MAXSIZE / NBBY) 1967 return (ERANGE); 1968 /* In Capability mode, you can only get your own domain set. */ 1969 if (IN_CAPABILITY_MODE(td)) { 1970 if (level != CPU_LEVEL_WHICH) 1971 return (ECAPMODE); 1972 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 1973 return (ECAPMODE); 1974 if (id != -1) 1975 return (ECAPMODE); 1976 } 1977 mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 1978 bzero(&outset, sizeof(outset)); 1979 error = cpuset_which(which, id, &p, &ttd, &set); 1980 if (error) 1981 goto out; 1982 switch (level) { 1983 case CPU_LEVEL_ROOT: 1984 case CPU_LEVEL_CPUSET: 1985 switch (which) { 1986 case CPU_WHICH_TID: 1987 case CPU_WHICH_PID: 1988 thread_lock(ttd); 1989 set = cpuset_ref(ttd->td_cpuset); 1990 thread_unlock(ttd); 1991 break; 1992 case CPU_WHICH_CPUSET: 1993 case CPU_WHICH_JAIL: 1994 break; 1995 case CPU_WHICH_IRQ: 1996 case CPU_WHICH_INTRHANDLER: 1997 case CPU_WHICH_ITHREAD: 1998 case CPU_WHICH_DOMAIN: 1999 error = EINVAL; 2000 goto out; 2001 } 2002 if (level == CPU_LEVEL_ROOT) 2003 nset = cpuset_refroot(set); 2004 else 2005 nset = cpuset_refbase(set); 2006 domainset_copy(nset->cs_domain, &outset); 2007 cpuset_rel(nset); 2008 break; 2009 case CPU_LEVEL_WHICH: 2010 switch (which) { 2011 case CPU_WHICH_TID: 2012 thread_lock(ttd); 2013 domainset_copy(ttd->td_cpuset->cs_domain, &outset); 2014 thread_unlock(ttd); 2015 break; 2016 case CPU_WHICH_PID: 2017 FOREACH_THREAD_IN_PROC(p, ttd) { 2018 thread_lock(ttd); 2019 dset = ttd->td_cpuset->cs_domain; 2020 /* Show all domains in the proc. */ 2021 DOMAINSET_OR(&outset.ds_mask, &dset->ds_mask); 2022 /* Last policy wins. */ 2023 outset.ds_policy = dset->ds_policy; 2024 outset.ds_prefer = dset->ds_prefer; 2025 thread_unlock(ttd); 2026 } 2027 break; 2028 case CPU_WHICH_CPUSET: 2029 case CPU_WHICH_JAIL: 2030 domainset_copy(set->cs_domain, &outset); 2031 break; 2032 case CPU_WHICH_IRQ: 2033 case CPU_WHICH_INTRHANDLER: 2034 case CPU_WHICH_ITHREAD: 2035 case CPU_WHICH_DOMAIN: 2036 error = EINVAL; 2037 break; 2038 } 2039 break; 2040 default: 2041 error = EINVAL; 2042 break; 2043 } 2044 if (set) 2045 cpuset_rel(set); 2046 if (p) 2047 PROC_UNLOCK(p); 2048 /* 2049 * Translate prefer into a set containing only the preferred domain, 2050 * not the entire fallback set. 2051 */ 2052 if (outset.ds_policy == DOMAINSET_POLICY_PREFER) { 2053 DOMAINSET_ZERO(&outset.ds_mask); 2054 DOMAINSET_SET(outset.ds_prefer, &outset.ds_mask); 2055 } 2056 DOMAINSET_COPY(&outset.ds_mask, mask); 2057 if (error == 0) 2058 error = copyout(mask, maskp, domainsetsize); 2059 if (error == 0) 2060 if (suword32(policyp, outset.ds_policy) != 0) 2061 error = EFAULT; 2062 out: 2063 free(mask, M_TEMP); 2064 return (error); 2065 } 2066 2067 #ifndef _SYS_SYSPROTO_H_ 2068 struct cpuset_setdomain_args { 2069 cpulevel_t level; 2070 cpuwhich_t which; 2071 id_t id; 2072 size_t domainsetsize; 2073 domainset_t *mask; 2074 int policy; 2075 }; 2076 #endif 2077 int 2078 sys_cpuset_setdomain(struct thread *td, struct cpuset_setdomain_args *uap) 2079 { 2080 2081 return (kern_cpuset_setdomain(td, uap->level, uap->which, 2082 uap->id, uap->domainsetsize, uap->mask, uap->policy)); 2083 } 2084 2085 int 2086 kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 2087 id_t id, size_t domainsetsize, const domainset_t *maskp, int policy) 2088 { 2089 struct cpuset *nset; 2090 struct cpuset *set; 2091 struct thread *ttd; 2092 struct proc *p; 2093 struct domainset domain; 2094 domainset_t *mask; 2095 int error; 2096 2097 if (domainsetsize < sizeof(domainset_t) || 2098 domainsetsize > DOMAINSET_MAXSIZE / NBBY) 2099 return (ERANGE); 2100 if (policy <= DOMAINSET_POLICY_INVALID || 2101 policy > DOMAINSET_POLICY_MAX) 2102 return (EINVAL); 2103 /* In Capability mode, you can only set your own CPU set. */ 2104 if (IN_CAPABILITY_MODE(td)) { 2105 if (level != CPU_LEVEL_WHICH) 2106 return (ECAPMODE); 2107 if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 2108 return (ECAPMODE); 2109 if (id != -1) 2110 return (ECAPMODE); 2111 } 2112 memset(&domain, 0, sizeof(domain)); 2113 mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 2114 error = copyin(maskp, mask, domainsetsize); 2115 if (error) 2116 goto out; 2117 /* 2118 * Verify that no high bits are set. 2119 */ 2120 if (domainsetsize > sizeof(domainset_t)) { 2121 char *end; 2122 char *cp; 2123 2124 end = cp = (char *)&mask->__bits; 2125 end += domainsetsize; 2126 cp += sizeof(domainset_t); 2127 while (cp != end) 2128 if (*cp++ != 0) { 2129 error = EINVAL; 2130 goto out; 2131 } 2132 2133 } 2134 DOMAINSET_COPY(mask, &domain.ds_mask); 2135 domain.ds_policy = policy; 2136 2137 /* Translate preferred policy into a mask and fallback. */ 2138 if (policy == DOMAINSET_POLICY_PREFER) { 2139 /* Only support a single preferred domain. */ 2140 if (DOMAINSET_COUNT(&domain.ds_mask) != 1) { 2141 error = EINVAL; 2142 goto out; 2143 } 2144 domain.ds_prefer = DOMAINSET_FFS(&domain.ds_mask) - 1; 2145 /* This will be constrained by domainset_shadow(). */ 2146 DOMAINSET_FILL(&domain.ds_mask); 2147 } 2148 2149 /* 2150 * When given an impossible policy, fall back to interleaving 2151 * across all domains 2152 */ 2153 if (domainset_empty_vm(&domain)) 2154 domainset_copy(&domainset2, &domain); 2155 2156 switch (level) { 2157 case CPU_LEVEL_ROOT: 2158 case CPU_LEVEL_CPUSET: 2159 error = cpuset_which(which, id, &p, &ttd, &set); 2160 if (error) 2161 break; 2162 switch (which) { 2163 case CPU_WHICH_TID: 2164 case CPU_WHICH_PID: 2165 thread_lock(ttd); 2166 set = cpuset_ref(ttd->td_cpuset); 2167 thread_unlock(ttd); 2168 PROC_UNLOCK(p); 2169 break; 2170 case CPU_WHICH_CPUSET: 2171 case CPU_WHICH_JAIL: 2172 break; 2173 case CPU_WHICH_IRQ: 2174 case CPU_WHICH_INTRHANDLER: 2175 case CPU_WHICH_ITHREAD: 2176 case CPU_WHICH_DOMAIN: 2177 error = EINVAL; 2178 goto out; 2179 } 2180 if (level == CPU_LEVEL_ROOT) 2181 nset = cpuset_refroot(set); 2182 else 2183 nset = cpuset_refbase(set); 2184 error = cpuset_modify_domain(nset, &domain); 2185 cpuset_rel(nset); 2186 cpuset_rel(set); 2187 break; 2188 case CPU_LEVEL_WHICH: 2189 switch (which) { 2190 case CPU_WHICH_TID: 2191 error = _cpuset_setthread(id, NULL, &domain); 2192 break; 2193 case CPU_WHICH_PID: 2194 error = cpuset_setproc(id, NULL, NULL, &domain); 2195 break; 2196 case CPU_WHICH_CPUSET: 2197 case CPU_WHICH_JAIL: 2198 error = cpuset_which(which, id, &p, &ttd, &set); 2199 if (error == 0) { 2200 error = cpuset_modify_domain(set, &domain); 2201 cpuset_rel(set); 2202 } 2203 break; 2204 case CPU_WHICH_IRQ: 2205 case CPU_WHICH_INTRHANDLER: 2206 case CPU_WHICH_ITHREAD: 2207 default: 2208 error = EINVAL; 2209 break; 2210 } 2211 break; 2212 default: 2213 error = EINVAL; 2214 break; 2215 } 2216 out: 2217 free(mask, M_TEMP); 2218 return (error); 2219 } 2220 2221 #ifdef DDB 2222 2223 static void 2224 ddb_display_bitset(const struct bitset *set, int size) 2225 { 2226 int bit, once; 2227 2228 for (once = 0, bit = 0; bit < size; bit++) { 2229 if (CPU_ISSET(bit, set)) { 2230 if (once == 0) { 2231 db_printf("%d", bit); 2232 once = 1; 2233 } else 2234 db_printf(",%d", bit); 2235 } 2236 } 2237 if (once == 0) 2238 db_printf("<none>"); 2239 } 2240 2241 void 2242 ddb_display_cpuset(const cpuset_t *set) 2243 { 2244 ddb_display_bitset((const struct bitset *)set, CPU_SETSIZE); 2245 } 2246 2247 static void 2248 ddb_display_domainset(const domainset_t *set) 2249 { 2250 ddb_display_bitset((const struct bitset *)set, DOMAINSET_SETSIZE); 2251 } 2252 2253 DB_SHOW_COMMAND(cpusets, db_show_cpusets) 2254 { 2255 struct cpuset *set; 2256 2257 LIST_FOREACH(set, &cpuset_ids, cs_link) { 2258 db_printf("set=%p id=%-6u ref=%-6d flags=0x%04x parent id=%d\n", 2259 set, set->cs_id, set->cs_ref, set->cs_flags, 2260 (set->cs_parent != NULL) ? set->cs_parent->cs_id : 0); 2261 db_printf(" cpu mask="); 2262 ddb_display_cpuset(&set->cs_mask); 2263 db_printf("\n"); 2264 db_printf(" domain policy %d prefer %d mask=", 2265 set->cs_domain->ds_policy, set->cs_domain->ds_prefer); 2266 ddb_display_domainset(&set->cs_domain->ds_mask); 2267 db_printf("\n"); 2268 if (db_pager_quit) 2269 break; 2270 } 2271 } 2272 2273 DB_SHOW_COMMAND(domainsets, db_show_domainsets) 2274 { 2275 struct domainset *set; 2276 2277 LIST_FOREACH(set, &cpuset_domains, ds_link) { 2278 db_printf("set=%p policy %d prefer %d cnt %d\n", 2279 set, set->ds_policy, set->ds_prefer, set->ds_cnt); 2280 db_printf(" mask ="); 2281 ddb_display_domainset(&set->ds_mask); 2282 db_printf("\n"); 2283 } 2284 } 2285 #endif /* DDB */ 2286