1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_CPUMASK_H 3 #define __LINUX_CPUMASK_H 4 5 /* 6 * Cpumasks provide a bitmap suitable for representing the 7 * set of CPUs in a system, one bit position per CPU number. In general, 8 * only nr_cpu_ids (<= NR_CPUS) bits are valid. 9 */ 10 #include <linux/atomic.h> 11 #include <linux/bitmap.h> 12 #include <linux/cleanup.h> 13 #include <linux/cpumask_types.h> 14 #include <linux/gfp_types.h> 15 #include <linux/numa.h> 16 #include <linux/sprintf.h> 17 #include <linux/threads.h> 18 #include <linux/types.h> 19 #include <vdso/page.h> 20 21 #include <asm/bug.h> 22 23 /** 24 * cpumask_pr_args - printf args to output a cpumask 25 * @maskp: cpumask to be printed 26 * 27 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. 28 */ 29 #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) 30 31 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 32 #define nr_cpu_ids ((unsigned int)NR_CPUS) 33 #else 34 extern unsigned int nr_cpu_ids; 35 #endif 36 37 static __always_inline void set_nr_cpu_ids(unsigned int nr) 38 { 39 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) 40 WARN_ON(nr != nr_cpu_ids); 41 #else 42 nr_cpu_ids = nr; 43 #endif 44 } 45 46 /* 47 * We have several different "preferred sizes" for the cpumask 48 * operations, depending on operation. 49 * 50 * For example, the bitmap scanning and operating operations have 51 * optimized routines that work for the single-word case, but only when 52 * the size is constant. So if NR_CPUS fits in one single word, we are 53 * better off using that small constant, in order to trigger the 54 * optimized bit finding. That is 'small_cpumask_size'. 55 * 56 * The clearing and copying operations will similarly perform better 57 * with a constant size, but we limit that size arbitrarily to four 58 * words. We call this 'large_cpumask_size'. 59 * 60 * Finally, some operations just want the exact limit, either because 61 * they set bits or just don't have any faster fixed-sized versions. We 62 * call this just 'nr_cpumask_bits'. 63 * 64 * Note that these optional constants are always guaranteed to be at 65 * least as big as 'nr_cpu_ids' itself is, and all our cpumask 66 * allocations are at least that size (see cpumask_size()). The 67 * optimization comes from being able to potentially use a compile-time 68 * constant instead of a run-time generated exact number of CPUs. 69 */ 70 #if NR_CPUS <= BITS_PER_LONG 71 #define small_cpumask_bits ((unsigned int)NR_CPUS) 72 #define large_cpumask_bits ((unsigned int)NR_CPUS) 73 #elif NR_CPUS <= 4*BITS_PER_LONG 74 #define small_cpumask_bits nr_cpu_ids 75 #define large_cpumask_bits ((unsigned int)NR_CPUS) 76 #else 77 #define small_cpumask_bits nr_cpu_ids 78 #define large_cpumask_bits nr_cpu_ids 79 #endif 80 #define nr_cpumask_bits nr_cpu_ids 81 82 /* 83 * The following particular system cpumasks and operations manage 84 * possible, present, active and online cpus. 85 * 86 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable 87 * cpu_present_mask - has bit 'cpu' set iff cpu is populated 88 * cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online 89 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler 90 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration 91 * 92 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. 93 * 94 * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs 95 * that it is possible might ever be plugged in at anytime during the 96 * life of that system boot. The cpu_present_mask is dynamic(*), 97 * representing which CPUs are currently plugged in. And 98 * cpu_online_mask is the dynamic subset of cpu_present_mask, 99 * indicating those CPUs available for scheduling. 100 * 101 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, 102 * depending on what ACPI reports as currently plugged in, otherwise 103 * cpu_present_mask is just a copy of cpu_possible_mask. 104 * 105 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not 106 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. 107 * 108 * Subtleties: 109 * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode 110 * assumption that their single CPU is online. The UP 111 * cpu_{online,possible,present}_masks are placebos. Changing them 112 * will have no useful affect on the following num_*_cpus() 113 * and cpu_*() macros in the UP case. This ugliness is a UP 114 * optimization - don't waste any instructions or memory references 115 * asking if you're online or how many CPUs there are if there is 116 * only one CPU. 117 */ 118 119 extern struct cpumask __cpu_possible_mask; 120 extern struct cpumask __cpu_online_mask; 121 extern struct cpumask __cpu_enabled_mask; 122 extern struct cpumask __cpu_present_mask; 123 extern struct cpumask __cpu_active_mask; 124 extern struct cpumask __cpu_dying_mask; 125 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) 126 #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) 127 #define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask) 128 #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) 129 #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) 130 #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) 131 132 extern atomic_t __num_online_cpus; 133 extern unsigned int __num_possible_cpus; 134 135 extern cpumask_t cpus_booted_once_mask; 136 137 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) 138 { 139 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 140 WARN_ON_ONCE(cpu >= bits); 141 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */ 142 } 143 144 /* verify cpu argument to cpumask_* operators */ 145 static __always_inline unsigned int cpumask_check(unsigned int cpu) 146 { 147 cpu_max_bits_warn(cpu, small_cpumask_bits); 148 return cpu; 149 } 150 151 /** 152 * cpumask_first - get the first cpu in a cpumask 153 * @srcp: the cpumask pointer 154 * 155 * Return: >= nr_cpu_ids if no cpus set. 156 */ 157 static __always_inline unsigned int cpumask_first(const struct cpumask *srcp) 158 { 159 return find_first_bit(cpumask_bits(srcp), small_cpumask_bits); 160 } 161 162 /** 163 * cpumask_first_zero - get the first unset cpu in a cpumask 164 * @srcp: the cpumask pointer 165 * 166 * Return: >= nr_cpu_ids if all cpus are set. 167 */ 168 static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp) 169 { 170 return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits); 171 } 172 173 /** 174 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 175 * @srcp1: the first input 176 * @srcp2: the second input 177 * 178 * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). 179 */ 180 static __always_inline 181 unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) 182 { 183 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 184 } 185 186 /** 187 * cpumask_first_andnot - return the first cpu from *srcp1 & ~*srcp2 188 * @srcp1: the first input 189 * @srcp2: the second input 190 * 191 * Return: >= nr_cpu_ids if no such cpu found. 192 */ 193 static __always_inline 194 unsigned int cpumask_first_andnot(const struct cpumask *srcp1, const struct cpumask *srcp2) 195 { 196 return find_first_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 197 } 198 199 /** 200 * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3 201 * @srcp1: the first input 202 * @srcp2: the second input 203 * @srcp3: the third input 204 * 205 * Return: >= nr_cpu_ids if no cpus set in all. 206 */ 207 static __always_inline 208 unsigned int cpumask_first_and_and(const struct cpumask *srcp1, 209 const struct cpumask *srcp2, 210 const struct cpumask *srcp3) 211 { 212 return find_first_and_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 213 cpumask_bits(srcp3), small_cpumask_bits); 214 } 215 216 /** 217 * cpumask_last - get the last CPU in a cpumask 218 * @srcp: - the cpumask pointer 219 * 220 * Return: >= nr_cpumask_bits if no CPUs set. 221 */ 222 static __always_inline unsigned int cpumask_last(const struct cpumask *srcp) 223 { 224 return find_last_bit(cpumask_bits(srcp), small_cpumask_bits); 225 } 226 227 /** 228 * cpumask_next - get the next cpu in a cpumask 229 * @n: the cpu prior to the place to search (i.e. return will be > @n) 230 * @srcp: the cpumask pointer 231 * 232 * Return: >= nr_cpu_ids if no further cpus set. 233 */ 234 static __always_inline 235 unsigned int cpumask_next(int n, const struct cpumask *srcp) 236 { 237 /* -1 is a legal arg here. */ 238 if (n != -1) 239 cpumask_check(n); 240 return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1); 241 } 242 243 /** 244 * cpumask_next_zero - get the next unset cpu in a cpumask 245 * @n: the cpu prior to the place to search (i.e. return will be > @n) 246 * @srcp: the cpumask pointer 247 * 248 * Return: >= nr_cpu_ids if no further cpus unset. 249 */ 250 static __always_inline 251 unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 252 { 253 /* -1 is a legal arg here. */ 254 if (n != -1) 255 cpumask_check(n); 256 return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1); 257 } 258 259 #if NR_CPUS == 1 260 /* Uniprocessor: there is only one valid CPU */ 261 static __always_inline 262 unsigned int cpumask_local_spread(unsigned int i, int node) 263 { 264 return 0; 265 } 266 267 static __always_inline 268 unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 269 const struct cpumask *src2p) 270 { 271 return cpumask_first_and(src1p, src2p); 272 } 273 274 static __always_inline 275 unsigned int cpumask_any_distribute(const struct cpumask *srcp) 276 { 277 return cpumask_first(srcp); 278 } 279 #else 280 unsigned int cpumask_local_spread(unsigned int i, int node); 281 unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 282 const struct cpumask *src2p); 283 unsigned int cpumask_any_distribute(const struct cpumask *srcp); 284 #endif /* NR_CPUS */ 285 286 /** 287 * cpumask_next_and - get the next cpu in *src1p & *src2p 288 * @n: the cpu prior to the place to search (i.e. return will be > @n) 289 * @src1p: the first cpumask pointer 290 * @src2p: the second cpumask pointer 291 * 292 * Return: >= nr_cpu_ids if no further cpus set in both. 293 */ 294 static __always_inline 295 unsigned int cpumask_next_and(int n, const struct cpumask *src1p, 296 const struct cpumask *src2p) 297 { 298 /* -1 is a legal arg here. */ 299 if (n != -1) 300 cpumask_check(n); 301 return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p), 302 small_cpumask_bits, n + 1); 303 } 304 305 /** 306 * cpumask_next_andnot - get the next cpu in *src1p & ~*src2p 307 * @n: the cpu prior to the place to search (i.e. return will be > @n) 308 * @src1p: the first cpumask pointer 309 * @src2p: the second cpumask pointer 310 * 311 * Return: >= nr_cpu_ids if no further cpus set in both. 312 */ 313 static __always_inline 314 unsigned int cpumask_next_andnot(int n, const struct cpumask *src1p, 315 const struct cpumask *src2p) 316 { 317 /* -1 is a legal arg here. */ 318 if (n != -1) 319 cpumask_check(n); 320 return find_next_andnot_bit(cpumask_bits(src1p), cpumask_bits(src2p), 321 small_cpumask_bits, n + 1); 322 } 323 324 /** 325 * cpumask_next_and_wrap - get the next cpu in *src1p & *src2p, starting from 326 * @n+1. If nothing found, wrap around and start from 327 * the beginning 328 * @n: the cpu prior to the place to search (i.e. search starts from @n+1) 329 * @src1p: the first cpumask pointer 330 * @src2p: the second cpumask pointer 331 * 332 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src1p & @src2p is empty. 333 */ 334 static __always_inline 335 unsigned int cpumask_next_and_wrap(int n, const struct cpumask *src1p, 336 const struct cpumask *src2p) 337 { 338 /* -1 is a legal arg here. */ 339 if (n != -1) 340 cpumask_check(n); 341 return find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), 342 small_cpumask_bits, n + 1); 343 } 344 345 /** 346 * cpumask_next_wrap - get the next cpu in *src, starting from @n+1. If nothing 347 * found, wrap around and start from the beginning 348 * @n: the cpu prior to the place to search (i.e. search starts from @n+1) 349 * @src: cpumask pointer 350 * 351 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src is empty. 352 */ 353 static __always_inline 354 unsigned int cpumask_next_wrap(int n, const struct cpumask *src) 355 { 356 /* -1 is a legal arg here. */ 357 if (n != -1) 358 cpumask_check(n); 359 return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1); 360 } 361 362 /** 363 * cpumask_random - get random cpu in *src. 364 * @src: cpumask pointer 365 * 366 * Return: random set bit, or >= nr_cpu_ids if @src is empty. 367 */ 368 static __always_inline 369 unsigned int cpumask_random(const struct cpumask *src) 370 { 371 return find_random_bit(cpumask_bits(src), nr_cpu_ids); 372 } 373 374 /** 375 * for_each_cpu - iterate over every cpu in a mask 376 * @cpu: the (optionally unsigned) integer iterator 377 * @mask: the cpumask pointer 378 * 379 * After the loop, cpu is >= nr_cpu_ids. 380 */ 381 #define for_each_cpu(cpu, mask) \ 382 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits) 383 384 /** 385 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location 386 * @cpu: the (optionally unsigned) integer iterator 387 * @mask: the cpumask pointer 388 * @start: the start location 389 * 390 * The implementation does not assume any bit in @mask is set (including @start). 391 * 392 * After the loop, cpu is >= nr_cpu_ids. 393 */ 394 #define for_each_cpu_wrap(cpu, mask, start) \ 395 for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start) 396 397 /** 398 * for_each_cpu_and - iterate over every cpu in both masks 399 * @cpu: the (optionally unsigned) integer iterator 400 * @mask1: the first cpumask pointer 401 * @mask2: the second cpumask pointer 402 * 403 * This saves a temporary CPU mask in many places. It is equivalent to: 404 * struct cpumask tmp; 405 * cpumask_and(&tmp, &mask1, &mask2); 406 * for_each_cpu(cpu, &tmp) 407 * ... 408 * 409 * After the loop, cpu is >= nr_cpu_ids. 410 */ 411 #define for_each_cpu_and(cpu, mask1, mask2) \ 412 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 413 414 /** 415 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding 416 * those present in another. 417 * @cpu: the (optionally unsigned) integer iterator 418 * @mask1: the first cpumask pointer 419 * @mask2: the second cpumask pointer 420 * 421 * This saves a temporary CPU mask in many places. It is equivalent to: 422 * struct cpumask tmp; 423 * cpumask_andnot(&tmp, &mask1, &mask2); 424 * for_each_cpu(cpu, &tmp) 425 * ... 426 * 427 * After the loop, cpu is >= nr_cpu_ids. 428 */ 429 #define for_each_cpu_andnot(cpu, mask1, mask2) \ 430 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 431 432 /** 433 * for_each_cpu_or - iterate over every cpu present in either mask 434 * @cpu: the (optionally unsigned) integer iterator 435 * @mask1: the first cpumask pointer 436 * @mask2: the second cpumask pointer 437 * 438 * This saves a temporary CPU mask in many places. It is equivalent to: 439 * struct cpumask tmp; 440 * cpumask_or(&tmp, &mask1, &mask2); 441 * for_each_cpu(cpu, &tmp) 442 * ... 443 * 444 * After the loop, cpu is >= nr_cpu_ids. 445 */ 446 #define for_each_cpu_or(cpu, mask1, mask2) \ 447 for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits) 448 449 /** 450 * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask. 451 * @cpu: the (optionally unsigned) integer iterator 452 * @mask: the cpumask pointer 453 * 454 * After the loop, cpu is >= nr_cpu_ids. 455 */ 456 #define for_each_cpu_from(cpu, mask) \ 457 for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits) 458 459 /** 460 * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one. 461 * @mask: the cpumask to search 462 * @cpu: the cpu to ignore. 463 * 464 * Often used to find any cpu but smp_processor_id() in a mask. 465 * If @cpu == -1, the function is equivalent to cpumask_any(). 466 * Return: >= nr_cpu_ids if no cpus set. 467 */ 468 static __always_inline 469 unsigned int cpumask_any_but(const struct cpumask *mask, int cpu) 470 { 471 unsigned int i; 472 473 /* -1 is a legal arg here. */ 474 if (cpu != -1) 475 cpumask_check(cpu); 476 477 for_each_cpu(i, mask) 478 if (i != cpu) 479 break; 480 return i; 481 } 482 483 /** 484 * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one. 485 * @mask1: the first input cpumask 486 * @mask2: the second input cpumask 487 * @cpu: the cpu to ignore 488 * 489 * If @cpu == -1, the function is equivalent to cpumask_any_and(). 490 * Returns >= nr_cpu_ids if no cpus set. 491 */ 492 static __always_inline 493 unsigned int cpumask_any_and_but(const struct cpumask *mask1, 494 const struct cpumask *mask2, 495 int cpu) 496 { 497 unsigned int i; 498 499 /* -1 is a legal arg here. */ 500 if (cpu != -1) 501 cpumask_check(cpu); 502 503 i = cpumask_first_and(mask1, mask2); 504 if (i != cpu) 505 return i; 506 507 return cpumask_next_and(cpu, mask1, mask2); 508 } 509 510 /** 511 * cpumask_any_andnot_but - pick an arbitrary cpu from *mask1 & ~*mask2, but not this one. 512 * @mask1: the first input cpumask 513 * @mask2: the second input cpumask 514 * @cpu: the cpu to ignore 515 * 516 * If @cpu == -1, the function returns the first matching cpu. 517 * Returns >= nr_cpu_ids if no cpus set. 518 */ 519 static __always_inline 520 unsigned int cpumask_any_andnot_but(const struct cpumask *mask1, 521 const struct cpumask *mask2, 522 int cpu) 523 { 524 unsigned int i; 525 526 /* -1 is a legal arg here. */ 527 if (cpu != -1) 528 cpumask_check(cpu); 529 530 i = cpumask_first_andnot(mask1, mask2); 531 if (i != cpu) 532 return i; 533 534 return cpumask_next_andnot(cpu, mask1, mask2); 535 } 536 537 /** 538 * cpumask_nth - get the Nth cpu in a cpumask 539 * @srcp: the cpumask pointer 540 * @cpu: the Nth cpu to find, starting from 0 541 * 542 * Return: >= nr_cpu_ids if such cpu doesn't exist. 543 */ 544 static __always_inline 545 unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) 546 { 547 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); 548 } 549 550 /** 551 * cpumask_nth_and - get the Nth cpu in 2 cpumasks 552 * @srcp1: the cpumask pointer 553 * @srcp2: the cpumask pointer 554 * @cpu: the Nth cpu to find, starting from 0 555 * 556 * Return: >= nr_cpu_ids if such cpu doesn't exist. 557 */ 558 static __always_inline 559 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, 560 const struct cpumask *srcp2) 561 { 562 return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), 563 small_cpumask_bits, cpumask_check(cpu)); 564 } 565 566 /** 567 * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd. 568 * @srcp1: the cpumask pointer 569 * @srcp2: the cpumask pointer 570 * @srcp3: the cpumask pointer 571 * @cpu: the Nth cpu to find, starting from 0 572 * 573 * Return: >= nr_cpu_ids if such cpu doesn't exist. 574 */ 575 static __always_inline 576 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, 577 const struct cpumask *srcp2, 578 const struct cpumask *srcp3) 579 { 580 return find_nth_and_andnot_bit(cpumask_bits(srcp1), 581 cpumask_bits(srcp2), 582 cpumask_bits(srcp3), 583 small_cpumask_bits, cpumask_check(cpu)); 584 } 585 586 #define CPU_BITS_NONE \ 587 { \ 588 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 589 } 590 591 #define CPU_BITS_CPU0 \ 592 { \ 593 [0] = 1UL \ 594 } 595 596 /** 597 * cpumask_set_cpu - set a cpu in a cpumask 598 * @cpu: cpu number (< nr_cpu_ids) 599 * @dstp: the cpumask pointer 600 */ 601 static __always_inline 602 void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 603 { 604 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 605 } 606 607 static __always_inline 608 void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 609 { 610 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 611 } 612 613 /** 614 * cpumask_clear_cpus - clear cpus in a cpumask 615 * @dstp: the cpumask pointer 616 * @cpu: cpu number (< nr_cpu_ids) 617 * @ncpus: number of cpus to clear (< nr_cpu_ids) 618 */ 619 static __always_inline void cpumask_clear_cpus(struct cpumask *dstp, 620 unsigned int cpu, unsigned int ncpus) 621 { 622 cpumask_check(cpu + ncpus - 1); 623 bitmap_clear(cpumask_bits(dstp), cpumask_check(cpu), ncpus); 624 } 625 626 /** 627 * cpumask_clear_cpu - clear a cpu in a cpumask 628 * @cpu: cpu number (< nr_cpu_ids) 629 * @dstp: the cpumask pointer 630 */ 631 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) 632 { 633 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 634 } 635 636 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) 637 { 638 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 639 } 640 641 /** 642 * cpumask_test_cpu - test for a cpu in a cpumask 643 * @cpu: cpu number (< nr_cpu_ids) 644 * @cpumask: the cpumask pointer 645 * 646 * Return: true if @cpu is set in @cpumask, else returns false 647 */ 648 static __always_inline 649 bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) 650 { 651 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 652 } 653 654 /** 655 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask 656 * @cpu: cpu number (< nr_cpu_ids) 657 * @cpumask: the cpumask pointer 658 * 659 * test_and_set_bit wrapper for cpumasks. 660 * 661 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false 662 */ 663 static __always_inline 664 bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) 665 { 666 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 667 } 668 669 /** 670 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask 671 * @cpu: cpu number (< nr_cpu_ids) 672 * @cpumask: the cpumask pointer 673 * 674 * test_and_clear_bit wrapper for cpumasks. 675 * 676 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false 677 */ 678 static __always_inline 679 bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) 680 { 681 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 682 } 683 684 /** 685 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask 686 * @dstp: the cpumask pointer 687 */ 688 static __always_inline void cpumask_setall(struct cpumask *dstp) 689 { 690 if (small_const_nbits(small_cpumask_bits)) { 691 cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits); 692 return; 693 } 694 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits); 695 } 696 697 /** 698 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask 699 * @dstp: the cpumask pointer 700 */ 701 static __always_inline void cpumask_clear(struct cpumask *dstp) 702 { 703 bitmap_zero(cpumask_bits(dstp), large_cpumask_bits); 704 } 705 706 /** 707 * cpumask_and - *dstp = *src1p & *src2p 708 * @dstp: the cpumask result 709 * @src1p: the first input 710 * @src2p: the second input 711 * 712 * Return: false if *@dstp is empty, else returns true 713 */ 714 static __always_inline 715 bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, 716 const struct cpumask *src2p) 717 { 718 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), 719 cpumask_bits(src2p), small_cpumask_bits); 720 } 721 722 /** 723 * cpumask_or - *dstp = *src1p | *src2p 724 * @dstp: the cpumask result 725 * @src1p: the first input 726 * @src2p: the second input 727 */ 728 static __always_inline 729 void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, 730 const struct cpumask *src2p) 731 { 732 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), 733 cpumask_bits(src2p), small_cpumask_bits); 734 } 735 736 /** 737 * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result 738 * @dstp: the cpumask result 739 * @src1p: the first input 740 * @src2p: the second input 741 * 742 * Return: The number of bits set in the resulting cpumask @dstp 743 */ 744 static __always_inline 745 unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p, 746 const struct cpumask *src2p) 747 { 748 return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p), 749 cpumask_bits(src2p), small_cpumask_bits); 750 } 751 752 /** 753 * cpumask_xor - *dstp = *src1p ^ *src2p 754 * @dstp: the cpumask result 755 * @src1p: the first input 756 * @src2p: the second input 757 */ 758 static __always_inline 759 void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p, 760 const struct cpumask *src2p) 761 { 762 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), 763 cpumask_bits(src2p), small_cpumask_bits); 764 } 765 766 /** 767 * cpumask_andnot - *dstp = *src1p & ~*src2p 768 * @dstp: the cpumask result 769 * @src1p: the first input 770 * @src2p: the second input 771 * 772 * Return: false if *@dstp is empty, else returns true 773 */ 774 static __always_inline 775 bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, 776 const struct cpumask *src2p) 777 { 778 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), 779 cpumask_bits(src2p), small_cpumask_bits); 780 } 781 782 /** 783 * cpumask_equal - *src1p == *src2p 784 * @src1p: the first input 785 * @src2p: the second input 786 * 787 * Return: true if the cpumasks are equal, false if not 788 */ 789 static __always_inline 790 bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) 791 { 792 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), 793 small_cpumask_bits); 794 } 795 796 /** 797 * cpumask_or_equal - *src1p | *src2p == *src3p 798 * @src1p: the first input 799 * @src2p: the second input 800 * @src3p: the third input 801 * 802 * Return: true if first cpumask ORed with second cpumask == third cpumask, 803 * otherwise false 804 */ 805 static __always_inline 806 bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p, 807 const struct cpumask *src3p) 808 { 809 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), 810 cpumask_bits(src3p), small_cpumask_bits); 811 } 812 813 /** 814 * cpumask_intersects - (*src1p & *src2p) != 0 815 * @src1p: the first input 816 * @src2p: the second input 817 * 818 * Return: true if first cpumask ANDed with second cpumask is non-empty, 819 * otherwise false 820 */ 821 static __always_inline 822 bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) 823 { 824 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), 825 small_cpumask_bits); 826 } 827 828 /** 829 * cpumask_subset - (*src1p & ~*src2p) == 0 830 * @src1p: the first input 831 * @src2p: the second input 832 * 833 * Return: true if *@src1p is a subset of *@src2p, else returns false 834 */ 835 static __always_inline 836 bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) 837 { 838 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), 839 small_cpumask_bits); 840 } 841 842 /** 843 * cpumask_empty - *srcp == 0 844 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. 845 * 846 * Return: true if srcp is empty (has no bits set), else false 847 */ 848 static __always_inline bool cpumask_empty(const struct cpumask *srcp) 849 { 850 return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits); 851 } 852 853 /** 854 * cpumask_full - *srcp == 0xFFFFFFFF... 855 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. 856 * 857 * Return: true if srcp is full (has all bits set), else false 858 */ 859 static __always_inline bool cpumask_full(const struct cpumask *srcp) 860 { 861 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); 862 } 863 864 /** 865 * cpumask_weight - Count of bits in *srcp 866 * @srcp: the cpumask to count bits (< nr_cpu_ids) in. 867 * 868 * Return: count of bits set in *srcp 869 */ 870 static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp) 871 { 872 return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits); 873 } 874 875 /** 876 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) 877 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 878 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 879 * 880 * Return: count of bits set in both *srcp1 and *srcp2 881 */ 882 static __always_inline 883 unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2) 884 { 885 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 886 } 887 888 /** 889 * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2) 890 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. 891 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. 892 * 893 * Return: count of bits set in both *srcp1 and *srcp2 894 */ 895 static __always_inline 896 unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, 897 const struct cpumask *srcp2) 898 { 899 return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); 900 } 901 902 /** 903 * cpumask_shift_right - *dstp = *srcp >> n 904 * @dstp: the cpumask result 905 * @srcp: the input to shift 906 * @n: the number of bits to shift by 907 */ 908 static __always_inline 909 void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n) 910 { 911 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, 912 small_cpumask_bits); 913 } 914 915 /** 916 * cpumask_shift_left - *dstp = *srcp << n 917 * @dstp: the cpumask result 918 * @srcp: the input to shift 919 * @n: the number of bits to shift by 920 */ 921 static __always_inline 922 void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n) 923 { 924 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, 925 nr_cpumask_bits); 926 } 927 928 /** 929 * cpumask_copy - *dstp = *srcp 930 * @dstp: the result 931 * @srcp: the input cpumask 932 */ 933 static __always_inline 934 void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) 935 { 936 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits); 937 } 938 939 /** 940 * cpumask_any - pick an arbitrary cpu from *srcp 941 * @srcp: the input cpumask 942 * 943 * Return: >= nr_cpu_ids if no cpus set. 944 */ 945 #define cpumask_any(srcp) cpumask_first(srcp) 946 947 /** 948 * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2 949 * @mask1: the first input cpumask 950 * @mask2: the second input cpumask 951 * 952 * Return: >= nr_cpu_ids if no cpus set. 953 */ 954 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) 955 956 /** 957 * cpumask_of - the cpumask containing just a given cpu 958 * @cpu: the cpu (<= nr_cpu_ids) 959 */ 960 #define cpumask_of(cpu) (get_cpu_mask(cpu)) 961 962 /** 963 * cpumask_parse_user - extract a cpumask from a user string 964 * @buf: the buffer to extract from 965 * @len: the length of the buffer 966 * @dstp: the cpumask to set. 967 * 968 * Return: -errno, or 0 for success. 969 */ 970 static __always_inline 971 int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) 972 { 973 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); 974 } 975 976 /** 977 * cpumask_parselist_user - extract a cpumask from a user string 978 * @buf: the buffer to extract from 979 * @len: the length of the buffer 980 * @dstp: the cpumask to set. 981 * 982 * Return: -errno, or 0 for success. 983 */ 984 static __always_inline 985 int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) 986 { 987 return bitmap_parselist_user(buf, len, cpumask_bits(dstp), 988 nr_cpumask_bits); 989 } 990 991 /** 992 * cpumask_parse - extract a cpumask from a string 993 * @buf: the buffer to extract from 994 * @dstp: the cpumask to set. 995 * 996 * Return: -errno, or 0 for success. 997 */ 998 static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp) 999 { 1000 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); 1001 } 1002 1003 /** 1004 * cpulist_parse - extract a cpumask from a user string of ranges 1005 * @buf: the buffer to extract from 1006 * @dstp: the cpumask to set. 1007 * 1008 * Return: -errno, or 0 for success. 1009 */ 1010 static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp) 1011 { 1012 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); 1013 } 1014 1015 /** 1016 * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes 1017 * 1018 * Return: size to allocate for a &struct cpumask in bytes 1019 */ 1020 static __always_inline unsigned int cpumask_size(void) 1021 { 1022 return bitmap_size(large_cpumask_bits); 1023 } 1024 1025 #ifdef CONFIG_CPUMASK_OFFSTACK 1026 1027 #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x) 1028 #define __cpumask_var_read_mostly __read_mostly 1029 #define CPUMASK_VAR_NULL NULL 1030 1031 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); 1032 1033 static __always_inline 1034 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) 1035 { 1036 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); 1037 } 1038 1039 /** 1040 * alloc_cpumask_var - allocate a struct cpumask 1041 * @mask: pointer to cpumask_var_t where the cpumask is returned 1042 * @flags: GFP_ flags 1043 * 1044 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is 1045 * a nop returning a constant 1 (in <linux/cpumask.h>). 1046 * 1047 * See alloc_cpumask_var_node. 1048 * 1049 * Return: %true if allocation succeeded, %false if not 1050 */ 1051 static __always_inline 1052 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1053 { 1054 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); 1055 } 1056 1057 static __always_inline 1058 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1059 { 1060 return alloc_cpumask_var(mask, flags | __GFP_ZERO); 1061 } 1062 1063 void alloc_bootmem_cpumask_var(cpumask_var_t *mask); 1064 void free_cpumask_var(cpumask_var_t mask); 1065 void free_bootmem_cpumask_var(cpumask_var_t mask); 1066 1067 static __always_inline bool cpumask_available(cpumask_var_t mask) 1068 { 1069 return mask != NULL; 1070 } 1071 1072 #else 1073 1074 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) 1075 #define __cpumask_var_read_mostly 1076 #define CPUMASK_VAR_NULL {} 1077 1078 static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1079 { 1080 return true; 1081 } 1082 1083 static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 1084 int node) 1085 { 1086 return true; 1087 } 1088 1089 static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 1090 { 1091 cpumask_clear(*mask); 1092 return true; 1093 } 1094 1095 static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 1096 int node) 1097 { 1098 cpumask_clear(*mask); 1099 return true; 1100 } 1101 1102 static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) 1103 { 1104 } 1105 1106 static __always_inline void free_cpumask_var(cpumask_var_t mask) 1107 { 1108 } 1109 1110 static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask) 1111 { 1112 } 1113 1114 static __always_inline bool cpumask_available(cpumask_var_t mask) 1115 { 1116 return true; 1117 } 1118 #endif /* CONFIG_CPUMASK_OFFSTACK */ 1119 1120 DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T)); 1121 1122 /* It's common to want to use cpu_all_mask in struct member initializers, 1123 * so it has to refer to an address rather than a pointer. */ 1124 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); 1125 #define cpu_all_mask to_cpumask(cpu_all_bits) 1126 1127 /* First bits of cpu_bit_bitmap are in fact unset. */ 1128 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0]) 1129 1130 #if NR_CPUS == 1 1131 /* Uniprocessor: the possible/online/present masks are always "1" */ 1132 #define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1133 #define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1134 #define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++) 1135 1136 #define for_each_possible_cpu_wrap(cpu, start) \ 1137 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++) 1138 #define for_each_online_cpu_wrap(cpu, start) \ 1139 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++) 1140 #else 1141 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) 1142 #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) 1143 #define for_each_enabled_cpu(cpu) for_each_cpu((cpu), cpu_enabled_mask) 1144 #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) 1145 1146 #define for_each_possible_cpu_wrap(cpu, start) \ 1147 for_each_cpu_wrap((cpu), cpu_possible_mask, (start)) 1148 #define for_each_online_cpu_wrap(cpu, start) \ 1149 for_each_cpu_wrap((cpu), cpu_online_mask, (start)) 1150 #endif 1151 1152 /* Wrappers for arch boot code to manipulate normally-constant masks */ 1153 void init_cpu_present(const struct cpumask *src); 1154 void init_cpu_possible(const struct cpumask *src); 1155 1156 #define assign_cpu(cpu, mask, val) \ 1157 assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val)) 1158 1159 #define __assign_cpu(cpu, mask, val) \ 1160 __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val)) 1161 1162 #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled)) 1163 #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present)) 1164 #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) 1165 #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) 1166 1167 void set_cpu_online(unsigned int cpu, bool online); 1168 void set_cpu_possible(unsigned int cpu, bool possible); 1169 1170 /** 1171 * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask * 1172 * @bitmap: the bitmap 1173 * 1174 * There are a few places where cpumask_var_t isn't appropriate and 1175 * static cpumasks must be used (eg. very early boot), yet we don't 1176 * expose the definition of 'struct cpumask'. 1177 * 1178 * This does the conversion, and can be used as a constant initializer. 1179 */ 1180 #define to_cpumask(bitmap) \ 1181 ((struct cpumask *)(1 ? (bitmap) \ 1182 : (void *)sizeof(__check_is_bitmap(bitmap)))) 1183 1184 static __always_inline int __check_is_bitmap(const unsigned long *bitmap) 1185 { 1186 return 1; 1187 } 1188 1189 /* 1190 * Special-case data structure for "single bit set only" constant CPU masks. 1191 * 1192 * We pre-generate all the 64 (or 32) possible bit positions, with enough 1193 * padding to the left and the right, and return the constant pointer 1194 * appropriately offset. 1195 */ 1196 extern const unsigned long 1197 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; 1198 1199 static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu) 1200 { 1201 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; 1202 p -= cpu / BITS_PER_LONG; 1203 return to_cpumask(p); 1204 } 1205 1206 #if NR_CPUS > 1 1207 /** 1208 * num_online_cpus() - Read the number of online CPUs 1209 * 1210 * Despite the fact that __num_online_cpus is of type atomic_t, this 1211 * interface gives only a momentary snapshot and is not protected against 1212 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held 1213 * region. 1214 * 1215 * Return: momentary snapshot of the number of online CPUs 1216 */ 1217 static __always_inline unsigned int num_online_cpus(void) 1218 { 1219 return raw_atomic_read(&__num_online_cpus); 1220 } 1221 1222 static __always_inline unsigned int num_possible_cpus(void) 1223 { 1224 return __num_possible_cpus; 1225 } 1226 1227 #define num_enabled_cpus() cpumask_weight(cpu_enabled_mask) 1228 #define num_present_cpus() cpumask_weight(cpu_present_mask) 1229 #define num_active_cpus() cpumask_weight(cpu_active_mask) 1230 1231 static __always_inline bool cpu_online(unsigned int cpu) 1232 { 1233 return cpumask_test_cpu(cpu, cpu_online_mask); 1234 } 1235 1236 static __always_inline bool cpu_enabled(unsigned int cpu) 1237 { 1238 return cpumask_test_cpu(cpu, cpu_enabled_mask); 1239 } 1240 1241 static __always_inline bool cpu_possible(unsigned int cpu) 1242 { 1243 return cpumask_test_cpu(cpu, cpu_possible_mask); 1244 } 1245 1246 static __always_inline bool cpu_present(unsigned int cpu) 1247 { 1248 return cpumask_test_cpu(cpu, cpu_present_mask); 1249 } 1250 1251 static __always_inline bool cpu_active(unsigned int cpu) 1252 { 1253 return cpumask_test_cpu(cpu, cpu_active_mask); 1254 } 1255 1256 static __always_inline bool cpu_dying(unsigned int cpu) 1257 { 1258 return cpumask_test_cpu(cpu, cpu_dying_mask); 1259 } 1260 1261 #else 1262 1263 #define num_online_cpus() 1U 1264 #define num_possible_cpus() 1U 1265 #define num_enabled_cpus() 1U 1266 #define num_present_cpus() 1U 1267 #define num_active_cpus() 1U 1268 1269 static __always_inline bool cpu_online(unsigned int cpu) 1270 { 1271 return cpu == 0; 1272 } 1273 1274 static __always_inline bool cpu_possible(unsigned int cpu) 1275 { 1276 return cpu == 0; 1277 } 1278 1279 static __always_inline bool cpu_enabled(unsigned int cpu) 1280 { 1281 return cpu == 0; 1282 } 1283 1284 static __always_inline bool cpu_present(unsigned int cpu) 1285 { 1286 return cpu == 0; 1287 } 1288 1289 static __always_inline bool cpu_active(unsigned int cpu) 1290 { 1291 return cpu == 0; 1292 } 1293 1294 static __always_inline bool cpu_dying(unsigned int cpu) 1295 { 1296 return false; 1297 } 1298 1299 #endif /* NR_CPUS > 1 */ 1300 1301 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) 1302 1303 #if NR_CPUS <= BITS_PER_LONG 1304 #define CPU_BITS_ALL \ 1305 { \ 1306 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1307 } 1308 1309 #else /* NR_CPUS > BITS_PER_LONG */ 1310 1311 #define CPU_BITS_ALL \ 1312 { \ 1313 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1314 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1315 } 1316 #endif /* NR_CPUS > BITS_PER_LONG */ 1317 1318 /** 1319 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either 1320 * as comma-separated list of cpus or hex values of cpumask 1321 * @list: indicates whether the cpumap must be list 1322 * @mask: the cpumask to copy 1323 * @buf: the buffer to copy into 1324 * 1325 * Return: the length of the (null-terminated) @buf string, zero if 1326 * nothing is copied. 1327 */ 1328 static __always_inline ssize_t 1329 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) 1330 { 1331 /* Opencode offset_in_page(buf) to not include linux/mm.h */ 1332 return scnprintf(buf, PAGE_SIZE - ((unsigned long)buf & ~PAGE_MASK), 1333 list ? "%*pbl\n" : "%*pb\n", cpumask_pr_args(mask)); 1334 } 1335 1336 /** 1337 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as 1338 * hex values of cpumask 1339 * 1340 * @buf: the buffer to copy into 1341 * @mask: the cpumask to copy 1342 * @off: in the string from which we are copying, we copy to @buf 1343 * @count: the maximum number of bytes to print 1344 * 1345 * The function prints the cpumask into the buffer as hex values of 1346 * cpumask; Typically used by bin_attribute to export cpumask bitmask 1347 * ABI. 1348 * 1349 * Return: the length of how many bytes have been copied, excluding 1350 * terminating '\0'. 1351 */ 1352 static __always_inline 1353 ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, 1354 loff_t off, size_t count) 1355 { 1356 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask), 1357 nr_cpu_ids, off, count) - 1; 1358 } 1359 1360 /** 1361 * cpumap_print_list_to_buf - copies the cpumask into the buffer as 1362 * comma-separated list of cpus 1363 * @buf: the buffer to copy into 1364 * @mask: the cpumask to copy 1365 * @off: in the string from which we are copying, we copy to @buf 1366 * @count: the maximum number of bytes to print 1367 * 1368 * Everything is same with the above cpumap_print_bitmask_to_buf() 1369 * except the print format. 1370 * 1371 * Return: the length of how many bytes have been copied, excluding 1372 * terminating '\0'. 1373 */ 1374 static __always_inline 1375 ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, 1376 loff_t off, size_t count) 1377 { 1378 return bitmap_print_list_to_buf(buf, cpumask_bits(mask), 1379 nr_cpu_ids, off, count) - 1; 1380 } 1381 1382 #if NR_CPUS <= BITS_PER_LONG 1383 #define CPU_MASK_ALL \ 1384 (cpumask_t) { { \ 1385 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1386 } } 1387 #else 1388 #define CPU_MASK_ALL \ 1389 (cpumask_t) { { \ 1390 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 1391 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 1392 } } 1393 #endif /* NR_CPUS > BITS_PER_LONG */ 1394 1395 #define CPU_MASK_NONE \ 1396 (cpumask_t) { { \ 1397 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 1398 } } 1399 1400 #define CPU_MASK_CPU0 \ 1401 (cpumask_t) { { \ 1402 [0] = 1UL \ 1403 } } 1404 1405 /* 1406 * Provide a valid theoretical max size for cpumap and cpulist sysfs files 1407 * to avoid breaking userspace which may allocate a buffer based on the size 1408 * reported by e.g. fstat. 1409 * 1410 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length. 1411 * 1412 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up 1413 * to 2 orders of magnitude larger than 8192. And then we divide by 2 to 1414 * cover a worst-case of every other cpu being on one of two nodes for a 1415 * very large NR_CPUS. 1416 * 1417 * Use PAGE_SIZE as a minimum for smaller configurations while avoiding 1418 * unsigned comparison to -1. 1419 */ 1420 #define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \ 1421 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE) 1422 #define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE) 1423 1424 #endif /* __LINUX_CPUMASK_H */ 1425