1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020 Kyle Evans <kevans@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD"); 30 31 #include <sys/param.h> 32 #include <sys/cpuset.h> 33 #include <sys/jail.h> 34 #include <sys/socket.h> 35 #include <sys/uio.h> 36 #include <sys/wait.h> 37 38 #include <errno.h> 39 #include <stdio.h> 40 #include <unistd.h> 41 42 #include <atf-c.h> 43 44 #define SP_PARENT 0 45 #define SP_CHILD 1 46 47 struct jail_test_info { 48 cpuset_t jail_tidmask; 49 cpusetid_t jail_cpuset; 50 cpusetid_t jail_child_cpuset; 51 }; 52 53 struct jail_test_cb_params { 54 struct jail_test_info info; 55 cpuset_t mask; 56 cpusetid_t rootid; 57 cpusetid_t setid; 58 }; 59 60 typedef void (*jail_test_cb)(struct jail_test_cb_params *); 61 62 #define FAILURE_JAIL 42 63 #define FAILURE_MASK 43 64 #define FAILURE_JAILSET 44 65 #define FAILURE_PIDSET 45 66 #define FAILURE_SEND 46 67 68 static const char * 69 do_jail_errstr(int error) 70 { 71 72 switch (error) { 73 case FAILURE_JAIL: 74 return ("jail_set(2) failed"); 75 case FAILURE_MASK: 76 return ("Failed to get the thread cpuset mask"); 77 case FAILURE_JAILSET: 78 return ("Failed to get the jail setid"); 79 case FAILURE_PIDSET: 80 return ("Failed to get the pid setid"); 81 case FAILURE_SEND: 82 return ("Failed to send(2) cpuset information"); 83 default: 84 return (NULL); 85 } 86 } 87 88 static void 89 skip_ltncpu(int ncpu, cpuset_t *mask) 90 { 91 92 CPU_ZERO(mask); 93 ATF_REQUIRE_EQ(0, cpuset_getaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, 94 -1, sizeof(*mask), mask)); 95 if (CPU_COUNT(mask) < ncpu) 96 atf_tc_skip("Test requires %d or more cores.", ncpu); 97 } 98 99 ATF_TC(newset); 100 ATF_TC_HEAD(newset, tc) 101 { 102 atf_tc_set_md_var(tc, "descr", "Test cpuset(2)"); 103 } 104 ATF_TC_BODY(newset, tc) 105 { 106 cpusetid_t nsetid, setid, qsetid; 107 108 /* Obtain our initial set id. */ 109 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_TID, -1, 110 &setid)); 111 112 /* Create a new one. */ 113 ATF_REQUIRE_EQ(0, cpuset(&nsetid)); 114 ATF_CHECK(nsetid != setid); 115 116 /* Query id again, make sure it's equal to the one we just got. */ 117 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_TID, -1, 118 &qsetid)); 119 ATF_CHECK_EQ(nsetid, qsetid); 120 } 121 122 ATF_TC(transient); 123 ATF_TC_HEAD(transient, tc) 124 { 125 atf_tc_set_md_var(tc, "descr", 126 "Test that transient cpusets are freed."); 127 } 128 ATF_TC_BODY(transient, tc) 129 { 130 cpusetid_t isetid, scratch, setid; 131 132 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, 133 &isetid)); 134 135 ATF_REQUIRE_EQ(0, cpuset(&setid)); 136 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, 137 setid, &scratch)); 138 139 /* 140 * Return back to our initial cpuset; the kernel should free the cpuset 141 * we just created. 142 */ 143 ATF_REQUIRE_EQ(0, cpuset_setid(CPU_WHICH_PID, -1, isetid)); 144 ATF_REQUIRE_EQ(-1, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_CPUSET, 145 setid, &scratch)); 146 ATF_CHECK_EQ(ESRCH, errno); 147 } 148 149 ATF_TC(deadlk); 150 ATF_TC_HEAD(deadlk, tc) 151 { 152 atf_tc_set_md_var(tc, "descr", "Test against disjoint cpusets."); 153 atf_tc_set_md_var(tc, "require.user", "root"); 154 } 155 ATF_TC_BODY(deadlk, tc) 156 { 157 cpusetid_t setid; 158 cpuset_t dismask, mask, omask; 159 int fcpu, i, found, ncpu, second; 160 161 /* Make sure we have 3 cpus, so we test partial overlap. */ 162 skip_ltncpu(3, &omask); 163 164 ATF_REQUIRE_EQ(0, cpuset(&setid)); 165 CPU_ZERO(&mask); 166 CPU_ZERO(&dismask); 167 CPU_COPY(&omask, &mask); 168 CPU_COPY(&omask, &dismask); 169 fcpu = CPU_FFS(&mask); 170 ncpu = CPU_COUNT(&mask); 171 172 /* 173 * Turn off all but the first two for mask, turn off the first for 174 * dismask and turn them all off for both after the third. 175 */ 176 for (i = fcpu - 1, found = 0; i < CPU_MAXSIZE && found != ncpu; i++) { 177 if (CPU_ISSET(i, &omask)) { 178 found++; 179 if (found == 1) { 180 CPU_CLR(i, &dismask); 181 } else if (found == 2) { 182 second = i; 183 } else if (found >= 3) { 184 CPU_CLR(i, &mask); 185 if (found > 3) 186 CPU_CLR(i, &dismask); 187 } 188 } 189 } 190 191 ATF_REQUIRE_EQ(0, cpuset_setaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, 192 -1, sizeof(mask), &mask)); 193 194 /* Must be a strict subset! */ 195 ATF_REQUIRE_EQ(-1, cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 196 -1, sizeof(dismask), &dismask)); 197 ATF_REQUIRE_EQ(EINVAL, errno); 198 199 /* 200 * We'll set our anonymous set to the 0,1 set that currently matches 201 * the process. If we then set the process to the 1,2 set that's in 202 * dismask, we should then personally be restricted down to the single 203 * overlapping CPOU. 204 */ 205 ATF_REQUIRE_EQ(0, cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 206 -1, sizeof(mask), &mask)); 207 ATF_REQUIRE_EQ(0, cpuset_setaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, 208 -1, sizeof(dismask), &dismask)); 209 ATF_REQUIRE_EQ(0, cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 210 -1, sizeof(mask), &mask)); 211 ATF_REQUIRE_EQ(1, CPU_COUNT(&mask)); 212 ATF_REQUIRE(CPU_ISSET(second, &mask)); 213 214 /* 215 * Finally, clearing the overlap and attempting to set the process 216 * cpuset to a completely disjoint mask should fail, because this 217 * process will then not have anything to run on. 218 */ 219 CPU_CLR(second, &dismask); 220 ATF_REQUIRE_EQ(-1, cpuset_setaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, 221 -1, sizeof(dismask), &dismask)); 222 ATF_REQUIRE_EQ(EDEADLK, errno); 223 } 224 225 static int 226 do_jail(int sock) 227 { 228 struct jail_test_info info; 229 struct iovec iov[2]; 230 char *name; 231 int error; 232 233 if (asprintf(&name, "cpuset_%d", getpid()) == -1) 234 _exit(42); 235 236 iov[0].iov_base = "name"; 237 iov[0].iov_len = 5; 238 239 iov[1].iov_base = name; 240 iov[1].iov_len = strlen(name) + 1; 241 242 if (jail_set(iov, 2, JAIL_CREATE | JAIL_ATTACH) < 0) 243 return (FAILURE_JAIL); 244 245 /* Record parameters, kick them over, then make a swift exit. */ 246 CPU_ZERO(&info.jail_tidmask); 247 error = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 248 -1, sizeof(info.jail_tidmask), &info.jail_tidmask); 249 if (error != 0) 250 return (FAILURE_MASK); 251 252 error = cpuset_getid(CPU_LEVEL_ROOT, CPU_WHICH_TID, -1, 253 &info.jail_cpuset); 254 if (error != 0) 255 return (FAILURE_JAILSET); 256 error = cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_TID, -1, 257 &info.jail_child_cpuset); 258 if (error != 0) 259 return (FAILURE_PIDSET); 260 if (send(sock, &info, sizeof(info), 0) != sizeof(info)) 261 return (FAILURE_SEND); 262 return (0); 263 } 264 265 static void 266 do_jail_test(int ncpu, bool newset, jail_test_cb prologue, 267 jail_test_cb epilogue) 268 { 269 struct jail_test_cb_params cbp; 270 const char *errstr; 271 pid_t pid; 272 int error, sock, sockpair[2], status; 273 274 memset(&cbp.info, '\0', sizeof(cbp.info)); 275 276 skip_ltncpu(ncpu, &cbp.mask); 277 278 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_ROOT, CPU_WHICH_PID, -1, 279 &cbp.rootid)); 280 if (newset) 281 ATF_REQUIRE_EQ(0, cpuset(&cbp.setid)); 282 else 283 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_PID, 284 -1, &cbp.setid)); 285 /* Special hack for prison0; it uses cpuset 1 as the root. */ 286 if (cbp.rootid == 0) 287 cbp.rootid = 1; 288 289 /* Not every test needs early setup. */ 290 if (prologue != NULL) 291 (*prologue)(&cbp); 292 293 ATF_REQUIRE_EQ(0, socketpair(PF_UNIX, SOCK_STREAM, 0, sockpair)); 294 ATF_REQUIRE((pid = fork()) != -1); 295 296 if (pid == 0) { 297 /* Child */ 298 close(sockpair[SP_PARENT]); 299 sock = sockpair[SP_CHILD]; 300 301 _exit(do_jail(sock)); 302 } else { 303 /* Parent */ 304 sock = sockpair[SP_PARENT]; 305 close(sockpair[SP_CHILD]); 306 307 while ((error = waitpid(pid, &status, 0)) == -1 && 308 errno == EINTR) { 309 } 310 311 ATF_REQUIRE_EQ(sizeof(cbp.info), recv(sock, &cbp.info, 312 sizeof(cbp.info), 0)); 313 314 /* Sanity check the exit info. */ 315 ATF_REQUIRE_EQ(pid, error); 316 ATF_REQUIRE(WIFEXITED(status)); 317 if (WEXITSTATUS(status) != 0) { 318 errstr = do_jail_errstr(WEXITSTATUS(status)); 319 if (errstr != NULL) 320 atf_tc_fail("%s", errstr); 321 else 322 atf_tc_fail("Unknown error '%d'", 323 WEXITSTATUS(status)); 324 } 325 326 epilogue(&cbp); 327 } 328 } 329 330 static void 331 jail_attach_mutate_pro(struct jail_test_cb_params *cbp) 332 { 333 cpuset_t *mask; 334 int count; 335 336 mask = &cbp->mask; 337 338 /* Knock out the first cpu. */ 339 count = CPU_COUNT(mask); 340 CPU_CLR(CPU_FFS(mask) - 1, mask); 341 ATF_REQUIRE_EQ(count - 1, CPU_COUNT(mask)); 342 ATF_REQUIRE_EQ(0, cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 343 -1, sizeof(*mask), mask)); 344 } 345 346 static void 347 jail_attach_newbase_epi(struct jail_test_cb_params *cbp) 348 { 349 struct jail_test_info *info; 350 cpuset_t *mask; 351 352 info = &cbp->info; 353 mask = &cbp->mask; 354 355 /* 356 * The rootid test has been thrown in because a bug was discovered 357 * where any newly derived cpuset during attach would be parented to 358 * the wrong cpuset. Otherwise, we should observe that a new cpuset 359 * has been created for this process. 360 */ 361 ATF_REQUIRE(info->jail_cpuset != cbp->rootid); 362 ATF_REQUIRE(info->jail_cpuset != cbp->setid); 363 ATF_REQUIRE(info->jail_cpuset != info->jail_child_cpuset); 364 ATF_REQUIRE_EQ(0, CPU_CMP(mask, &info->jail_tidmask)); 365 } 366 367 ATF_TC(jail_attach_newbase); 368 ATF_TC_HEAD(jail_attach_newbase, tc) 369 { 370 atf_tc_set_md_var(tc, "descr", 371 "Test jail attachment effect on affinity with a new base cpuset."); 372 atf_tc_set_md_var(tc, "require.user", "root"); 373 } 374 ATF_TC_BODY(jail_attach_newbase, tc) 375 { 376 377 /* Need >= 2 cpus to test restriction. */ 378 do_jail_test(2, true, &jail_attach_mutate_pro, 379 &jail_attach_newbase_epi); 380 } 381 382 ATF_TC(jail_attach_newbase_plain); 383 ATF_TC_HEAD(jail_attach_newbase_plain, tc) 384 { 385 atf_tc_set_md_var(tc, "descr", 386 "Test jail attachment effect on affinity with a new, unmodified base cpuset."); 387 atf_tc_set_md_var(tc, "require.user", "root"); 388 } 389 ATF_TC_BODY(jail_attach_newbase_plain, tc) 390 { 391 392 do_jail_test(2, true, NULL, &jail_attach_newbase_epi); 393 } 394 395 /* 396 * Generic epilogue for tests that are expecting to use the jail's root cpuset 397 * with their own mask, whether that's been modified or not. 398 */ 399 static void 400 jail_attach_jset_epi(struct jail_test_cb_params *cbp) 401 { 402 struct jail_test_info *info; 403 cpuset_t *mask; 404 405 info = &cbp->info; 406 mask = &cbp->mask; 407 408 ATF_REQUIRE(info->jail_cpuset != cbp->setid); 409 ATF_REQUIRE_EQ(info->jail_cpuset, info->jail_child_cpuset); 410 ATF_REQUIRE_EQ(0, CPU_CMP(mask, &info->jail_tidmask)); 411 } 412 413 ATF_TC(jail_attach_prevbase); 414 ATF_TC_HEAD(jail_attach_prevbase, tc) 415 { 416 atf_tc_set_md_var(tc, "descr", 417 "Test jail attachment effect on affinity without a new base."); 418 atf_tc_set_md_var(tc, "require.user", "root"); 419 } 420 ATF_TC_BODY(jail_attach_prevbase, tc) 421 { 422 423 do_jail_test(2, false, &jail_attach_mutate_pro, &jail_attach_jset_epi); 424 } 425 426 static void 427 jail_attach_plain_pro(struct jail_test_cb_params *cbp) 428 { 429 430 if (cbp->setid != cbp->rootid) 431 atf_tc_skip("Must be running with the root cpuset."); 432 } 433 434 ATF_TC(jail_attach_plain); 435 ATF_TC_HEAD(jail_attach_plain, tc) 436 { 437 atf_tc_set_md_var(tc, "descr", 438 "Test jail attachment effect on affinity without specialization."); 439 atf_tc_set_md_var(tc, "require.user", "root"); 440 } 441 ATF_TC_BODY(jail_attach_plain, tc) 442 { 443 444 do_jail_test(1, false, &jail_attach_plain_pro, &jail_attach_jset_epi); 445 } 446 447 ATF_TC(badparent); 448 ATF_TC_HEAD(badparent, tc) 449 { 450 atf_tc_set_md_var(tc, "descr", 451 "Test parent assignment when assigning a new cpuset."); 452 } 453 ATF_TC_BODY(badparent, tc) 454 { 455 cpuset_t mask; 456 cpusetid_t finalsetid, origsetid, setid; 457 458 /* Need to mask off at least one CPU. */ 459 skip_ltncpu(2, &mask); 460 461 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_TID, -1, 462 &origsetid)); 463 464 ATF_REQUIRE_EQ(0, cpuset(&setid)); 465 466 /* 467 * Mask off the first CPU, then we'll reparent ourselves to our original 468 * set. 469 */ 470 CPU_CLR(CPU_FFS(&mask) - 1, &mask); 471 ATF_REQUIRE_EQ(0, cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, 472 -1, sizeof(mask), &mask)); 473 474 ATF_REQUIRE_EQ(0, cpuset_setid(CPU_WHICH_PID, -1, origsetid)); 475 ATF_REQUIRE_EQ(0, cpuset_getid(CPU_LEVEL_CPUSET, CPU_WHICH_TID, -1, 476 &finalsetid)); 477 478 ATF_REQUIRE_EQ(finalsetid, origsetid); 479 } 480 481 ATF_TP_ADD_TCS(tp) 482 { 483 484 ATF_TP_ADD_TC(tp, newset); 485 ATF_TP_ADD_TC(tp, transient); 486 ATF_TP_ADD_TC(tp, deadlk); 487 ATF_TP_ADD_TC(tp, jail_attach_newbase); 488 ATF_TP_ADD_TC(tp, jail_attach_newbase_plain); 489 ATF_TP_ADD_TC(tp, jail_attach_prevbase); 490 ATF_TP_ADD_TC(tp, jail_attach_plain); 491 ATF_TP_ADD_TC(tp, badparent); 492 return (atf_no_error()); 493 } 494