1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Freezing a block device that several superblocks share. 4 * 5 * erofs claims its blob devices with the filesystem type as the holder, so 6 * several erofs superblocks can share a blob. Merge two images from a common 7 * layer and a layer of their own, put the common layer on a device-mapper 8 * device and mount both. A suspend of that device has to freeze each of the 9 * two superblocks exactly once and the resume has to thaw both, also while 10 * mounts of one of them come and go. erofs is read-only, so the freezes and 11 * thaws don't write to the suspended device. 12 */ 13 #define _GNU_SOURCE 14 #include <errno.h> 15 #include <fcntl.h> 16 #include <ftw.h> 17 #include <linux/dm-ioctl.h> 18 #include <linux/fs.h> 19 #include <sched.h> 20 #include <stdbool.h> 21 #include <stdint.h> 22 #include <stdio.h> 23 #include <stdlib.h> 24 #include <string.h> 25 #include <sys/ioctl.h> 26 #include <sys/mount.h> 27 #include <sys/stat.h> 28 #include <sys/syscall.h> 29 #include <sys/sysmacros.h> 30 #include <sys/wait.h> 31 #include <unistd.h> 32 33 #include "../../kselftest_harness.h" 34 #include "loop_util.h" 35 36 #define NR_CHURN 100 37 38 enum { M12, M13, L1, L2, L3, NR_IMGS }; 39 40 static const char *const img_names[NR_IMGS] = { "m12", "m13", "l1", "l2", "l3" }; 41 42 FIXTURE(shared_bdev) { 43 char dir[64]; 44 char a[96]; 45 char b[96]; 46 char dm[96]; 47 char name[32]; 48 char src_a[32]; 49 char src_b[32]; 50 char opts_a[256]; 51 char opts_b[256]; 52 int loop[NR_IMGS]; 53 int fd_a; 54 int fd_b; 55 dev_t dmdev; 56 bool dm_created; 57 bool suspended; 58 bool mounted_a; 59 bool mounted_b; 60 }; 61 62 static int dm_ioctl(unsigned long cmd, const char *name, uint32_t flags, dev_t *dev) 63 { 64 struct dm_ioctl dmi = { 65 .version = { DM_VERSION_MAJOR, 0, 0 }, 66 .data_size = sizeof(dmi), 67 .data_start = sizeof(dmi), 68 .flags = flags, 69 }; 70 int fd, ret = 0; 71 72 strncpy(dmi.name, name, sizeof(dmi.name) - 1); 73 fd = open("/dev/mapper/control", O_RDWR | O_CLOEXEC); 74 if (fd < 0) 75 return -errno; 76 if (ioctl(fd, cmd, &dmi)) 77 ret = -errno; 78 else if (dev) 79 *dev = makedev((dmi.dev >> 8) & 0xfff, 80 (dmi.dev & 0xff) | ((dmi.dev >> 12) & 0xfff00)); 81 close(fd); 82 return ret; 83 } 84 85 /* Create @name as a linear mapping of all of @backing and make it live. */ 86 static int dm_create_linear(const char *name, dev_t backing, uint64_t sectors, 87 dev_t *dev) 88 { 89 struct { 90 struct dm_ioctl dmi; 91 struct dm_target_spec spec; 92 char params[64]; 93 } __attribute__((aligned(8))) t = {}; 94 int fd, ret; 95 96 ret = dm_ioctl(DM_DEV_CREATE, name, 0, NULL); 97 if (ret) 98 return ret; 99 100 t.dmi.version[0] = DM_VERSION_MAJOR; 101 t.dmi.data_size = sizeof(t); 102 t.dmi.data_start = sizeof(t.dmi); 103 t.dmi.target_count = 1; 104 strncpy(t.dmi.name, name, sizeof(t.dmi.name) - 1); 105 t.spec.length = sectors; 106 strcpy(t.spec.target_type, "linear"); 107 snprintf(t.params, sizeof(t.params), "%u:%u 0", major(backing), minor(backing)); 108 109 fd = open("/dev/mapper/control", O_RDWR | O_CLOEXEC); 110 if (fd >= 0) { 111 ret = ioctl(fd, DM_TABLE_LOAD, &t) ? -errno : 0; 112 close(fd); 113 } else { 114 ret = -errno; 115 } 116 /* Resuming makes the loaded table live. */ 117 if (!ret) 118 ret = dm_ioctl(DM_DEV_SUSPEND, name, 0, dev); 119 if (ret) 120 dm_ioctl(DM_DEV_REMOVE, name, 0, NULL); 121 return ret; 122 } 123 124 static int suspend(FIXTURE_DATA(shared_bdev) *self) 125 { 126 int ret = dm_ioctl(DM_DEV_SUSPEND, self->name, DM_SUSPEND_FLAG, NULL); 127 128 if (!ret) 129 self->suspended = true; 130 return ret; 131 } 132 133 static int resume(FIXTURE_DATA(shared_bdev) *self) 134 { 135 int ret = dm_ioctl(DM_DEV_SUSPEND, self->name, 0, NULL); 136 137 if (!ret) 138 self->suspended = false; 139 return ret; 140 } 141 142 /* Drop one freeze with FITHAW; EINVAL if the superblock isn't frozen. */ 143 static bool fithaw(int fd) 144 { 145 return !ioctl(fd, FITHAW, 0); 146 } 147 148 static int write_file(const char *path, const char *val) 149 { 150 ssize_t len = strlen(val); 151 int fd, ret = 0; 152 153 fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); 154 if (fd < 0) 155 return -1; 156 if (write(fd, val, len) != len) 157 ret = -1; 158 close(fd); 159 return ret; 160 } 161 162 static int check_file(const char *path, const char *val) 163 { 164 char buf[32] = {}; 165 int fd; 166 167 fd = open(path, O_RDONLY | O_CLOEXEC); 168 if (fd < 0) 169 return -1; 170 if (read(fd, buf, sizeof(buf) - 1) < 0) 171 buf[0] = '\0'; 172 close(fd); 173 return strcmp(buf, val) ? -1 : 0; 174 } 175 176 static int rm_one(const char *path, const struct stat *st, int type, struct FTW *ftw) 177 { 178 return remove(path); 179 } 180 181 static int mount_b(FIXTURE_DATA(shared_bdev) *self) 182 { 183 int ret = mount(self->src_b, self->b, "erofs", MS_RDONLY, self->opts_b); 184 185 if (!ret) 186 self->mounted_b = true; 187 return ret; 188 } 189 190 static void shared_bdev_cleanup(FIXTURE_DATA(shared_bdev) *self) 191 { 192 if (self->suspended) 193 resume(self); 194 if (self->fd_a >= 0) 195 close(self->fd_a); 196 if (self->fd_b >= 0) 197 close(self->fd_b); 198 if (self->mounted_a) 199 umount2(self->a, MNT_DETACH); 200 umount2(self->b, MNT_DETACH); 201 if (self->dm_created) 202 dm_ioctl(DM_DEV_REMOVE, self->name, 0, NULL); 203 for (int i = 0; i < NR_IMGS; i++) { 204 if (self->loop[i] >= 0) 205 loop_clear(self->loop[i]); 206 } 207 nftw(self->dir, rm_one, 16, FTW_DEPTH | FTW_PHYS); 208 } 209 210 /* Two layers of their own, a common one, and two images merged from them. */ 211 static int make_images(const char *dir) 212 { 213 static const char *const files[][2] = { 214 { "src-l1/d1/f1", "one\n" }, 215 { "src-l2/f2", "two\n" }, 216 { "src-l3/f3", "three\n" }, 217 }; 218 char path[160], cmd[512]; 219 220 for (int i = 0; i < 3; i++) { 221 snprintf(path, sizeof(path), "%s/src-l%d", dir, i + 1); 222 if (mkdir(path, 0755)) 223 return -1; 224 } 225 snprintf(path, sizeof(path), "%s/src-l1/d1", dir); 226 if (mkdir(path, 0755)) 227 return -1; 228 for (int i = 0; i < 3; i++) { 229 snprintf(path, sizeof(path), "%s/%s", dir, files[i][0]); 230 if (write_file(path, files[i][1])) 231 return -1; 232 } 233 234 /* Chunk-based layers can be referenced by the merged images. */ 235 snprintf(cmd, sizeof(cmd), 236 "cd %s && (for l in l1 l2 l3; do " 237 "mkfs.erofs --chunksize=4096 -Enoinline_data $l src-$l || exit 1; done && " 238 "mkfs.erofs m12 l1 l2 && mkfs.erofs m13 l1 l3) >/dev/null 2>&1", dir); 239 return system(cmd) ? -1 : 0; 240 } 241 242 FIXTURE_SETUP(shared_bdev) 243 { 244 struct stat img, lst, st; 245 char path[160], dev[32]; 246 int fd; 247 248 for (int i = 0; i < NR_IMGS; i++) 249 self->loop[i] = -1; 250 self->fd_a = -1; 251 self->fd_b = -1; 252 self->dm_created = false; 253 self->suspended = false; 254 self->mounted_a = false; 255 self->mounted_b = false; 256 257 if (geteuid() != 0) 258 SKIP(return, "test requires CAP_SYS_ADMIN"); 259 if (access("/dev/loop-control", R_OK | W_OK)) 260 SKIP(return, "test requires loop devices"); 261 if (access("/dev/mapper/control", R_OK | W_OK)) 262 SKIP(return, "test requires device-mapper"); 263 264 ASSERT_EQ(unshare(CLONE_NEWNS), 0); 265 ASSERT_EQ(mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL), 0); 266 267 snprintf(self->dir, sizeof(self->dir), "/tmp/shared_bdev.XXXXXX"); 268 ASSERT_NE(mkdtemp(self->dir), NULL); 269 snprintf(self->a, sizeof(self->a), "%s/a", self->dir); 270 snprintf(self->b, sizeof(self->b), "%s/b", self->dir); 271 snprintf(self->name, sizeof(self->name), "shared_bdev_%d", getpid()); 272 /* The harness doesn't run the teardown when the setup fails. */ 273 ASSERT_EQ(mkdir(self->a, 0700), 0) 274 shared_bdev_cleanup(self); 275 ASSERT_EQ(mkdir(self->b, 0700), 0) 276 shared_bdev_cleanup(self); 277 278 if (make_images(self->dir)) { 279 shared_bdev_cleanup(self); 280 SKIP(return, "mkfs.erofs failed, is missing or can't merge images"); 281 } 282 283 for (int i = 0; i < NR_IMGS; i++) { 284 snprintf(path, sizeof(path), "%s/%s", self->dir, img_names[i]); 285 fd = open(path, O_RDWR | O_CLOEXEC); 286 ASSERT_GE(fd, 0) 287 shared_bdev_cleanup(self); 288 self->loop[i] = loop_bind(fd); 289 if (i == L1) { 290 ASSERT_EQ(fstat(fd, &img), 0) 291 shared_bdev_cleanup(self); 292 } 293 close(fd); 294 ASSERT_GE(self->loop[i], 0) 295 shared_bdev_cleanup(self); 296 } 297 298 /* The common layer goes on a device-mapper device. */ 299 snprintf(dev, sizeof(dev), "/dev/loop%d", self->loop[L1]); 300 ASSERT_EQ(stat(dev, &lst), 0) 301 shared_bdev_cleanup(self); 302 ASSERT_EQ(dm_create_linear(self->name, lst.st_rdev, img.st_size >> 9, &self->dmdev), 0) 303 shared_bdev_cleanup(self); 304 self->dm_created = true; 305 snprintf(self->dm, sizeof(self->dm), "/dev/dm-%u", minor(self->dmdev)); 306 if (stat(self->dm, &st) || st.st_rdev != self->dmdev) { 307 snprintf(self->dm, sizeof(self->dm), "%s/dm", self->dir); 308 ASSERT_EQ(mknod(self->dm, S_IFBLK | 0600, self->dmdev), 0) 309 shared_bdev_cleanup(self); 310 } 311 312 snprintf(self->src_a, sizeof(self->src_a), "/dev/loop%d", self->loop[M12]); 313 snprintf(self->src_b, sizeof(self->src_b), "/dev/loop%d", self->loop[M13]); 314 snprintf(self->opts_a, sizeof(self->opts_a), "device=%s,device=/dev/loop%d", 315 self->dm, self->loop[L2]); 316 snprintf(self->opts_b, sizeof(self->opts_b), "device=%s,device=/dev/loop%d", 317 self->dm, self->loop[L3]); 318 319 if (mount(self->src_a, self->a, "erofs", MS_RDONLY, self->opts_a)) { 320 if (errno == ENODEV) { 321 shared_bdev_cleanup(self); 322 SKIP(return, "kernel lacks erofs"); 323 } 324 ASSERT_EQ(errno, 0) 325 shared_bdev_cleanup(self); 326 } 327 self->mounted_a = true; 328 ASSERT_EQ(mount_b(self), 0) 329 shared_bdev_cleanup(self); 330 331 /* Both read through the shared blob. */ 332 snprintf(path, sizeof(path), "%s/d1/f1", self->a); 333 ASSERT_EQ(check_file(path, "one\n"), 0) 334 shared_bdev_cleanup(self); 335 snprintf(path, sizeof(path), "%s/f2", self->a); 336 ASSERT_EQ(check_file(path, "two\n"), 0) 337 shared_bdev_cleanup(self); 338 snprintf(path, sizeof(path), "%s/d1/f1", self->b); 339 ASSERT_EQ(check_file(path, "one\n"), 0) 340 shared_bdev_cleanup(self); 341 snprintf(path, sizeof(path), "%s/f3", self->b); 342 ASSERT_EQ(check_file(path, "three\n"), 0) 343 shared_bdev_cleanup(self); 344 345 self->fd_a = open(self->a, O_RDONLY | O_DIRECTORY | O_CLOEXEC); 346 ASSERT_GE(self->fd_a, 0) 347 shared_bdev_cleanup(self); 348 self->fd_b = open(self->b, O_RDONLY | O_DIRECTORY | O_CLOEXEC); 349 ASSERT_GE(self->fd_b, 0) 350 shared_bdev_cleanup(self); 351 } 352 353 FIXTURE_TEARDOWN(shared_bdev) 354 { 355 shared_bdev_cleanup(self); 356 } 357 358 TEST_F(shared_bdev, suspend_freezes_each_once) 359 { 360 ASSERT_EQ(suspend(self), 0); 361 362 /* One FITHAW drops each freeze, a second one finds nothing frozen. */ 363 ASSERT_TRUE(fithaw(self->fd_a)); 364 ASSERT_FALSE(fithaw(self->fd_a)); 365 ASSERT_EQ(errno, EINVAL); 366 ASSERT_TRUE(fithaw(self->fd_b)); 367 ASSERT_FALSE(fithaw(self->fd_b)); 368 ASSERT_EQ(errno, EINVAL); 369 370 /* The resume finds nothing left to thaw and still thaws the device. */ 371 ASSERT_EQ(resume(self), 0); 372 ASSERT_EQ(suspend(self), 0); 373 ASSERT_EQ(resume(self), 0); 374 } 375 376 TEST_F(shared_bdev, resume_thaws_all) 377 { 378 ASSERT_EQ(suspend(self), 0); 379 ASSERT_EQ(resume(self), 0); 380 381 ASSERT_FALSE(fithaw(self->fd_a)); 382 ASSERT_EQ(errno, EINVAL); 383 ASSERT_FALSE(fithaw(self->fd_b)); 384 ASSERT_EQ(errno, EINVAL); 385 } 386 387 TEST_F(shared_bdev, unmounted_sharer) 388 { 389 close(self->fd_b); 390 self->fd_b = -1; 391 ASSERT_EQ(umount2(self->b, 0), 0); 392 self->mounted_b = false; 393 394 /* Only the remaining superblock is frozen, once. */ 395 ASSERT_EQ(suspend(self), 0); 396 ASSERT_TRUE(fithaw(self->fd_a)); 397 ASSERT_FALSE(fithaw(self->fd_a)); 398 ASSERT_EQ(errno, EINVAL); 399 ASSERT_EQ(resume(self), 0); 400 401 /* The blob can be claimed again. */ 402 ASSERT_EQ(mount_b(self), 0); 403 } 404 405 TEST_F(shared_bdev, flushbuf) 406 { 407 int fd = open(self->dm, O_RDONLY | O_CLOEXEC); 408 409 ASSERT_GE(fd, 0); 410 /* BLKFLSBUF syncs every superblock that uses the device. */ 411 ASSERT_EQ(ioctl(fd, BLKFLSBUF, 0), 0); 412 close(fd); 413 } 414 415 #ifdef __NR_ustat 416 /* struct ustat is not exported through UAPI, mirror include/linux/types.h. */ 417 struct ustat_buf { 418 int f_tfree; 419 unsigned long f_tinode; 420 char f_fname[6]; 421 char f_fpack[6]; 422 char pad[64]; 423 }; 424 425 TEST_F(shared_bdev, ustat) 426 { 427 struct ustat_buf buf; 428 429 /* ustat() stops at the first superblock it can lock... */ 430 ASSERT_EQ(syscall(__NR_ustat, (unsigned int)self->dmdev, &buf), 0); 431 432 /* ...and the walks after it still reach both. */ 433 ASSERT_EQ(suspend(self), 0); 434 ASSERT_TRUE(fithaw(self->fd_a)); 435 ASSERT_TRUE(fithaw(self->fd_b)); 436 ASSERT_EQ(resume(self), 0); 437 } 438 #endif 439 440 TEST_F(shared_bdev, churn) 441 { 442 int status; 443 pid_t pid; 444 445 close(self->fd_b); 446 self->fd_b = -1; 447 ASSERT_EQ(umount2(self->b, 0), 0); 448 self->mounted_b = false; 449 450 pid = fork(); 451 ASSERT_GE(pid, 0); 452 if (pid == 0) { 453 /* A mount while the device is frozen is refused with EBUSY. */ 454 for (int i = 0; i < NR_CHURN; i++) { 455 if (!mount(self->src_b, self->b, "erofs", MS_RDONLY, self->opts_b)) { 456 if (umount2(self->b, 0)) 457 _exit(1); 458 } else if (errno != EBUSY) { 459 _exit(2); 460 } 461 } 462 _exit(0); 463 } 464 465 for (int i = 0; i < NR_CHURN; i++) { 466 ASSERT_EQ(suspend(self), 0); 467 ASSERT_EQ(resume(self), 0); 468 } 469 ASSERT_EQ(waitpid(pid, &status, 0), pid); 470 ASSERT_TRUE(WIFEXITED(status)); 471 ASSERT_EQ(WEXITSTATUS(status), 0); 472 473 /* Nothing is left frozen and the blob can be claimed again. */ 474 ASSERT_FALSE(fithaw(self->fd_a)); 475 ASSERT_EQ(errno, EINVAL); 476 ASSERT_EQ(mount_b(self), 0); 477 } 478 479 TEST_HARNESS_MAIN 480