xref: /linux/tools/testing/selftests/filesystems/freeze/shared_bdev_test.c (revision 073d71c5118f40449d9c7df828677aaa437372e1)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Freezing a block device that several superblocks share.
4  *
5  * erofs claims its blob devices with the filesystem type as the holder, so
6  * several erofs superblocks can share a blob. Merge two images from a common
7  * layer and a layer of their own, put the common layer on a device-mapper
8  * device and mount both. A suspend of that device has to freeze each of the
9  * two superblocks exactly once and the resume has to thaw both, also while
10  * mounts of one of them come and go. erofs is read-only, so the freezes and
11  * thaws don't write to the suspended device.
12  */
13 #define _GNU_SOURCE
14 #include <errno.h>
15 #include <fcntl.h>
16 #include <ftw.h>
17 #include <linux/dm-ioctl.h>
18 #include <linux/fs.h>
19 #include <sched.h>
20 #include <stdbool.h>
21 #include <stdint.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 #include <sys/ioctl.h>
26 #include <sys/mount.h>
27 #include <sys/stat.h>
28 #include <sys/syscall.h>
29 #include <sys/sysmacros.h>
30 #include <sys/wait.h>
31 #include <unistd.h>
32 
33 #include "../../kselftest_harness.h"
34 #include "loop_util.h"
35 
36 #define NR_CHURN 100
37 
38 enum { M12, M13, L1, L2, L3, NR_IMGS };
39 
40 static const char *const img_names[NR_IMGS] = { "m12", "m13", "l1", "l2", "l3" };
41 
42 FIXTURE(shared_bdev) {
43 	char dir[64];
44 	char a[96];
45 	char b[96];
46 	char dm[96];
47 	char name[32];
48 	char src_a[32];
49 	char src_b[32];
50 	char opts_a[256];
51 	char opts_b[256];
52 	int loop[NR_IMGS];
53 	int fd_a;
54 	int fd_b;
55 	dev_t dmdev;
56 	bool dm_created;
57 	bool suspended;
58 	bool mounted_a;
59 	bool mounted_b;
60 };
61 
62 static int dm_ioctl(unsigned long cmd, const char *name, uint32_t flags, dev_t *dev)
63 {
64 	struct dm_ioctl dmi = {
65 		.version = { DM_VERSION_MAJOR, 0, 0 },
66 		.data_size = sizeof(dmi),
67 		.data_start = sizeof(dmi),
68 		.flags = flags,
69 	};
70 	int fd, ret = 0;
71 
72 	strncpy(dmi.name, name, sizeof(dmi.name) - 1);
73 	fd = open("/dev/mapper/control", O_RDWR | O_CLOEXEC);
74 	if (fd < 0)
75 		return -errno;
76 	if (ioctl(fd, cmd, &dmi))
77 		ret = -errno;
78 	else if (dev)
79 		*dev = makedev((dmi.dev >> 8) & 0xfff,
80 			       (dmi.dev & 0xff) | ((dmi.dev >> 12) & 0xfff00));
81 	close(fd);
82 	return ret;
83 }
84 
85 /* Create @name as a linear mapping of all of @backing and make it live. */
86 static int dm_create_linear(const char *name, dev_t backing, uint64_t sectors,
87 			    dev_t *dev)
88 {
89 	struct {
90 		struct dm_ioctl dmi;
91 		struct dm_target_spec spec;
92 		char params[64];
93 	} __attribute__((aligned(8))) t = {};
94 	int fd, ret;
95 
96 	ret = dm_ioctl(DM_DEV_CREATE, name, 0, NULL);
97 	if (ret)
98 		return ret;
99 
100 	t.dmi.version[0] = DM_VERSION_MAJOR;
101 	t.dmi.data_size = sizeof(t);
102 	t.dmi.data_start = sizeof(t.dmi);
103 	t.dmi.target_count = 1;
104 	strncpy(t.dmi.name, name, sizeof(t.dmi.name) - 1);
105 	t.spec.length = sectors;
106 	strcpy(t.spec.target_type, "linear");
107 	snprintf(t.params, sizeof(t.params), "%u:%u 0", major(backing), minor(backing));
108 
109 	fd = open("/dev/mapper/control", O_RDWR | O_CLOEXEC);
110 	if (fd >= 0) {
111 		ret = ioctl(fd, DM_TABLE_LOAD, &t) ? -errno : 0;
112 		close(fd);
113 	} else {
114 		ret = -errno;
115 	}
116 	/* Resuming makes the loaded table live. */
117 	if (!ret)
118 		ret = dm_ioctl(DM_DEV_SUSPEND, name, 0, dev);
119 	if (ret)
120 		dm_ioctl(DM_DEV_REMOVE, name, 0, NULL);
121 	return ret;
122 }
123 
124 static int suspend(FIXTURE_DATA(shared_bdev) *self)
125 {
126 	int ret = dm_ioctl(DM_DEV_SUSPEND, self->name, DM_SUSPEND_FLAG, NULL);
127 
128 	if (!ret)
129 		self->suspended = true;
130 	return ret;
131 }
132 
133 static int resume(FIXTURE_DATA(shared_bdev) *self)
134 {
135 	int ret = dm_ioctl(DM_DEV_SUSPEND, self->name, 0, NULL);
136 
137 	if (!ret)
138 		self->suspended = false;
139 	return ret;
140 }
141 
142 /* Drop one freeze with FITHAW; EINVAL if the superblock isn't frozen. */
143 static bool fithaw(int fd)
144 {
145 	return !ioctl(fd, FITHAW, 0);
146 }
147 
148 static int write_file(const char *path, const char *val)
149 {
150 	ssize_t len = strlen(val);
151 	int fd, ret = 0;
152 
153 	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
154 	if (fd < 0)
155 		return -1;
156 	if (write(fd, val, len) != len)
157 		ret = -1;
158 	close(fd);
159 	return ret;
160 }
161 
162 static int check_file(const char *path, const char *val)
163 {
164 	char buf[32] = {};
165 	int fd;
166 
167 	fd = open(path, O_RDONLY | O_CLOEXEC);
168 	if (fd < 0)
169 		return -1;
170 	if (read(fd, buf, sizeof(buf) - 1) < 0)
171 		buf[0] = '\0';
172 	close(fd);
173 	return strcmp(buf, val) ? -1 : 0;
174 }
175 
176 static int rm_one(const char *path, const struct stat *st, int type, struct FTW *ftw)
177 {
178 	return remove(path);
179 }
180 
181 static int mount_b(FIXTURE_DATA(shared_bdev) *self)
182 {
183 	int ret = mount(self->src_b, self->b, "erofs", MS_RDONLY, self->opts_b);
184 
185 	if (!ret)
186 		self->mounted_b = true;
187 	return ret;
188 }
189 
190 static void shared_bdev_cleanup(FIXTURE_DATA(shared_bdev) *self)
191 {
192 	if (self->suspended)
193 		resume(self);
194 	if (self->fd_a >= 0)
195 		close(self->fd_a);
196 	if (self->fd_b >= 0)
197 		close(self->fd_b);
198 	if (self->mounted_a)
199 		umount2(self->a, MNT_DETACH);
200 	umount2(self->b, MNT_DETACH);
201 	if (self->dm_created)
202 		dm_ioctl(DM_DEV_REMOVE, self->name, 0, NULL);
203 	for (int i = 0; i < NR_IMGS; i++) {
204 		if (self->loop[i] >= 0)
205 			loop_clear(self->loop[i]);
206 	}
207 	nftw(self->dir, rm_one, 16, FTW_DEPTH | FTW_PHYS);
208 }
209 
210 /* Two layers of their own, a common one, and two images merged from them. */
211 static int make_images(const char *dir)
212 {
213 	static const char *const files[][2] = {
214 		{ "src-l1/d1/f1", "one\n" },
215 		{ "src-l2/f2", "two\n" },
216 		{ "src-l3/f3", "three\n" },
217 	};
218 	char path[160], cmd[512];
219 
220 	for (int i = 0; i < 3; i++) {
221 		snprintf(path, sizeof(path), "%s/src-l%d", dir, i + 1);
222 		if (mkdir(path, 0755))
223 			return -1;
224 	}
225 	snprintf(path, sizeof(path), "%s/src-l1/d1", dir);
226 	if (mkdir(path, 0755))
227 		return -1;
228 	for (int i = 0; i < 3; i++) {
229 		snprintf(path, sizeof(path), "%s/%s", dir, files[i][0]);
230 		if (write_file(path, files[i][1]))
231 			return -1;
232 	}
233 
234 	/* Chunk-based layers can be referenced by the merged images. */
235 	snprintf(cmd, sizeof(cmd),
236 		 "cd %s && (for l in l1 l2 l3; do "
237 		 "mkfs.erofs --chunksize=4096 -Enoinline_data $l src-$l || exit 1; done && "
238 		 "mkfs.erofs m12 l1 l2 && mkfs.erofs m13 l1 l3) >/dev/null 2>&1", dir);
239 	return system(cmd) ? -1 : 0;
240 }
241 
242 FIXTURE_SETUP(shared_bdev)
243 {
244 	struct stat img, lst, st;
245 	char path[160], dev[32];
246 	int fd;
247 
248 	for (int i = 0; i < NR_IMGS; i++)
249 		self->loop[i] = -1;
250 	self->fd_a = -1;
251 	self->fd_b = -1;
252 	self->dm_created = false;
253 	self->suspended = false;
254 	self->mounted_a = false;
255 	self->mounted_b = false;
256 
257 	if (geteuid() != 0)
258 		SKIP(return, "test requires CAP_SYS_ADMIN");
259 	if (access("/dev/loop-control", R_OK | W_OK))
260 		SKIP(return, "test requires loop devices");
261 	if (access("/dev/mapper/control", R_OK | W_OK))
262 		SKIP(return, "test requires device-mapper");
263 
264 	ASSERT_EQ(unshare(CLONE_NEWNS), 0);
265 	ASSERT_EQ(mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL), 0);
266 
267 	snprintf(self->dir, sizeof(self->dir), "/tmp/shared_bdev.XXXXXX");
268 	ASSERT_NE(mkdtemp(self->dir), NULL);
269 	snprintf(self->a, sizeof(self->a), "%s/a", self->dir);
270 	snprintf(self->b, sizeof(self->b), "%s/b", self->dir);
271 	snprintf(self->name, sizeof(self->name), "shared_bdev_%d", getpid());
272 	/* The harness doesn't run the teardown when the setup fails. */
273 	ASSERT_EQ(mkdir(self->a, 0700), 0)
274 		shared_bdev_cleanup(self);
275 	ASSERT_EQ(mkdir(self->b, 0700), 0)
276 		shared_bdev_cleanup(self);
277 
278 	if (make_images(self->dir)) {
279 		shared_bdev_cleanup(self);
280 		SKIP(return, "mkfs.erofs failed, is missing or can't merge images");
281 	}
282 
283 	for (int i = 0; i < NR_IMGS; i++) {
284 		snprintf(path, sizeof(path), "%s/%s", self->dir, img_names[i]);
285 		fd = open(path, O_RDWR | O_CLOEXEC);
286 		ASSERT_GE(fd, 0)
287 			shared_bdev_cleanup(self);
288 		self->loop[i] = loop_bind(fd);
289 		if (i == L1) {
290 			ASSERT_EQ(fstat(fd, &img), 0)
291 				shared_bdev_cleanup(self);
292 		}
293 		close(fd);
294 		ASSERT_GE(self->loop[i], 0)
295 			shared_bdev_cleanup(self);
296 	}
297 
298 	/* The common layer goes on a device-mapper device. */
299 	snprintf(dev, sizeof(dev), "/dev/loop%d", self->loop[L1]);
300 	ASSERT_EQ(stat(dev, &lst), 0)
301 		shared_bdev_cleanup(self);
302 	ASSERT_EQ(dm_create_linear(self->name, lst.st_rdev, img.st_size >> 9, &self->dmdev), 0)
303 		shared_bdev_cleanup(self);
304 	self->dm_created = true;
305 	snprintf(self->dm, sizeof(self->dm), "/dev/dm-%u", minor(self->dmdev));
306 	if (stat(self->dm, &st) || st.st_rdev != self->dmdev) {
307 		snprintf(self->dm, sizeof(self->dm), "%s/dm", self->dir);
308 		ASSERT_EQ(mknod(self->dm, S_IFBLK | 0600, self->dmdev), 0)
309 			shared_bdev_cleanup(self);
310 	}
311 
312 	snprintf(self->src_a, sizeof(self->src_a), "/dev/loop%d", self->loop[M12]);
313 	snprintf(self->src_b, sizeof(self->src_b), "/dev/loop%d", self->loop[M13]);
314 	snprintf(self->opts_a, sizeof(self->opts_a), "device=%s,device=/dev/loop%d",
315 		 self->dm, self->loop[L2]);
316 	snprintf(self->opts_b, sizeof(self->opts_b), "device=%s,device=/dev/loop%d",
317 		 self->dm, self->loop[L3]);
318 
319 	if (mount(self->src_a, self->a, "erofs", MS_RDONLY, self->opts_a)) {
320 		if (errno == ENODEV) {
321 			shared_bdev_cleanup(self);
322 			SKIP(return, "kernel lacks erofs");
323 		}
324 		ASSERT_EQ(errno, 0)
325 			shared_bdev_cleanup(self);
326 	}
327 	self->mounted_a = true;
328 	ASSERT_EQ(mount_b(self), 0)
329 		shared_bdev_cleanup(self);
330 
331 	/* Both read through the shared blob. */
332 	snprintf(path, sizeof(path), "%s/d1/f1", self->a);
333 	ASSERT_EQ(check_file(path, "one\n"), 0)
334 		shared_bdev_cleanup(self);
335 	snprintf(path, sizeof(path), "%s/f2", self->a);
336 	ASSERT_EQ(check_file(path, "two\n"), 0)
337 		shared_bdev_cleanup(self);
338 	snprintf(path, sizeof(path), "%s/d1/f1", self->b);
339 	ASSERT_EQ(check_file(path, "one\n"), 0)
340 		shared_bdev_cleanup(self);
341 	snprintf(path, sizeof(path), "%s/f3", self->b);
342 	ASSERT_EQ(check_file(path, "three\n"), 0)
343 		shared_bdev_cleanup(self);
344 
345 	self->fd_a = open(self->a, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
346 	ASSERT_GE(self->fd_a, 0)
347 		shared_bdev_cleanup(self);
348 	self->fd_b = open(self->b, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
349 	ASSERT_GE(self->fd_b, 0)
350 		shared_bdev_cleanup(self);
351 }
352 
353 FIXTURE_TEARDOWN(shared_bdev)
354 {
355 	shared_bdev_cleanup(self);
356 }
357 
358 TEST_F(shared_bdev, suspend_freezes_each_once)
359 {
360 	ASSERT_EQ(suspend(self), 0);
361 
362 	/* One FITHAW drops each freeze, a second one finds nothing frozen. */
363 	ASSERT_TRUE(fithaw(self->fd_a));
364 	ASSERT_FALSE(fithaw(self->fd_a));
365 	ASSERT_EQ(errno, EINVAL);
366 	ASSERT_TRUE(fithaw(self->fd_b));
367 	ASSERT_FALSE(fithaw(self->fd_b));
368 	ASSERT_EQ(errno, EINVAL);
369 
370 	/* The resume finds nothing left to thaw and still thaws the device. */
371 	ASSERT_EQ(resume(self), 0);
372 	ASSERT_EQ(suspend(self), 0);
373 	ASSERT_EQ(resume(self), 0);
374 }
375 
376 TEST_F(shared_bdev, resume_thaws_all)
377 {
378 	ASSERT_EQ(suspend(self), 0);
379 	ASSERT_EQ(resume(self), 0);
380 
381 	ASSERT_FALSE(fithaw(self->fd_a));
382 	ASSERT_EQ(errno, EINVAL);
383 	ASSERT_FALSE(fithaw(self->fd_b));
384 	ASSERT_EQ(errno, EINVAL);
385 }
386 
387 TEST_F(shared_bdev, unmounted_sharer)
388 {
389 	close(self->fd_b);
390 	self->fd_b = -1;
391 	ASSERT_EQ(umount2(self->b, 0), 0);
392 	self->mounted_b = false;
393 
394 	/* Only the remaining superblock is frozen, once. */
395 	ASSERT_EQ(suspend(self), 0);
396 	ASSERT_TRUE(fithaw(self->fd_a));
397 	ASSERT_FALSE(fithaw(self->fd_a));
398 	ASSERT_EQ(errno, EINVAL);
399 	ASSERT_EQ(resume(self), 0);
400 
401 	/* The blob can be claimed again. */
402 	ASSERT_EQ(mount_b(self), 0);
403 }
404 
405 TEST_F(shared_bdev, flushbuf)
406 {
407 	int fd = open(self->dm, O_RDONLY | O_CLOEXEC);
408 
409 	ASSERT_GE(fd, 0);
410 	/* BLKFLSBUF syncs every superblock that uses the device. */
411 	ASSERT_EQ(ioctl(fd, BLKFLSBUF, 0), 0);
412 	close(fd);
413 }
414 
415 #ifdef __NR_ustat
416 /* struct ustat is not exported through UAPI, mirror include/linux/types.h. */
417 struct ustat_buf {
418 	int		f_tfree;
419 	unsigned long	f_tinode;
420 	char		f_fname[6];
421 	char		f_fpack[6];
422 	char		pad[64];
423 };
424 
425 TEST_F(shared_bdev, ustat)
426 {
427 	struct ustat_buf buf;
428 
429 	/* ustat() stops at the first superblock it can lock... */
430 	ASSERT_EQ(syscall(__NR_ustat, (unsigned int)self->dmdev, &buf), 0);
431 
432 	/* ...and the walks after it still reach both. */
433 	ASSERT_EQ(suspend(self), 0);
434 	ASSERT_TRUE(fithaw(self->fd_a));
435 	ASSERT_TRUE(fithaw(self->fd_b));
436 	ASSERT_EQ(resume(self), 0);
437 }
438 #endif
439 
440 TEST_F(shared_bdev, churn)
441 {
442 	int status;
443 	pid_t pid;
444 
445 	close(self->fd_b);
446 	self->fd_b = -1;
447 	ASSERT_EQ(umount2(self->b, 0), 0);
448 	self->mounted_b = false;
449 
450 	pid = fork();
451 	ASSERT_GE(pid, 0);
452 	if (pid == 0) {
453 		/* A mount while the device is frozen is refused with EBUSY. */
454 		for (int i = 0; i < NR_CHURN; i++) {
455 			if (!mount(self->src_b, self->b, "erofs", MS_RDONLY, self->opts_b)) {
456 				if (umount2(self->b, 0))
457 					_exit(1);
458 			} else if (errno != EBUSY) {
459 				_exit(2);
460 			}
461 		}
462 		_exit(0);
463 	}
464 
465 	for (int i = 0; i < NR_CHURN; i++) {
466 		ASSERT_EQ(suspend(self), 0);
467 		ASSERT_EQ(resume(self), 0);
468 	}
469 	ASSERT_EQ(waitpid(pid, &status, 0), pid);
470 	ASSERT_TRUE(WIFEXITED(status));
471 	ASSERT_EQ(WEXITSTATUS(status), 0);
472 
473 	/* Nothing is left frozen and the blob can be claimed again. */
474 	ASSERT_FALSE(fithaw(self->fd_a));
475 	ASSERT_EQ(errno, EINVAL);
476 	ASSERT_EQ(mount_b(self), 0);
477 }
478 
479 TEST_HARNESS_MAIN
480