1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright 2026 Klara, Inc.
15 */
16
17 #ifdef __FreeBSD__
18 #include <sys/disk.h>
19 #endif
20 #include <sys/dmu.h>
21 #include <sys/ioctl.h>
22 #include <sys/spa.h>
23 #include <sys/stat.h>
24 #include <sys/uio.h>
25 #include <sys/zap.h>
26 #include <sys/zap_impl.h>
27 #include <sys/zfs_ioctl.h>
28 #include <sys/zio.h>
29 #include <sys/zstd/zstd.h>
30 #include <sys/zvol.h>
31 #include <err.h>
32 #include <libnvpair.h>
33 #ifdef __linux__
34 #include <linux/falloc.h>
35 #include <linux/fs.h>
36 #endif
37 #include <stddef.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <unistd.h>
42
43 #include "zstream.h"
44 #include "zstream_modules.h"
45 #include "zstream_util.h"
46
47 /*
48 * Supported feature flags (in drr_versioninfo)
49 */
50 #define SUPPORTED_FEATURES (DMU_BACKUP_FEATURE_EMBED_DATA | \
51 DMU_BACKUP_FEATURE_LZ4 | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \
52 DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_ZSTD)
53
54 typedef struct {
55 struct raw_stream {
56 uint64_t guid;
57 boolean_t inprop;
58 boolean_t inzvol;
59 } stream;
60 struct raw_volume {
61 int fd;
62 size_t size;
63 uint64_t sectorsize;
64 boolean_t isreg;
65 boolean_t punch_holes;
66 unsigned long freeop;
67 } volume;
68 struct raw_limits {
69 long iov_max;
70 long buffers_max;
71 long pagesize;
72 } limits;
73 struct raw_zeros {
74 struct iovec *iov;
75 } zeros;
76 struct raw_buffer {
77 struct iovec *iov;
78 off_t position;
79 size_t length;
80 int iovcnt;
81 } buffer;
82 } raw_context_t;
83
84 static void
write_zeros(raw_context_t * context,off_t offset,size_t length)85 write_zeros(raw_context_t *context, off_t offset, size_t length)
86 {
87 ASSERT3U(offset + length, >=, offset);
88
89 struct raw_limits *limits = &context->limits;
90 int iovcnt = MIN(howmany(length, limits->pagesize), limits->iov_max);
91 if (iovcnt == 0)
92 return;
93
94 int fd = context->volume.fd;
95 long pagesize = limits->pagesize;
96 struct iovec *iov = context->zeros.iov;
97 size_t resid = length;
98 while (resid > 0) {
99 size_t iovsz = resid;
100 int i;
101
102 for (i = 0; i < iovcnt && resid > 0; i++) {
103 iov[i].iov_len = MIN(resid, pagesize);
104 resid -= iov[i].iov_len;
105 }
106 ssize_t res = pwritev(fd, iov, i, offset);
107 if (res < 0)
108 err(EXIT_FAILURE, "pwritev");
109 iovsz -= resid;
110 VERIFY3U(res, ==, iovsz);
111 offset += iovsz;
112 }
113 ASSERT0(resid);
114 }
115
116 /*
117 * buffer_write - pwrite with buffer vectoring and error handling
118 *
119 * Appends buf to a buffer vector, issuing the pending vector if not contiguous.
120 * Ownership of buf is taken; it will be freed after issuing the write.
121 */
122 static void
buffer_write(raw_context_t * context,void * buf,size_t nbytes,off_t offset)123 buffer_write(raw_context_t *context, void *buf, size_t nbytes, off_t offset)
124 {
125 struct raw_limits *limits = &context->limits;
126 struct raw_buffer *buffer = &context->buffer;
127 struct iovec *iov = buffer->iov;
128
129 if (buffer->iovcnt == 0)
130 buffer->position = offset;
131 else if (buffer->position + buffer->length != offset ||
132 buffer->iovcnt == limits->buffers_max) {
133 ASSERT3U(offset + nbytes, >=, offset);
134 ASSERT3U(buffer->iovcnt, >, 0);
135 ssize_t res = pwritev(context->volume.fd, iov, buffer->iovcnt,
136 buffer->position);
137 if (res < 0)
138 err(EXIT_FAILURE, "pwritev");
139 VERIFY3U(res, ==, buffer->length);
140 buffer->position = offset;
141 buffer->length = 0;
142 for (int i = 0; i < buffer->iovcnt; i++) {
143 free(iov[i].iov_base);
144 iov[i].iov_base = NULL;
145 }
146 buffer->iovcnt = 0;
147 }
148 if (buf == NULL) {
149 /* Sentinel buf for flush. */
150 ASSERT0(buffer->iovcnt);
151 return;
152 }
153 iov[buffer->iovcnt].iov_base = buf;
154 iov[buffer->iovcnt].iov_len = nbytes;
155 buffer->length += nbytes;
156 buffer->iovcnt++;
157 }
158
159 static inline void
buffer_finish(raw_context_t * context)160 buffer_finish(raw_context_t *context)
161 {
162 buffer_write(context, NULL, 0, 0);
163 if (fsync(context->volume.fd) != 0)
164 err(EXIT_FAILURE, "fsync");
165 }
166
167 static inline void
resize(raw_context_t * context,size_t size)168 resize(raw_context_t *context, size_t size)
169 {
170 if (ftruncate(context->volume.fd, size) < 0)
171 err(EXIT_FAILURE, "ftruncate");
172 }
173
174 static inline void
free_tail(raw_context_t * context,off_t offset)175 free_tail(raw_context_t *context, off_t offset)
176 {
177 /*
178 * This style of FREE is frequently a large range covering most of the
179 * volume from the offset to the end. Truncating the file and extending
180 * it back out works cheaply even on filesystems that do not support
181 * hole punching, avoiding the need to write zeros.
182 */
183 resize(context, offset);
184 if (offset < context->volume.size)
185 resize(context, context->volume.size);
186 else
187 /* Volume size unknown, but it must be at least this big. */
188 context->volume.size = offset;
189 }
190
191 static inline boolean_t
punch_hole(raw_context_t * context,off_t offset,size_t length)192 punch_hole(raw_context_t *context, off_t offset, size_t length)
193 {
194 int fd = context->volume.fd;
195
196 #if defined(__FreeBSD__)
197 struct spacectl_range range = { offset, length };
198 return (fspacectl(fd, SPACECTL_DEALLOC, &range, 0, NULL) == 0);
199 #elif defined(__linux__)
200 int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
201 return (fallocate(fd, mode, offset, length) == 0);
202 #else
203 (void) fd, (void) offset, (void) length;
204 return (B_FALSE);
205 #endif
206 }
207
208 static boolean_t
free_blocks(raw_context_t * context,off_t offset,size_t length)209 free_blocks(raw_context_t *context, off_t offset, size_t length)
210 {
211 /*
212 * Ensure sector alignment in case the source zvol's sector size is
213 * smaller than the target device's.
214 *
215 * TODO: Optional secure erase.
216 */
217 uint64_t sectorsize = context->volume.sectorsize;
218 uint64_t start = P2ROUNDUP(offset, sectorsize);
219 uint64_t limit = offset + length;
220 uint64_t end = P2ALIGN_TYPED(limit, sectorsize, uint64_t);
221 if (offset < start)
222 write_zeros(context, offset, start - offset);
223 if (start < end) {
224 #ifdef __FreeBSD__
225 off_t range[2];
226 #else
227 uint64_t range[2];
228 #endif
229 int fd = context->volume.fd;
230
231 range[0] = start;
232 range[1] = end - start;
233 if (ioctl(fd, context->volume.freeop, range) != 0) {
234 ASSERT3U(errno, ==, EOPNOTSUPP);
235 context->volume.freeop = B_FALSE;
236 return (B_FALSE);
237 }
238 }
239 if (end < limit)
240 write_zeros(context, end, limit - end);
241 return (B_TRUE);
242 }
243
244 static void
free_range(raw_context_t * context,off_t offset,size_t length)245 free_range(raw_context_t *context, off_t offset, size_t length)
246 {
247 if (context->volume.isreg) {
248 if (length == (size_t)-1) {
249 free_tail(context, offset);
250 return;
251 }
252 if (context->volume.punch_holes) {
253 if (punch_hole(context, offset, length))
254 return;
255 context->volume.punch_holes = B_FALSE;
256 }
257 }
258 if (length == (size_t)-1)
259 length = context->volume.size - offset;
260 if (context->volume.freeop && free_blocks(context, offset, length))
261 return;
262 /* If all else fails, the range must be zeroed the slow way. */
263 write_zeros(context, offset, length);
264 }
265
266 /*
267 * apply_properties - read the properties zap to adjust file size.
268 *
269 * Returns the value of the "size" property.
270 */
271 static uint64_t
apply_properties(raw_context_t * context,void * buf,size_t len)272 apply_properties(raw_context_t *context, void *buf, size_t len)
273 {
274 const mzap_phys_t *mzap = buf;
275
276 ASSERT(context->volume.isreg);
277 ASSERT3U(len, >=, sizeof (*mzap));
278 ASSERT3U(MZAP_ENT_LEN, ==, sizeof (mzap_ent_phys_t));
279
280 if (mzap->mz_block_type == BSWAP_64(ZBT_MICRO))
281 zap_byteswap(buf, len);
282
283 ASSERT3U(mzap->mz_block_type, ==, ZBT_MICRO);
284 ASSERT0(strcmp(mzap->mz_chunk[0].mze_name, "size"));
285
286 uint64_t size = mzap->mz_chunk[0].mze_value;
287 resize(context, size);
288 return (size);
289 }
290
291 static disposition_t
chain_replay_raw(void * item_in,void * context_in)292 chain_replay_raw(void *item_in, void *context_in)
293 {
294 drr_packet_t *item = item_in;
295 if (item == NULL)
296 return (D_OK);
297
298 raw_context_t *context = context_in;
299 dmu_replay_record_t *drr = &item->dp_drr;
300
301 switch (drr->drr_type) {
302 case DRR_BEGIN: {
303 struct drr_begin *drrb = &drr->drr_u.drr_begin;
304
305 uint64_t featureflags, unsupported_features;
306 featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
307 unsupported_features = featureflags & ~SUPPORTED_FEATURES;
308 if (unsupported_features != 0)
309 errx(EXIT_FAILURE, "unsupported stream features: "
310 "%#llx of %#llx, aborting...",
311 (u_longlong_t)unsupported_features,
312 (u_longlong_t)featureflags);
313
314 int hdrtype = DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo);
315 if (hdrtype == DMU_SUBSTREAM) {
316 uint64_t guid = context->stream.guid;
317 if (guid != 0 && drrb->drr_fromguid != guid)
318 errx(EXIT_FAILURE, "wrong fromguid: "
319 "%llu != %llu, aborting...",
320 (u_longlong_t)drrb->drr_fromguid,
321 (u_longlong_t)guid);
322 context->stream.guid = drrb->drr_toguid;
323 }
324 break;
325 }
326 case DRR_OBJECT: {
327 struct drr_object *drro = &drr->drr_u.drr_object;
328
329 context->stream.inzvol = drro->drr_object == ZVOL_OBJ &&
330 drro->drr_type == DMU_OT_ZVOL;
331 context->stream.inprop = drro->drr_object == ZVOL_ZAP_OBJ &&
332 drro->drr_type == DMU_OT_ZVOL_PROP;
333 break;
334 }
335 case DRR_WRITE: {
336 struct drr_write *drrw = &drr->drr_u.drr_write;
337
338 if (context->stream.inzvol) {
339 buffer_write(context, item->dp_payload,
340 item->dp_payload_size, drrw->drr_offset);
341 /*
342 * The buffer is no longer owned by the chain. We will
343 * free it when safe.
344 */
345 item->dp_payload = NULL;
346 } else if (context->volume.isreg && context->stream.inprop) {
347 ASSERT0(drrw->drr_offset);
348 context->volume.size = apply_properties(context,
349 item->dp_payload, item->dp_payload_size);
350 }
351 break;
352 }
353 case DRR_FREE: {
354 if (!context->stream.inzvol)
355 break;
356
357 struct drr_free *drrf = &drr->drr_u.drr_free;
358 free_range(context, drrf->drr_offset, drrf->drr_length);
359 break;
360 }
361 case DRR_WRITE_EMBEDDED: {
362 struct drr_write_embedded *drrwe =
363 &drr->drr_u.drr_write_embedded;
364
365 if (!ctype_is_uncompressed(drrwe->drr_compression)) {
366 uint8_t *buffer = item->dp_payload;
367 uint32_t lsize = drrwe->drr_lsize;
368
369 ASSERT3U(item->dp_payload_size, <=, lsize);
370
371 item->dp_payload = decompress_buffer(buffer,
372 item->dp_payload_size, lsize,
373 drrwe->drr_compression);
374 if (item->dp_payload == NULL)
375 errx(EXIT_FAILURE,
376 "decompression failed at offset %llu",
377 (u_longlong_t)drrwe->drr_offset);
378 item->dp_payload_size = lsize;
379 free(buffer);
380 }
381 if (context->stream.inzvol) {
382 buffer_write(context, item->dp_payload,
383 item->dp_payload_size, drrwe->drr_offset);
384 /*
385 * The buffer is no longer owned by the chain. We will
386 * free it when safe.
387 */
388 item->dp_payload = NULL;
389 } else if (context->volume.isreg && context->stream.inprop) {
390 ASSERT0(drrwe->drr_offset);
391 context->volume.size = apply_properties(context,
392 item->dp_payload, item->dp_payload_size);
393 }
394 break;
395 }
396 default:
397 break;
398 }
399 return (D_OK);
400 }
401
402 /* Keep this small enough to not accidentally run systems out of memory. */
403 #define BUFFERS_MAX_DEFAULT 32
404
405 int
zstream_do_raw(int argc,char * argv[])406 zstream_do_raw(int argc, char *argv[])
407 {
408 raw_context_t context = { 0 };
409 context.limits.buffers_max = BUFFERS_MAX_DEFAULT;
410
411 chain_attrs_t attrs = { 0 };
412 ENABLE_OPTION(&attrs, CA_FORBID_DEDUP);
413
414 int c;
415 while ((c = getopt(argc, argv, ":b:g:v")) != -1) {
416 switch (c) {
417 case 'b':
418 context.limits.buffers_max = strtol(optarg, NULL, 0);
419 if (context.limits.buffers_max <= 0) {
420 warnx("invalid number of buffers");
421 zstream_usage();
422 }
423 break;
424 case 'g':
425 context.stream.guid = strtoull(optarg, NULL, 0);
426 if (context.stream.guid == 0) {
427 warnx("invalid guid");
428 zstream_usage();
429 }
430 break;
431 case 'v':
432 ENABLE_OPTION(&attrs, CA_VERBOSE);
433 ENABLE_OPTION(&attrs, CA_DUMP_ALL_RECORDS);
434 ENABLE_OPTION(&attrs, CA_DUMP_CHECKSUMS);
435 break;
436 case ':':
437 warnx("missing argument for '%c' option", optopt);
438 zstream_usage();
439 case '?':
440 warnx("invalid option '%c'", optopt);
441 zstream_usage();
442 }
443 }
444 argc -= optind;
445 argv += optind;
446 if (argc < 1) {
447 warnx("missing path to raw volume");
448 zstream_usage();
449 }
450
451 const char *raw_path = argv[0];
452 int fd = open(raw_path, O_WRONLY | O_CREAT, 0666);
453 if (fd < 0)
454 err(EXIT_FAILURE, "error while opening file '%s'", raw_path);
455 context.volume.fd = fd;
456 struct stat64 st;
457 if (fstat64_blk(fd, &st) < 0)
458 err(EXIT_FAILURE, "fstat64_blk");
459 context.volume.size = st.st_size;
460 context.volume.isreg = S_ISREG(st.st_mode);
461 if (!context.volume.isreg) {
462 #if defined(__FreeBSD__)
463 uint_t sectorsize;
464
465 if (ioctl(fd, DIOCGSECTORSIZE, §orsize) == 0) {
466 context.volume.sectorsize = sectorsize;
467 context.volume.freeop = DIOCGDELETE;
468 }
469 #elif defined(__linux__)
470 if (ioctl(fd, BLKSSZGET, &context.volume.sectorsize) == 0) {
471 /* TODO: optional BLKSECDISCARD/BLKZEROOUT */
472 context.volume.freeop = BLKDISCARD;
473 }
474 #endif
475 }
476
477 long iov_max = sysconf(_SC_IOV_MAX);
478 long pagesize = sysconf(_SC_PAGESIZE);
479
480 context.limits.iov_max = iov_max;
481 context.limits.buffers_max = MIN(context.limits.buffers_max, iov_max);
482 context.limits.pagesize = pagesize;
483
484 context.buffer.iov = safe_calloc(context.limits.buffers_max *
485 sizeof (struct iovec));
486
487 context.zeros.iov = safe_malloc(iov_max * sizeof (struct iovec));
488 void *zero_page = safe_calloc(pagesize);
489 for (int i = 0; i < iov_max; i++)
490 context.zeros.iov[i].iov_base = zero_page;
491
492 uint32_t drop_mask = DROP_END | DROP_FREEOBJECTS | DROP_OBJECT_RANGE |
493 DROP_REDACT | DROP_SPILL;
494 zstream_chain_t raw_chain = {
495 STANDARD_INPUT_STACK((argc > 1) ? argv[1] : NULL),
496 serial_dump_records(),
497 serial_drop_record_types(drop_mask),
498 parallel_decompress_writes(NULL),
499 {
500 .cs_type = CS_SERIAL,
501 .cs_in_size = sizeof (drr_packet_t),
502 .cs_out_size = sizeof (drr_packet_t),
503 .cs_context = &context,
504 .cs_serial = {.process = chain_replay_raw},
505 },
506 NULL_OUTPUT_STACK()
507 };
508 zstream_chain_exec(raw_chain, &attrs);
509
510 buffer_finish(&context);
511 free(zero_page);
512 free(context.zeros.iov);
513 free(context.buffer.iov);
514
515 if (OPTION_ENABLED(CA_VERBOSE))
516 (void) printf("now at guid %llu\n",
517 (u_longlong_t)context.stream.guid);
518 else
519 (void) printf("%llu\n", (u_longlong_t)context.stream.guid);
520
521 return (EXIT_SUCCESS);
522 }
523