1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2022 Axcient. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Copyright (c) 2024, Klara, Inc. 27 */ 28 29 #include <err.h> 30 #include <search.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <unistd.h> 34 #include <sys/zfs_ioctl.h> 35 #include <sys/zio_checksum.h> 36 #include <sys/zstd/zstd.h> 37 #include "zfs_fletcher.h" 38 #include "zstream.h" 39 40 static int 41 dump_record(dmu_replay_record_t *drr, void *payload, int payload_len, 42 zio_cksum_t *zc, int outfd) 43 { 44 assert(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum) 45 == sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); 46 fletcher_4_incremental_native(drr, 47 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), zc); 48 if (drr->drr_type != DRR_BEGIN) { 49 assert(ZIO_CHECKSUM_IS_ZERO(&drr->drr_u. 50 drr_checksum.drr_checksum)); 51 drr->drr_u.drr_checksum.drr_checksum = *zc; 52 } 53 fletcher_4_incremental_native(&drr->drr_u.drr_checksum.drr_checksum, 54 sizeof (zio_cksum_t), zc); 55 if (write(outfd, drr, sizeof (*drr)) == -1) 56 return (errno); 57 if (payload_len != 0) { 58 fletcher_4_incremental_native(payload, payload_len, zc); 59 if (write(outfd, payload, payload_len) == -1) 60 return (errno); 61 } 62 return (0); 63 } 64 65 int 66 zstream_do_decompress(int argc, char *argv[]) 67 { 68 const int KEYSIZE = 64; 69 int bufsz = SPA_MAXBLOCKSIZE; 70 char *buf = safe_malloc(bufsz); 71 dmu_replay_record_t thedrr; 72 dmu_replay_record_t *drr = &thedrr; 73 zio_cksum_t stream_cksum; 74 int c; 75 boolean_t verbose = B_FALSE; 76 77 while ((c = getopt(argc, argv, "v")) != -1) { 78 switch (c) { 79 case 'v': 80 verbose = B_TRUE; 81 break; 82 case '?': 83 (void) fprintf(stderr, "invalid option '%c'\n", 84 optopt); 85 zstream_usage(); 86 break; 87 } 88 } 89 90 argc -= optind; 91 argv += optind; 92 93 if (argc < 0) 94 zstream_usage(); 95 96 if (hcreate(argc) == 0) 97 errx(1, "hcreate"); 98 for (int i = 0; i < argc; i++) { 99 uint64_t object, offset; 100 char *obj_str; 101 char *offset_str; 102 char *key; 103 char *end; 104 enum zio_compress type = ZIO_COMPRESS_LZ4; 105 106 obj_str = strsep(&argv[i], ","); 107 if (argv[i] == NULL) { 108 zstream_usage(); 109 exit(2); 110 } 111 errno = 0; 112 object = strtoull(obj_str, &end, 0); 113 if (errno || *end != '\0') 114 errx(1, "invalid value for object"); 115 offset_str = strsep(&argv[i], ","); 116 offset = strtoull(offset_str, &end, 0); 117 if (errno || *end != '\0') 118 errx(1, "invalid value for offset"); 119 if (argv[i]) { 120 if (0 == strcmp("off", argv[i])) 121 type = ZIO_COMPRESS_OFF; 122 else if (0 == strcmp("lz4", argv[i])) 123 type = ZIO_COMPRESS_LZ4; 124 else if (0 == strcmp("lzjb", argv[i])) 125 type = ZIO_COMPRESS_LZJB; 126 else if (0 == strcmp("gzip", argv[i])) 127 type = ZIO_COMPRESS_GZIP_1; 128 else if (0 == strcmp("zle", argv[i])) 129 type = ZIO_COMPRESS_ZLE; 130 else if (0 == strcmp("zstd", argv[i])) 131 type = ZIO_COMPRESS_ZSTD; 132 else { 133 fprintf(stderr, "Invalid compression type %s.\n" 134 "Supported types are off, lz4, lzjb, gzip, " 135 "zle, and zstd\n", 136 argv[i]); 137 exit(2); 138 } 139 } 140 141 if (asprintf(&key, "%llu,%llu", (u_longlong_t)object, 142 (u_longlong_t)offset) < 0) { 143 err(1, "asprintf"); 144 } 145 ENTRY e = {.key = key}; 146 ENTRY *p; 147 148 p = hsearch(e, ENTER); 149 if (p == NULL) 150 errx(1, "hsearch"); 151 p->data = (void*)(intptr_t)type; 152 } 153 154 if (isatty(STDIN_FILENO)) { 155 (void) fprintf(stderr, 156 "Error: The send stream is a binary format " 157 "and can not be read from a\n" 158 "terminal. Standard input must be redirected.\n"); 159 exit(1); 160 } 161 162 fletcher_4_init(); 163 int begin = 0; 164 boolean_t seen = B_FALSE; 165 while (sfread(drr, sizeof (*drr), stdin) != 0) { 166 struct drr_write *drrw; 167 uint64_t payload_size = 0; 168 169 /* 170 * We need to regenerate the checksum. 171 */ 172 if (drr->drr_type != DRR_BEGIN) { 173 memset(&drr->drr_u.drr_checksum.drr_checksum, 0, 174 sizeof (drr->drr_u.drr_checksum.drr_checksum)); 175 } 176 177 switch (drr->drr_type) { 178 case DRR_BEGIN: 179 { 180 ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); 181 VERIFY0(begin++); 182 seen = B_TRUE; 183 184 uint32_t sz = drr->drr_payloadlen; 185 186 VERIFY3U(sz, <=, 1U << 28); 187 188 if (sz != 0) { 189 if (sz > bufsz) { 190 buf = realloc(buf, sz); 191 if (buf == NULL) 192 err(1, "realloc"); 193 bufsz = sz; 194 } 195 (void) sfread(buf, sz, stdin); 196 } 197 payload_size = sz; 198 break; 199 } 200 case DRR_END: 201 { 202 struct drr_end *drre = &drr->drr_u.drr_end; 203 /* 204 * We would prefer to just check --begin == 0, but 205 * replication streams have an end of stream END 206 * record, so we must avoid tripping it. 207 */ 208 VERIFY3B(seen, ==, B_TRUE); 209 begin--; 210 /* 211 * Use the recalculated checksum, unless this is 212 * the END record of a stream package, which has 213 * no checksum. 214 */ 215 if (!ZIO_CHECKSUM_IS_ZERO(&drre->drr_checksum)) 216 drre->drr_checksum = stream_cksum; 217 break; 218 } 219 220 case DRR_OBJECT: 221 { 222 struct drr_object *drro = &drr->drr_u.drr_object; 223 VERIFY3S(begin, ==, 1); 224 225 if (drro->drr_bonuslen > 0) { 226 payload_size = DRR_OBJECT_PAYLOAD_SIZE(drro); 227 (void) sfread(buf, payload_size, stdin); 228 } 229 break; 230 } 231 232 case DRR_SPILL: 233 { 234 struct drr_spill *drrs = &drr->drr_u.drr_spill; 235 VERIFY3S(begin, ==, 1); 236 payload_size = DRR_SPILL_PAYLOAD_SIZE(drrs); 237 (void) sfread(buf, payload_size, stdin); 238 break; 239 } 240 241 case DRR_WRITE_BYREF: 242 VERIFY3S(begin, ==, 1); 243 fprintf(stderr, 244 "Deduplicated streams are not supported\n"); 245 exit(1); 246 break; 247 248 case DRR_WRITE: 249 { 250 VERIFY3S(begin, ==, 1); 251 drrw = &thedrr.drr_u.drr_write; 252 payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); 253 ENTRY *p; 254 char key[KEYSIZE]; 255 256 snprintf(key, KEYSIZE, "%llu,%llu", 257 (u_longlong_t)drrw->drr_object, 258 (u_longlong_t)drrw->drr_offset); 259 ENTRY e = {.key = key}; 260 261 p = hsearch(e, FIND); 262 if (p == NULL) { 263 /* 264 * Read the contents of the block unaltered 265 */ 266 (void) sfread(buf, payload_size, stdin); 267 break; 268 } 269 270 /* 271 * Read and decompress the block 272 */ 273 enum zio_compress c = 274 (enum zio_compress)(intptr_t)p->data; 275 276 if (c == ZIO_COMPRESS_OFF) { 277 (void) sfread(buf, payload_size, stdin); 278 drrw->drr_compressiontype = 0; 279 drrw->drr_compressed_size = 0; 280 if (verbose) 281 fprintf(stderr, 282 "Resetting compression type to " 283 "off for ino %llu offset %llu\n", 284 (u_longlong_t)drrw->drr_object, 285 (u_longlong_t)drrw->drr_offset); 286 break; 287 } 288 289 uint64_t lsize = drrw->drr_logical_size; 290 ASSERT3U(payload_size, <=, lsize); 291 292 char *lzbuf = safe_calloc(payload_size); 293 (void) sfread(lzbuf, payload_size, stdin); 294 295 abd_t sabd, dabd; 296 abd_get_from_buf_struct(&sabd, lzbuf, payload_size); 297 abd_get_from_buf_struct(&dabd, buf, lsize); 298 int err = zio_decompress_data(c, &sabd, &dabd, 299 payload_size, lsize, NULL); 300 abd_free(&dabd); 301 abd_free(&sabd); 302 303 if (err == 0) { 304 drrw->drr_compressiontype = 0; 305 drrw->drr_compressed_size = 0; 306 payload_size = lsize; 307 if (verbose) { 308 fprintf(stderr, 309 "successfully decompressed " 310 "ino %llu offset %llu\n", 311 (u_longlong_t)drrw->drr_object, 312 (u_longlong_t)drrw->drr_offset); 313 } 314 } else { 315 /* 316 * The block must not be compressed, at least 317 * not with this compression type, possibly 318 * because it gets written multiple times in 319 * this stream. 320 */ 321 warnx("decompression failed for " 322 "ino %llu offset %llu", 323 (u_longlong_t)drrw->drr_object, 324 (u_longlong_t)drrw->drr_offset); 325 memcpy(buf, lzbuf, payload_size); 326 } 327 328 free(lzbuf); 329 break; 330 } 331 332 case DRR_WRITE_EMBEDDED: 333 { 334 VERIFY3S(begin, ==, 1); 335 struct drr_write_embedded *drrwe = 336 &drr->drr_u.drr_write_embedded; 337 payload_size = 338 P2ROUNDUP((uint64_t)drrwe->drr_psize, 8); 339 (void) sfread(buf, payload_size, stdin); 340 break; 341 } 342 343 case DRR_FREEOBJECTS: 344 case DRR_FREE: 345 case DRR_OBJECT_RANGE: 346 VERIFY3S(begin, ==, 1); 347 break; 348 349 default: 350 (void) fprintf(stderr, "INVALID record type 0x%x\n", 351 drr->drr_type); 352 /* should never happen, so assert */ 353 assert(B_FALSE); 354 } 355 356 if (feof(stdout)) { 357 fprintf(stderr, "Error: unexpected end-of-file\n"); 358 exit(1); 359 } 360 if (ferror(stdout)) { 361 fprintf(stderr, "Error while reading file: %s\n", 362 strerror(errno)); 363 exit(1); 364 } 365 366 /* 367 * We need to recalculate the checksum, and it needs to be 368 * initially zero to do that. BEGIN records don't have 369 * a checksum. 370 */ 371 if (drr->drr_type != DRR_BEGIN) { 372 memset(&drr->drr_u.drr_checksum.drr_checksum, 0, 373 sizeof (drr->drr_u.drr_checksum.drr_checksum)); 374 } 375 if (dump_record(drr, buf, payload_size, 376 &stream_cksum, STDOUT_FILENO) != 0) 377 break; 378 if (drr->drr_type == DRR_END) { 379 /* 380 * Typically the END record is either the last 381 * thing in the stream, or it is followed 382 * by a BEGIN record (which also zeros the checksum). 383 * However, a stream package ends with two END 384 * records. The last END record's checksum starts 385 * from zero. 386 */ 387 ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0); 388 } 389 } 390 free(buf); 391 fletcher_4_fini(); 392 hdestroy(); 393 394 return (0); 395 } 396