1 /*-
2 * Copyright (c) 2017 Sean Purcell
3 * Copyright (c) 2023-2024 Klara, Inc.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "archive_platform.h"
28
29 #ifdef HAVE_ERRNO_H
30 #include <errno.h>
31 #endif
32 #ifdef HAVE_LIMITS_H
33 #include <limits.h>
34 #endif
35 #ifdef HAVE_STDINT_H
36 #include <stdint.h>
37 #endif
38 #ifdef HAVE_STDLIB_H
39 #include <stdlib.h>
40 #endif
41 #ifdef HAVE_STRING_H
42 #include <string.h>
43 #endif
44 #ifdef HAVE_UNISTD_H
45 #include <unistd.h>
46 #endif
47 #ifdef HAVE_ZSTD_H
48 #include <zstd.h>
49 #endif
50
51 #include "archive.h"
52 #include "archive_private.h"
53 #include "archive_string.h"
54 #include "archive_write_private.h"
55
56 /* Don't compile this if we don't have zstd.h */
57
58 struct zstd {
59 int compression_level;
60 int threads;
61 int long_distance;
62 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
63 enum {
64 running,
65 finishing,
66 resetting,
67 } state;
68 int frame_per_file;
69 size_t min_frame_in;
70 size_t max_frame_in;
71 size_t min_frame_out;
72 size_t max_frame_out;
73 size_t cur_frame;
74 size_t cur_frame_in;
75 size_t cur_frame_out;
76 ZSTD_CStream *cstream;
77 ZSTD_outBuffer out;
78 #else
79 struct archive_write_program_data *pdata;
80 #endif
81 };
82
83 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */
84 #define CLEVEL_MIN -99
85 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
86 #define CLEVEL_DEFAULT 3
87 #define CLEVEL_STD_MAX 19 /* without using --ultra */
88 #define CLEVEL_MAX 22
89
90 #define LONG_STD 27
91
92 #define MINVER_NEGCLEVEL 10304
93 #define MINVER_MINCLEVEL 10306
94 #define MINVER_LONG 10302
95
96 static int archive_compressor_zstd_options(struct archive_write_filter *,
97 const char *, const char *);
98 static int archive_compressor_zstd_open(struct archive_write_filter *);
99 static int archive_compressor_zstd_write(struct archive_write_filter *,
100 const void *, size_t);
101 static int archive_compressor_zstd_flush(struct archive_write_filter *);
102 static int archive_compressor_zstd_close(struct archive_write_filter *);
103 static int archive_compressor_zstd_free(struct archive_write_filter *);
104 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
105 static int drive_compressor(struct archive_write_filter *,
106 struct zstd *, int, const void *, size_t);
107 #endif
108 static void free_data(struct zstd *);
109
110
111 /*
112 * Add a zstd compression filter to this write handle.
113 */
114 int
archive_write_add_filter_zstd(struct archive * a)115 archive_write_add_filter_zstd(struct archive *a)
116 {
117 struct archive_write_filter *f;
118 struct zstd *zstd;
119 int r;
120
121 archive_check_magic(a, ARCHIVE_WRITE_MAGIC,
122 ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
123
124 zstd = calloc(1, sizeof(*zstd));
125 if (zstd == NULL)
126 goto memerr;
127 zstd->compression_level = CLEVEL_DEFAULT;
128 zstd->threads = 0;
129 zstd->long_distance = 0;
130 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
131 zstd->frame_per_file = 0;
132 zstd->min_frame_in = 0;
133 zstd->max_frame_in = SIZE_MAX;
134 zstd->min_frame_out = 0;
135 zstd->max_frame_out = SIZE_MAX;
136 zstd->cur_frame_in = 0;
137 zstd->cur_frame_out = 0;
138 zstd->cstream = ZSTD_createCStream();
139 if (zstd->cstream == NULL)
140 goto memerr;
141
142 r = ARCHIVE_OK;
143 #else
144 zstd->pdata = __archive_write_program_allocate("zstd");
145 if (zstd->pdata == NULL)
146 goto memerr;
147
148 archive_set_error(a, ARCHIVE_ERRNO_MISC,
149 "Using external zstd program");
150 r = ARCHIVE_WARN;
151 #endif
152
153 f = __archive_write_allocate_filter(a);
154 if (f == NULL)
155 goto memerr;
156 f->name = "zstd";
157 f->code = ARCHIVE_FILTER_ZSTD;
158 f->data = zstd;
159 f->options = archive_compressor_zstd_options;
160 f->open = archive_compressor_zstd_open;
161 f->write = archive_compressor_zstd_write;
162 f->flush = archive_compressor_zstd_flush;
163 f->close = archive_compressor_zstd_close;
164 f->free = archive_compressor_zstd_free;
165
166 return (r);
167 memerr:
168 free_data(zstd);
169 archive_set_error(a, ENOMEM, "Out of memory");
170 return (ARCHIVE_FATAL);
171 }
172
173 static int
archive_compressor_zstd_free(struct archive_write_filter * f)174 archive_compressor_zstd_free(struct archive_write_filter *f)
175 {
176 free_data(f->data);
177 f->data = NULL;
178 return (ARCHIVE_OK);
179 }
180
181 static int
string_to_number(const char * string,intmax_t * numberp)182 string_to_number(const char *string, intmax_t *numberp)
183 {
184 char *end;
185
186 if (string == NULL || *string == '\0')
187 return (ARCHIVE_WARN);
188 *numberp = strtoimax(string, &end, 10);
189 if (end == string || *end != '\0' || errno == EOVERFLOW) {
190 *numberp = 0;
191 return (ARCHIVE_WARN);
192 }
193 return (ARCHIVE_OK);
194 }
195
196 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
197 static int
string_to_size(const char * string,size_t * numberp)198 string_to_size(const char *string, size_t *numberp)
199 {
200 uintmax_t number;
201 char *end;
202 unsigned int shift = 0;
203
204 if (string == NULL || *string == '\0' || *string == '-')
205 return (ARCHIVE_WARN);
206 number = strtoumax(string, &end, 10);
207 if (end > string) {
208 if (*end == 'K' || *end == 'k') {
209 shift = 10;
210 end++;
211 } else if (*end == 'M' || *end == 'm') {
212 shift = 20;
213 end++;
214 } else if (*end == 'G' || *end == 'g') {
215 shift = 30;
216 end++;
217 }
218 if (*end == 'B' || *end == 'b') {
219 end++;
220 }
221 }
222 if (end == string || *end != '\0' || errno == EOVERFLOW) {
223 return (ARCHIVE_WARN);
224 }
225 if (number > (uintmax_t)SIZE_MAX >> shift) {
226 return (ARCHIVE_WARN);
227 }
228 *numberp = (size_t)(number << shift);
229 return (ARCHIVE_OK);
230 }
231 #endif
232
233 /*
234 * Set write options.
235 */
236 static int
archive_compressor_zstd_options(struct archive_write_filter * f,const char * key,const char * value)237 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
238 const char *value)
239 {
240 struct zstd *zstd = f->data;
241
242 if (strcmp(key, "compression-level") == 0) {
243 intmax_t level;
244 if (string_to_number(value, &level) != ARCHIVE_OK) {
245 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
246 "compression-level invalid");
247 return (ARCHIVE_FAILED);
248 }
249 /* If we don't have the library, hard-code the max level */
250 int minimum = CLEVEL_MIN;
251 int maximum = CLEVEL_MAX;
252 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
253 maximum = ZSTD_maxCLevel();
254 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
255 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
256 minimum = ZSTD_minCLevel();
257 }
258 else
259 #endif
260 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
261 minimum = CLEVEL_STD_MIN;
262 }
263 #endif
264 if (level < minimum || level > maximum) {
265 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
266 "compression-level out of range");
267 return (ARCHIVE_FAILED);
268 }
269 zstd->compression_level = (int)level;
270 return (ARCHIVE_OK);
271 } else if (strcmp(key, "threads") == 0) {
272 intmax_t threads;
273 if (string_to_number(value, &threads) != ARCHIVE_OK) {
274 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
275 "threads invalid");
276 return (ARCHIVE_FAILED);
277 }
278
279 #if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)
280 if (threads == 0) {
281 threads = sysconf(_SC_NPROCESSORS_ONLN);
282 }
283 #elif !defined(__CYGWIN__) && defined(_WIN32_WINNT) && \
284 _WIN32_WINNT >= 0x0601 /* _WIN32_WINNT_WIN7 */
285 if (threads == 0) {
286 DWORD winCores = GetActiveProcessorCount(
287 ALL_PROCESSOR_GROUPS);
288 threads = (intmax_t)winCores;
289 }
290 #endif
291 if (threads < 0 || threads > INT_MAX) {
292 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
293 "threads out of rnage");
294 return (ARCHIVE_FAILED);
295 }
296 zstd->threads = (int)threads;
297 return (ARCHIVE_OK);
298 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
299 } else if (strcmp(key, "frame-per-file") == 0) {
300 zstd->frame_per_file = 1;
301 return (ARCHIVE_OK);
302 } else if (strcmp(key, "min-frame-in") == 0) {
303 if (string_to_size(value, &zstd->min_frame_in) != ARCHIVE_OK) {
304 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
305 "min-frame-in invalid");
306 return (ARCHIVE_FAILED);
307 }
308 return (ARCHIVE_OK);
309 } else if (strcmp(key, "min-frame-out") == 0 ||
310 strcmp(key, "min-frame-size") == 0) {
311 if (string_to_size(value, &zstd->min_frame_out) != ARCHIVE_OK) {
312 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
313 "min-frame-out invalid");
314 return (ARCHIVE_FAILED);
315 }
316 return (ARCHIVE_OK);
317 } else if (strcmp(key, "max-frame-in") == 0 ||
318 strcmp(key, "max-frame-size") == 0) {
319 if (string_to_size(value, &zstd->max_frame_in) != ARCHIVE_OK ||
320 zstd->max_frame_in < 1024) {
321 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
322 "max-frame-size invalid");
323 return (ARCHIVE_FAILED);
324 }
325 return (ARCHIVE_OK);
326 } else if (strcmp(key, "max-frame-out") == 0) {
327 if (string_to_size(value, &zstd->max_frame_out) != ARCHIVE_OK ||
328 zstd->max_frame_out < 1024) {
329 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
330 "max-frame-out invalid");
331 return (ARCHIVE_FAILED);
332 }
333 return (ARCHIVE_OK);
334 #endif
335 }
336 else if (strcmp(key, "long") == 0) {
337 intmax_t long_distance;
338 if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
339 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
340 "long invalid");
341 return (ARCHIVE_FAILED);
342 }
343 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
344 ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
345 if (ZSTD_isError(bounds.error)) {
346 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
347 if (((int)long_distance) < 10 || (int)long_distance > max_distance) {
348 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
349 "long out of range");
350 return (ARCHIVE_FAILED);
351 }
352 } else {
353 if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound) {
354 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
355 "long out of range");
356 return (ARCHIVE_FAILED);
357 }
358 }
359 #else
360 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
361 if (((int)long_distance) < 10 || (int)long_distance > max_distance)
362 return (ARCHIVE_FAILED);
363 #endif
364 zstd->long_distance = (int)long_distance;
365 return (ARCHIVE_OK);
366 }
367
368 /* Note: The "warn" return is just to inform the options
369 * supervisor that we didn't handle it. It will generate
370 * a suitable error if no one used this option. */
371 return (ARCHIVE_WARN);
372 }
373
374 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
375 /*
376 * Setup callback.
377 */
378 static int
archive_compressor_zstd_open(struct archive_write_filter * f)379 archive_compressor_zstd_open(struct archive_write_filter *f)
380 {
381 struct zstd *zstd = f->data;
382
383 if (zstd->out.dst == NULL) {
384 size_t bs = ZSTD_CStreamOutSize(), bpb;
385 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
386 /* Buffer size should be a multiple number of
387 * the of bytes per block for performance. */
388 bpb = archive_write_get_bytes_per_block(f->archive);
389 if (bpb > bs)
390 bs = bpb;
391 else if (bpb != 0)
392 bs -= bs % bpb;
393 }
394 zstd->out.size = bs;
395 zstd->out.pos = 0;
396 zstd->out.dst = malloc(zstd->out.size);
397 if (zstd->out.dst == NULL) {
398 archive_set_error(f->archive, ENOMEM,
399 "Can't allocate data for compression buffer");
400 return (ARCHIVE_FATAL);
401 }
402 }
403
404 if (ZSTD_isError(ZSTD_initCStream(zstd->cstream,
405 zstd->compression_level))) {
406 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
407 "Internal error initializing zstd compressor object");
408 return (ARCHIVE_FATAL);
409 }
410
411 ZSTD_CCtx_setParameter(zstd->cstream, ZSTD_c_nbWorkers, zstd->threads);
412
413 ZSTD_CCtx_setParameter(zstd->cstream, ZSTD_c_checksumFlag, 1);
414
415 #if ZSTD_VERSION_NUMBER >= MINVER_LONG
416 ZSTD_CCtx_setParameter(zstd->cstream, ZSTD_c_windowLog, zstd->long_distance);
417 #endif
418
419 return (ARCHIVE_OK);
420 }
421
422 /*
423 * Write data to the compressed stream.
424 */
425 static int
archive_compressor_zstd_write(struct archive_write_filter * f,const void * buff,size_t length)426 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
427 size_t length)
428 {
429 struct zstd *zstd = f->data;
430
431 return (drive_compressor(f, zstd, 0, buff, length));
432 }
433
434 /*
435 * Flush the compressed stream.
436 */
437 static int
archive_compressor_zstd_flush(struct archive_write_filter * f)438 archive_compressor_zstd_flush(struct archive_write_filter *f)
439 {
440 struct zstd *zstd = f->data;
441
442 if (zstd->frame_per_file && zstd->state == running) {
443 if (zstd->cur_frame_in > zstd->min_frame_in &&
444 zstd->cur_frame_out > zstd->min_frame_out) {
445 zstd->state = finishing;
446 }
447 }
448 return (drive_compressor(f, zstd, 1, NULL, 0));
449 }
450
451 /*
452 * Finish the compression...
453 */
454 static int
archive_compressor_zstd_close(struct archive_write_filter * f)455 archive_compressor_zstd_close(struct archive_write_filter *f)
456 {
457 struct zstd *zstd = f->data;
458
459 if (zstd->state == running)
460 zstd->state = finishing;
461 return (drive_compressor(f, zstd, 1, NULL, 0));
462 }
463
464 /*
465 * Utility function to push input data through compressor,
466 * writing full output blocks as necessary.
467 */
468 static int
drive_compressor(struct archive_write_filter * f,struct zstd * zstd,int flush,const void * src,size_t length)469 drive_compressor(struct archive_write_filter *f,
470 struct zstd *zstd, int flush, const void *src, size_t length)
471 {
472 ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
473 size_t ipos, opos, zstdret = 0;
474 int ret;
475
476 for (;;) {
477 ipos = in.pos;
478 opos = zstd->out.pos;
479 switch (zstd->state) {
480 case running:
481 if (in.pos == in.size)
482 return (ARCHIVE_OK);
483 zstdret = ZSTD_compressStream(zstd->cstream,
484 &zstd->out, &in);
485 if (ZSTD_isError(zstdret))
486 goto zstd_fatal;
487 break;
488 case finishing:
489 zstdret = ZSTD_endStream(zstd->cstream, &zstd->out);
490 if (ZSTD_isError(zstdret))
491 goto zstd_fatal;
492 if (zstdret == 0)
493 zstd->state = resetting;
494 break;
495 case resetting:
496 ZSTD_CCtx_reset(zstd->cstream, ZSTD_reset_session_only);
497 zstd->cur_frame++;
498 zstd->cur_frame_in = 0;
499 zstd->cur_frame_out = 0;
500 zstd->state = running;
501 break;
502 }
503 zstd->cur_frame_in += in.pos - ipos;
504 zstd->cur_frame_out += zstd->out.pos - opos;
505 if (zstd->state == running) {
506 if (zstd->cur_frame_in >= zstd->max_frame_in ||
507 zstd->cur_frame_out >= zstd->max_frame_out) {
508 zstd->state = finishing;
509 }
510 }
511 if (zstd->out.pos == zstd->out.size ||
512 (flush && zstd->out.pos > 0)) {
513 ret = __archive_write_filter(f->next_filter,
514 zstd->out.dst, zstd->out.pos);
515 if (ret != ARCHIVE_OK)
516 goto fatal;
517 zstd->out.pos = 0;
518 }
519 }
520 zstd_fatal:
521 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
522 "Zstd compression failed: %s",
523 ZSTD_getErrorName(zstdret));
524 fatal:
525 return (ARCHIVE_FATAL);
526 }
527
528 static void
free_data(struct zstd * zstd)529 free_data(struct zstd *zstd)
530 {
531 if (zstd != NULL) {
532 ZSTD_freeCStream(zstd->cstream);
533 free(zstd->out.dst);
534 free(zstd);
535 }
536 }
537
538 #else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
539
540 static int
archive_compressor_zstd_open(struct archive_write_filter * f)541 archive_compressor_zstd_open(struct archive_write_filter *f)
542 {
543 struct zstd *zstd = f->data;
544 struct archive_string as;
545 int r;
546
547 archive_string_init(&as);
548 /* --no-check matches library default */
549 archive_strcpy(&as, "zstd --no-check");
550
551 if (zstd->compression_level < CLEVEL_STD_MIN) {
552 archive_string_sprintf(&as, " --fast=%d", -zstd->compression_level);
553 } else {
554 archive_string_sprintf(&as, " -%d", zstd->compression_level);
555 }
556
557 if (zstd->compression_level > CLEVEL_STD_MAX) {
558 archive_strcat(&as, " --ultra");
559 }
560
561 if (zstd->threads != 0) {
562 archive_string_sprintf(&as, " --threads=%d", zstd->threads);
563 }
564
565 if (zstd->long_distance != 0) {
566 archive_string_sprintf(&as, " --long=%d", zstd->long_distance);
567 }
568
569 r = __archive_write_program_open(f, zstd->pdata, as.s);
570 archive_string_free(&as);
571 return (r);
572 }
573
574 static int
archive_compressor_zstd_write(struct archive_write_filter * f,const void * buff,size_t length)575 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
576 size_t length)
577 {
578 struct zstd *zstd = f->data;
579
580 return __archive_write_program_write(f, zstd->pdata, buff, length);
581 }
582
583 static int
archive_compressor_zstd_flush(struct archive_write_filter * f)584 archive_compressor_zstd_flush(struct archive_write_filter *f)
585 {
586 (void)f; /* UNUSED */
587
588 return (ARCHIVE_OK);
589 }
590
591 static int
archive_compressor_zstd_close(struct archive_write_filter * f)592 archive_compressor_zstd_close(struct archive_write_filter *f)
593 {
594 struct zstd *zstd = f->data;
595
596 return __archive_write_program_close(f, zstd->pdata);
597 }
598
599 static void
free_data(struct zstd * zstd)600 free_data(struct zstd *zstd)
601 {
602 if (zstd != NULL) {
603 __archive_write_program_free(zstd->pdata);
604 free(zstd);
605 }
606 }
607
608 #endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
609