1 /////////////////////////////////////////////////////////////////////////////// 2 // 3 /// \file block_buffer_encoder.c 4 /// \brief Single-call .xz Block encoder 5 // 6 // Author: Lasse Collin 7 // 8 // This file has been put into the public domain. 9 // You can do whatever you want with this file. 10 // 11 /////////////////////////////////////////////////////////////////////////////// 12 13 #include "block_buffer_encoder.h" 14 #include "block_encoder.h" 15 #include "filter_encoder.h" 16 #include "lzma2_encoder.h" 17 #include "check.h" 18 19 20 /// Estimate the maximum size of the Block Header and Check fields for 21 /// a Block that uses LZMA2 uncompressed chunks. We could use 22 /// lzma_block_header_size() but this is simpler. 23 /// 24 /// Block Header Size + Block Flags + Compressed Size 25 /// + Uncompressed Size + Filter Flags for LZMA2 + CRC32 + Check 26 /// and round up to the next multiple of four to take Header Padding 27 /// into account. 28 #define HEADERS_BOUND ((1 + 1 + 2 * LZMA_VLI_BYTES_MAX + 3 + 4 \ 29 + LZMA_CHECK_SIZE_MAX + 3) & ~3) 30 31 32 static uint64_t 33 lzma2_bound(uint64_t uncompressed_size) 34 { 35 // Prevent integer overflow in overhead calculation. 36 if (uncompressed_size > COMPRESSED_SIZE_MAX) 37 return 0; 38 39 // Calculate the exact overhead of the LZMA2 headers: Round 40 // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, 41 // multiply by the size of per-chunk header, and add one byte for 42 // the end marker. 43 const uint64_t overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) 44 / LZMA2_CHUNK_MAX) 45 * LZMA2_HEADER_UNCOMPRESSED + 1; 46 47 // Catch the possible integer overflow. 48 if (COMPRESSED_SIZE_MAX - overhead < uncompressed_size) 49 return 0; 50 51 return uncompressed_size + overhead; 52 } 53 54 55 extern uint64_t 56 lzma_block_buffer_bound64(uint64_t uncompressed_size) 57 { 58 // If the data doesn't compress, we always use uncompressed 59 // LZMA2 chunks. 60 uint64_t lzma2_size = lzma2_bound(uncompressed_size); 61 if (lzma2_size == 0) 62 return 0; 63 64 // Take Block Padding into account. 65 lzma2_size = (lzma2_size + 3) & ~UINT64_C(3); 66 67 // No risk of integer overflow because lzma2_bound() already takes 68 // into account the size of the headers in the Block. 69 return HEADERS_BOUND + lzma2_size; 70 } 71 72 73 extern LZMA_API(size_t) 74 lzma_block_buffer_bound(size_t uncompressed_size) 75 { 76 uint64_t ret = lzma_block_buffer_bound64(uncompressed_size); 77 78 #if SIZE_MAX < UINT64_MAX 79 // Catch the possible integer overflow on 32-bit systems. 80 if (ret > SIZE_MAX) 81 return 0; 82 #endif 83 84 return ret; 85 } 86 87 88 static lzma_ret 89 block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, 90 uint8_t *out, size_t *out_pos, size_t out_size) 91 { 92 // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at 93 // all, but LZMA2 always requires a dictionary, so use the minimum 94 // value to minimize memory usage of the decoder. 95 lzma_options_lzma lzma2 = { 96 .dict_size = LZMA_DICT_SIZE_MIN, 97 }; 98 99 lzma_filter filters[2]; 100 filters[0].id = LZMA_FILTER_LZMA2; 101 filters[0].options = &lzma2; 102 filters[1].id = LZMA_VLI_UNKNOWN; 103 104 // Set the above filter options to *block temporarily so that we can 105 // encode the Block Header. 106 lzma_filter *filters_orig = block->filters; 107 block->filters = filters; 108 109 if (lzma_block_header_size(block) != LZMA_OK) { 110 block->filters = filters_orig; 111 return LZMA_PROG_ERROR; 112 } 113 114 // Check that there's enough output space. The caller has already 115 // set block->compressed_size to what lzma2_bound() has returned, 116 // so we can reuse that value. We know that compressed_size is a 117 // known valid VLI and header_size is a small value so their sum 118 // will never overflow. 119 assert(block->compressed_size == lzma2_bound(in_size)); 120 if (out_size - *out_pos 121 < block->header_size + block->compressed_size) { 122 block->filters = filters_orig; 123 return LZMA_BUF_ERROR; 124 } 125 126 if (lzma_block_header_encode(block, out + *out_pos) != LZMA_OK) { 127 block->filters = filters_orig; 128 return LZMA_PROG_ERROR; 129 } 130 131 block->filters = filters_orig; 132 *out_pos += block->header_size; 133 134 // Encode the data using LZMA2 uncompressed chunks. 135 size_t in_pos = 0; 136 uint8_t control = 0x01; // Dictionary reset 137 138 while (in_pos < in_size) { 139 // Control byte: Indicate uncompressed chunk, of which 140 // the first resets the dictionary. 141 out[(*out_pos)++] = control; 142 control = 0x02; // No dictionary reset 143 144 // Size of the uncompressed chunk 145 const size_t copy_size 146 = my_min(in_size - in_pos, LZMA2_CHUNK_MAX); 147 out[(*out_pos)++] = (copy_size - 1) >> 8; 148 out[(*out_pos)++] = (copy_size - 1) & 0xFF; 149 150 // The actual data 151 assert(*out_pos + copy_size <= out_size); 152 memcpy(out + *out_pos, in + in_pos, copy_size); 153 154 in_pos += copy_size; 155 *out_pos += copy_size; 156 } 157 158 // End marker 159 out[(*out_pos)++] = 0x00; 160 assert(*out_pos <= out_size); 161 162 return LZMA_OK; 163 } 164 165 166 static lzma_ret 167 block_encode_normal(lzma_block *block, const lzma_allocator *allocator, 168 const uint8_t *in, size_t in_size, 169 uint8_t *out, size_t *out_pos, size_t out_size) 170 { 171 // Find out the size of the Block Header. 172 return_if_error(lzma_block_header_size(block)); 173 174 // Reserve space for the Block Header and skip it for now. 175 if (out_size - *out_pos <= block->header_size) 176 return LZMA_BUF_ERROR; 177 178 const size_t out_start = *out_pos; 179 *out_pos += block->header_size; 180 181 // Limit out_size so that we stop encoding if the output would grow 182 // bigger than what uncompressed Block would be. 183 if (out_size - *out_pos > block->compressed_size) 184 out_size = *out_pos + block->compressed_size; 185 186 // TODO: In many common cases this could be optimized to use 187 // significantly less memory. 188 lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; 189 lzma_ret ret = lzma_raw_encoder_init( 190 &raw_encoder, allocator, block->filters); 191 192 if (ret == LZMA_OK) { 193 size_t in_pos = 0; 194 ret = raw_encoder.code(raw_encoder.coder, allocator, 195 in, &in_pos, in_size, out, out_pos, out_size, 196 LZMA_FINISH); 197 } 198 199 // NOTE: This needs to be run even if lzma_raw_encoder_init() failed. 200 lzma_next_end(&raw_encoder, allocator); 201 202 if (ret == LZMA_STREAM_END) { 203 // Compression was successful. Write the Block Header. 204 block->compressed_size 205 = *out_pos - (out_start + block->header_size); 206 ret = lzma_block_header_encode(block, out + out_start); 207 if (ret != LZMA_OK) 208 ret = LZMA_PROG_ERROR; 209 210 } else if (ret == LZMA_OK) { 211 // Output buffer became full. 212 ret = LZMA_BUF_ERROR; 213 } 214 215 // Reset *out_pos if something went wrong. 216 if (ret != LZMA_OK) 217 *out_pos = out_start; 218 219 return ret; 220 } 221 222 223 static lzma_ret 224 block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, 225 const uint8_t *in, size_t in_size, 226 uint8_t *out, size_t *out_pos, size_t out_size, 227 bool try_to_compress) 228 { 229 // Validate the arguments. 230 if (block == NULL || (in == NULL && in_size != 0) || out == NULL 231 || out_pos == NULL || *out_pos > out_size) 232 return LZMA_PROG_ERROR; 233 234 // The contents of the structure may depend on the version so 235 // check the version before validating the contents of *block. 236 if (block->version > 1) 237 return LZMA_OPTIONS_ERROR; 238 239 if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX 240 || (try_to_compress && block->filters == NULL)) 241 return LZMA_PROG_ERROR; 242 243 if (!lzma_check_is_supported(block->check)) 244 return LZMA_UNSUPPORTED_CHECK; 245 246 // Size of a Block has to be a multiple of four, so limit the size 247 // here already. This way we don't need to check it again when adding 248 // Block Padding. 249 out_size -= (out_size - *out_pos) & 3; 250 251 // Get the size of the Check field. 252 const size_t check_size = lzma_check_size(block->check); 253 assert(check_size != UINT32_MAX); 254 255 // Reserve space for the Check field. 256 if (out_size - *out_pos <= check_size) 257 return LZMA_BUF_ERROR; 258 259 out_size -= check_size; 260 261 // Initialize block->uncompressed_size and calculate the worst-case 262 // value for block->compressed_size. 263 block->uncompressed_size = in_size; 264 block->compressed_size = lzma2_bound(in_size); 265 if (block->compressed_size == 0) 266 return LZMA_DATA_ERROR; 267 268 // Do the actual compression. 269 lzma_ret ret = LZMA_BUF_ERROR; 270 if (try_to_compress) 271 ret = block_encode_normal(block, allocator, 272 in, in_size, out, out_pos, out_size); 273 274 if (ret != LZMA_OK) { 275 // If the error was something else than output buffer 276 // becoming full, return the error now. 277 if (ret != LZMA_BUF_ERROR) 278 return ret; 279 280 // The data was incompressible (at least with the options 281 // given to us) or the output buffer was too small. Use the 282 // uncompressed chunks of LZMA2 to wrap the data into a valid 283 // Block. If we haven't been given enough output space, even 284 // this may fail. 285 return_if_error(block_encode_uncompressed(block, in, in_size, 286 out, out_pos, out_size)); 287 } 288 289 assert(*out_pos <= out_size); 290 291 // Block Padding. No buffer overflow here, because we already adjusted 292 // out_size so that (out_size - out_start) is a multiple of four. 293 // Thus, if the buffer is full, the loop body can never run. 294 for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { 295 assert(*out_pos < out_size); 296 out[(*out_pos)++] = 0x00; 297 } 298 299 // If there's no Check field, we are done now. 300 if (check_size > 0) { 301 // Calculate the integrity check. We reserved space for 302 // the Check field earlier so we don't need to check for 303 // available output space here. 304 lzma_check_state check; 305 lzma_check_init(&check, block->check); 306 lzma_check_update(&check, block->check, in, in_size); 307 lzma_check_finish(&check, block->check); 308 309 memcpy(block->raw_check, check.buffer.u8, check_size); 310 memcpy(out + *out_pos, check.buffer.u8, check_size); 311 *out_pos += check_size; 312 } 313 314 return LZMA_OK; 315 } 316 317 318 extern LZMA_API(lzma_ret) 319 lzma_block_buffer_encode(lzma_block *block, const lzma_allocator *allocator, 320 const uint8_t *in, size_t in_size, 321 uint8_t *out, size_t *out_pos, size_t out_size) 322 { 323 return block_buffer_encode(block, allocator, 324 in, in_size, out, out_pos, out_size, true); 325 } 326 327 328 #ifdef HAVE_SYMBOL_VERSIONS_LINUX 329 // This is for compatibility with binaries linked against liblzma that 330 // has been patched with xz-5.2.2-compat-libs.patch from RHEL/CentOS 7. 331 LZMA_SYMVER_API("lzma_block_uncomp_encode@XZ_5.2.2", 332 lzma_ret, lzma_block_uncomp_encode_522)(lzma_block *block, 333 const uint8_t *in, size_t in_size, 334 uint8_t *out, size_t *out_pos, size_t out_size) 335 lzma_nothrow lzma_attr_warn_unused_result 336 __attribute__((__alias__("lzma_block_uncomp_encode_52"))); 337 338 LZMA_SYMVER_API("lzma_block_uncomp_encode@@XZ_5.2", 339 lzma_ret, lzma_block_uncomp_encode_52)(lzma_block *block, 340 const uint8_t *in, size_t in_size, 341 uint8_t *out, size_t *out_pos, size_t out_size) 342 lzma_nothrow lzma_attr_warn_unused_result; 343 344 #define lzma_block_uncomp_encode lzma_block_uncomp_encode_52 345 #endif 346 extern LZMA_API(lzma_ret) 347 lzma_block_uncomp_encode(lzma_block *block, 348 const uint8_t *in, size_t in_size, 349 uint8_t *out, size_t *out_pos, size_t out_size) 350 { 351 // It won't allocate any memory from heap so no need 352 // for lzma_allocator. 353 return block_buffer_encode(block, NULL, 354 in, in_size, out, out_pos, out_size, false); 355 } 356