Lines Matching +full:internal +full:- +full:mem
1 // SPDX-License-Identifier: BSD-3-Clause
3 * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
33 * Copyright (c) 2016-2018, Klara Inc.
34 * Copyright (c) 2016-2018, Allan Jude
35 * Copyright (c) 2018-2020, Sebastian Gottschall
36 * Copyright (c) 2019-2020, Michael Niewöhner
77 * LZ4 first-pass early abort verdict
82 * zstd-1 second-pass early abort verdict
157 void *mem; member
173 void *mem; member
212 /* Level map for converting ZFS internal levels to ZSTD levels and vice versa */
233 {-1, ZIO_ZSTD_LEVEL_FAST_1},
234 {-2, ZIO_ZSTD_LEVEL_FAST_2},
235 {-3, ZIO_ZSTD_LEVEL_FAST_3},
236 {-4, ZIO_ZSTD_LEVEL_FAST_4},
237 {-5, ZIO_ZSTD_LEVEL_FAST_5},
238 {-6, ZIO_ZSTD_LEVEL_FAST_6},
239 {-7, ZIO_ZSTD_LEVEL_FAST_7},
240 {-8, ZIO_ZSTD_LEVEL_FAST_8},
241 {-9, ZIO_ZSTD_LEVEL_FAST_9},
242 {-10, ZIO_ZSTD_LEVEL_FAST_10},
243 {-20, ZIO_ZSTD_LEVEL_FAST_20},
244 {-30, ZIO_ZSTD_LEVEL_FAST_30},
245 {-40, ZIO_ZSTD_LEVEL_FAST_40},
246 {-50, ZIO_ZSTD_LEVEL_FAST_50},
247 {-60, ZIO_ZSTD_LEVEL_FAST_60},
248 {-70, ZIO_ZSTD_LEVEL_FAST_70},
249 {-80, ZIO_ZSTD_LEVEL_FAST_80},
250 {-90, ZIO_ZSTD_LEVEL_FAST_90},
251 {-100, ZIO_ZSTD_LEVEL_FAST_100},
252 {-500, ZIO_ZSTD_LEVEL_FAST_500},
253 {-1000, ZIO_ZSTD_LEVEL_FAST_1000},
297 if (pool->mem && mutex_tryenter(&pool->barrier)) { in zstd_mempool_reap()
299 if (pool->mem && gethrestime_sec() > pool->timeout) { in zstd_mempool_reap()
300 vmem_free(pool->mem, pool->size); in zstd_mempool_reap()
302 ZSTDSTAT_SUB(zstd_stat_size, pool->size); in zstd_mempool_reap()
303 pool->mem = NULL; in zstd_mempool_reap()
304 pool->size = 0; in zstd_mempool_reap()
305 pool->timeout = 0; in zstd_mempool_reap()
307 mutex_exit(&pool->barrier); in zstd_mempool_reap()
330 struct zstd_kmem *mem = NULL; in zstd_mempool_alloc() local
348 if (mutex_tryenter(&pool->barrier)) { in zstd_mempool_alloc()
353 if (pool->mem && size <= pool->size) { in zstd_mempool_alloc()
354 pool->timeout = gethrestime_sec() + in zstd_mempool_alloc()
356 mem = pool->mem; in zstd_mempool_alloc()
357 return (mem); in zstd_mempool_alloc()
359 mutex_exit(&pool->barrier); in zstd_mempool_alloc()
373 if (mutex_tryenter(&pool->barrier)) { in zstd_mempool_alloc()
375 if (!pool->mem) { in zstd_mempool_alloc()
376 mem = vmem_alloc(size, KM_SLEEP); in zstd_mempool_alloc()
377 if (mem) { in zstd_mempool_alloc()
380 pool->mem = mem; in zstd_mempool_alloc()
381 pool->size = size; in zstd_mempool_alloc()
383 mem->pool = pool; in zstd_mempool_alloc()
384 mem->kmem_type = ZSTD_KMEM_POOL; in zstd_mempool_alloc()
385 mem->kmem_size = size; in zstd_mempool_alloc()
389 if (size <= pool->size) { in zstd_mempool_alloc()
391 pool->timeout = gethrestime_sec() + in zstd_mempool_alloc()
394 return (pool->mem); in zstd_mempool_alloc()
397 mutex_exit(&pool->barrier); in zstd_mempool_alloc()
405 if (!mem) { in zstd_mempool_alloc()
406 mem = vmem_alloc(size, KM_NOSLEEP); in zstd_mempool_alloc()
407 if (mem) { in zstd_mempool_alloc()
408 mem->pool = NULL; in zstd_mempool_alloc()
409 mem->kmem_type = ZSTD_KMEM_DEFAULT; in zstd_mempool_alloc()
410 mem->kmem_size = size; in zstd_mempool_alloc()
414 return (mem); in zstd_mempool_alloc()
421 mutex_exit(&z->pool->barrier); in zstd_mempool_free()
424 /* Convert ZFS internal enum to ZSTD level */
429 *zstd_level = zstd_levels[level - 1].zstd_level; in zstd_enum_to_level()
434 *zstd_level = zstd_levels[level - ZIO_ZSTD_LEVEL_FAST_1 in zstd_enum_to_level()
439 /* Invalid/unknown zfs compression enum - this should never happen. */ in zstd_enum_to_level()
469 * Out of kernel memory, gently fall through - this will disable in zfs_zstd_compress_impl()
491 hdr->data, in zfs_zstd_compress_impl()
492 d_len - sizeof (*hdr), in zfs_zstd_compress_impl()
518 hdr->c_len = BE_32(c_len); in zfs_zstd_compress_impl()
545 hdr->raw_version_level = BE_32(hdr->raw_version_level); in zfs_zstd_compress_impl()
562 * - Zeroth, if this is <= zstd-3, or < zstd_abort_size (currently in zfs_zstd_compress_buf()
566 * - First, we try LZ4 compression, and if it doesn't early abort, we in zfs_zstd_compress_buf()
568 * - Second, we try zstd-1 - if that errors out (usually, but not in zfs_zstd_compress_buf()
575 * savings versus no early abort, and all the zstd-fast levels are in zfs_zstd_compress_buf()
626 c_len = BE_32(hdr->c_len); in zfs_zstd_decompress_level_buf()
632 hdr_copy.raw_version_level = BE_32(hdr->raw_version_level); in zfs_zstd_decompress_level_buf()
670 result = ZSTD_decompressDCtx(dctx, d_start, d_len, hdr->data, c_len); in zfs_zstd_decompress_level_buf()
674 * Returns 0 on success (decompression function returned non-negative) in zfs_zstd_decompress_level_buf()
675 * and non-zero on failure (decompression function returned negative. in zfs_zstd_decompress_level_buf()
749 z->pool = NULL; in zstd_dctx_alloc()
765 z = zstd_dctx_fallback.mem; in zstd_dctx_alloc()
775 z->kmem_type = type; in zstd_dctx_alloc()
776 z->kmem_size = nbytes; in zstd_dctx_alloc()
785 struct zstd_kmem *z = (ptr - sizeof (struct zstd_kmem)); in zstd_free()
788 ASSERT3U(z->kmem_type, <, ZSTD_KMEM_COUNT); in zstd_free()
789 ASSERT3U(z->kmem_type, >, ZSTD_KMEM_UNKNOWN); in zstd_free()
791 type = z->kmem_type; in zstd_free()
794 vmem_free(z, z->kmem_size); in zstd_free()
809 create_fallback_mem(struct zstd_fallback_mem *mem, size_t size) in create_fallback_mem() argument
811 mem->mem_size = size; in create_fallback_mem()
812 mem->mem = vmem_zalloc(mem->mem_size, KM_SLEEP); in create_fallback_mem()
813 mutex_init(&mem->barrier, NULL, MUTEX_DEFAULT, NULL); in create_fallback_mem()
833 /* Initialize zstd-related memory handling */
854 mutex_destroy(&pool->barrier); in release_pool()
855 vmem_free(pool->mem, pool->size); in release_pool()
856 pool->mem = NULL; in release_pool()
857 pool->size = 0; in release_pool()
882 * Short-circuit if there are no buffers to begin with. in zfs_zstd_cache_reap_now()
907 zstd_ksp->ks_data = &zstd_stats; in zstd_init()
910 zstd_ksp->ks_update = kstat_zstd_update; in zstd_init()
927 vmem_free(zstd_dctx_fallback.mem, zstd_dctx_fallback.mem_size); in zstd_fini()