1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/kernel.h> 4 #include <linux/slab.h> 5 #include <linux/vmalloc.h> 6 #include <linux/zstd.h> 7 8 #include "backend_zstd.h" 9 10 struct zstd_ctx { 11 zstd_cctx *cctx; 12 zstd_dctx *dctx; 13 void *cctx_mem; 14 void *dctx_mem; 15 }; 16 17 struct zstd_params { 18 zstd_custom_mem custom_mem; 19 zstd_cdict *cdict; 20 zstd_ddict *ddict; 21 zstd_parameters cprm; 22 }; 23 24 /* 25 * For C/D dictionaries we need to provide zstd with zstd_custom_mem, 26 * which zstd uses internally to allocate/free memory when needed. 27 * 28 * This means that allocator.customAlloc() can be called from zcomp_compress() 29 * under local-lock (per-CPU compression stream), in which case we must use 30 * GFP_ATOMIC. 31 * 32 * Another complication here is that we can be configured as a swap device. 33 */ 34 static void *zstd_custom_alloc(void *opaque, size_t size) 35 { 36 if (!preemptible()) 37 return kvzalloc(size, GFP_ATOMIC); 38 39 return kvzalloc(size, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN); 40 } 41 42 static void zstd_custom_free(void *opaque, void *address) 43 { 44 kvfree(address); 45 } 46 47 static void zstd_release_params(struct zcomp_params *params) 48 { 49 struct zstd_params *zp = params->drv_data; 50 51 params->drv_data = NULL; 52 if (!zp) 53 return; 54 55 zstd_free_cdict(zp->cdict); 56 zstd_free_ddict(zp->ddict); 57 kfree(zp); 58 } 59 60 static int zstd_setup_params(struct zcomp_params *params) 61 { 62 zstd_compression_parameters prm; 63 struct zstd_params *zp; 64 65 zp = kzalloc(sizeof(*zp), GFP_KERNEL); 66 if (!zp) 67 return -ENOMEM; 68 69 params->drv_data = zp; 70 if (params->level == ZCOMP_PARAM_NO_LEVEL) 71 params->level = zstd_default_clevel(); 72 73 zp->cprm = zstd_get_params(params->level, PAGE_SIZE); 74 75 zp->custom_mem.customAlloc = zstd_custom_alloc; 76 zp->custom_mem.customFree = zstd_custom_free; 77 78 prm = zstd_get_cparams(params->level, PAGE_SIZE, 79 params->dict_sz); 80 81 zp->cdict = zstd_create_cdict_byreference(params->dict, 82 params->dict_sz, 83 prm, 84 zp->custom_mem); 85 if (!zp->cdict) 86 goto error; 87 88 zp->ddict = zstd_create_ddict_byreference(params->dict, 89 params->dict_sz, 90 zp->custom_mem); 91 if (!zp->ddict) 92 goto error; 93 94 return 0; 95 96 error: 97 zstd_release_params(params); 98 return -EINVAL; 99 } 100 101 static void zstd_destroy(struct zcomp_ctx *ctx) 102 { 103 struct zstd_ctx *zctx = ctx->context; 104 105 if (!zctx) 106 return; 107 108 /* 109 * If ->cctx_mem and ->dctx_mem were allocated then we didn't use 110 * C/D dictionary and ->cctx / ->dctx were "embedded" into these 111 * buffers. 112 * 113 * If otherwise then we need to explicitly release ->cctx / ->dctx. 114 */ 115 if (zctx->cctx_mem) 116 vfree(zctx->cctx_mem); 117 else 118 zstd_free_cctx(zctx->cctx); 119 120 if (zctx->dctx_mem) 121 vfree(zctx->dctx_mem); 122 else 123 zstd_free_dctx(zctx->dctx); 124 125 kfree(zctx); 126 } 127 128 static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx) 129 { 130 struct zstd_ctx *zctx; 131 zstd_parameters prm; 132 size_t sz; 133 134 zctx = kzalloc(sizeof(*zctx), GFP_KERNEL); 135 if (!zctx) 136 return -ENOMEM; 137 138 ctx->context = zctx; 139 if (params->dict_sz == 0) { 140 prm = zstd_get_params(params->level, PAGE_SIZE); 141 sz = zstd_cctx_workspace_bound(&prm.cParams); 142 zctx->cctx_mem = vzalloc(sz); 143 if (!zctx->cctx_mem) 144 goto error; 145 146 zctx->cctx = zstd_init_cctx(zctx->cctx_mem, sz); 147 if (!zctx->cctx) 148 goto error; 149 150 sz = zstd_dctx_workspace_bound(); 151 zctx->dctx_mem = vzalloc(sz); 152 if (!zctx->dctx_mem) 153 goto error; 154 155 zctx->dctx = zstd_init_dctx(zctx->dctx_mem, sz); 156 if (!zctx->dctx) 157 goto error; 158 } else { 159 struct zstd_params *zp = params->drv_data; 160 161 zctx->cctx = zstd_create_cctx_advanced(zp->custom_mem); 162 if (!zctx->cctx) 163 goto error; 164 165 zctx->dctx = zstd_create_dctx_advanced(zp->custom_mem); 166 if (!zctx->dctx) 167 goto error; 168 } 169 170 return 0; 171 172 error: 173 zstd_release_params(params); 174 zstd_destroy(ctx); 175 return -EINVAL; 176 } 177 178 static int zstd_compress(struct zcomp_params *params, struct zcomp_ctx *ctx, 179 struct zcomp_req *req) 180 { 181 struct zstd_params *zp = params->drv_data; 182 struct zstd_ctx *zctx = ctx->context; 183 size_t ret; 184 185 if (params->dict_sz == 0) 186 ret = zstd_compress_cctx(zctx->cctx, req->dst, req->dst_len, 187 req->src, req->src_len, &zp->cprm); 188 else 189 ret = zstd_compress_using_cdict(zctx->cctx, req->dst, 190 req->dst_len, req->src, 191 req->src_len, 192 zp->cdict); 193 if (zstd_is_error(ret)) 194 return -EINVAL; 195 req->dst_len = ret; 196 return 0; 197 } 198 199 static int zstd_decompress(struct zcomp_params *params, struct zcomp_ctx *ctx, 200 struct zcomp_req *req) 201 { 202 struct zstd_params *zp = params->drv_data; 203 struct zstd_ctx *zctx = ctx->context; 204 size_t ret; 205 206 if (params->dict_sz == 0) 207 ret = zstd_decompress_dctx(zctx->dctx, req->dst, req->dst_len, 208 req->src, req->src_len); 209 else 210 ret = zstd_decompress_using_ddict(zctx->dctx, req->dst, 211 req->dst_len, req->src, 212 req->src_len, zp->ddict); 213 if (zstd_is_error(ret)) 214 return -EINVAL; 215 return 0; 216 } 217 218 const struct zcomp_ops backend_zstd = { 219 .compress = zstd_compress, 220 .decompress = zstd_decompress, 221 .create_ctx = zstd_create, 222 .destroy_ctx = zstd_destroy, 223 .setup_params = zstd_setup_params, 224 .release_params = zstd_release_params, 225 .name = "zstd", 226 }; 227