1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Cryptographic API. 4 * 5 * Copyright (c) 2017-present, Facebook, Inc. 6 */ 7 #include <linux/crypto.h> 8 #include <linux/init.h> 9 #include <linux/interrupt.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/net.h> 13 #include <linux/overflow.h> 14 #include <linux/vmalloc.h> 15 #include <linux/zstd.h> 16 #include <crypto/internal/acompress.h> 17 #include <crypto/scatterwalk.h> 18 19 20 #define ZSTD_DEF_LEVEL 3 21 #define ZSTD_MAX_WINDOWLOG 18 22 #define ZSTD_MAX_SIZE BIT(ZSTD_MAX_WINDOWLOG) 23 24 struct zstd_ctx { 25 zstd_cctx *cctx; 26 zstd_dctx *dctx; 27 size_t wksp_size; 28 zstd_parameters params; 29 u8 wksp[] __aligned(8) __counted_by(wksp_size); 30 }; 31 32 static DEFINE_MUTEX(zstd_stream_lock); 33 34 static void *zstd_alloc_stream(void) 35 { 36 zstd_parameters params; 37 struct zstd_ctx *ctx; 38 size_t wksp_size; 39 40 params = zstd_get_params(ZSTD_DEF_LEVEL, ZSTD_MAX_SIZE); 41 42 wksp_size = max(zstd_cstream_workspace_bound(¶ms.cParams), 43 zstd_dstream_workspace_bound(ZSTD_MAX_SIZE)); 44 if (!wksp_size) 45 return ERR_PTR(-EINVAL); 46 47 ctx = kvmalloc_flex(*ctx, wksp, wksp_size); 48 if (!ctx) 49 return ERR_PTR(-ENOMEM); 50 51 ctx->params = params; 52 ctx->wksp_size = wksp_size; 53 54 return ctx; 55 } 56 57 static void zstd_free_stream(void *ctx) 58 { 59 kvfree(ctx); 60 } 61 62 static struct crypto_acomp_streams zstd_streams = { 63 .alloc_ctx = zstd_alloc_stream, 64 .free_ctx = zstd_free_stream, 65 }; 66 67 static int zstd_init(struct crypto_acomp *acomp_tfm) 68 { 69 int ret = 0; 70 71 mutex_lock(&zstd_stream_lock); 72 ret = crypto_acomp_alloc_streams(&zstd_streams); 73 mutex_unlock(&zstd_stream_lock); 74 75 return ret; 76 } 77 78 static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx, 79 const void *src, void *dst, unsigned int *dlen) 80 { 81 size_t out_len; 82 83 ctx->cctx = zstd_init_cctx(ctx->wksp, ctx->wksp_size); 84 if (!ctx->cctx) 85 return -EINVAL; 86 87 out_len = zstd_compress_cctx(ctx->cctx, dst, req->dlen, src, req->slen, 88 &ctx->params); 89 if (zstd_is_error(out_len)) 90 return -EINVAL; 91 92 *dlen = out_len; 93 94 return 0; 95 } 96 97 static int zstd_compress(struct acomp_req *req) 98 { 99 bool stream_initialized = false; 100 struct crypto_acomp_stream *s; 101 unsigned int pos, scur, dcur; 102 unsigned int total_out = 0; 103 bool data_available = true; 104 zstd_out_buffer outbuf; 105 struct acomp_walk walk; 106 zstd_in_buffer inbuf; 107 struct zstd_ctx *ctx; 108 size_t pending_bytes; 109 size_t num_bytes; 110 int ret; 111 112 s = crypto_acomp_lock_stream_bh(&zstd_streams); 113 ctx = s->ctx; 114 115 ret = acomp_walk_virt(&walk, req, true); 116 if (ret) 117 goto out; 118 119 do { 120 dcur = acomp_walk_next_dst(&walk); 121 if (!dcur) { 122 ret = -ENOSPC; 123 goto out; 124 } 125 126 outbuf.pos = 0; 127 outbuf.dst = (u8 *)walk.dst.virt.addr; 128 outbuf.size = dcur; 129 130 do { 131 scur = acomp_walk_next_src(&walk); 132 if (dcur == req->dlen && scur == req->slen) { 133 ret = zstd_compress_one(req, ctx, walk.src.virt.addr, 134 walk.dst.virt.addr, &total_out); 135 acomp_walk_done_src(&walk, scur); 136 acomp_walk_done_dst(&walk, dcur); 137 goto out; 138 } 139 140 if (!stream_initialized) { 141 ctx->cctx = zstd_init_cstream(&ctx->params, 0, 142 ctx->wksp, ctx->wksp_size); 143 if (!ctx->cctx) { 144 /* Release in the reverse of the map order. */ 145 acomp_walk_done_src(&walk, 0); 146 acomp_walk_done_dst(&walk, 0); 147 ret = -EINVAL; 148 goto out; 149 } 150 stream_initialized = true; 151 } 152 153 if (scur) { 154 inbuf.pos = 0; 155 inbuf.src = walk.src.virt.addr; 156 inbuf.size = scur; 157 } else { 158 data_available = false; 159 break; 160 } 161 162 num_bytes = zstd_compress_stream(ctx->cctx, &outbuf, &inbuf); 163 if (ZSTD_isError(num_bytes)) { 164 ret = -EIO; 165 goto out; 166 } 167 168 pending_bytes = zstd_flush_stream(ctx->cctx, &outbuf); 169 if (ZSTD_isError(pending_bytes)) { 170 ret = -EIO; 171 goto out; 172 } 173 acomp_walk_done_src(&walk, inbuf.pos); 174 } while (dcur != outbuf.pos); 175 176 total_out += outbuf.pos; 177 acomp_walk_done_dst(&walk, dcur); 178 } while (data_available); 179 180 pos = outbuf.pos; 181 num_bytes = zstd_end_stream(ctx->cctx, &outbuf); 182 if (ZSTD_isError(num_bytes)) 183 ret = -EIO; 184 else 185 total_out += (outbuf.pos - pos); 186 187 out: 188 if (ret) 189 req->dlen = 0; 190 else 191 req->dlen = total_out; 192 193 crypto_acomp_unlock_stream_bh(s); 194 195 return ret; 196 } 197 198 static int zstd_decompress_one(struct acomp_req *req, struct zstd_ctx *ctx, 199 const void *src, void *dst, unsigned int *dlen) 200 { 201 size_t out_len; 202 203 ctx->dctx = zstd_init_dctx(ctx->wksp, ctx->wksp_size); 204 if (!ctx->dctx) 205 return -EINVAL; 206 207 out_len = zstd_decompress_dctx(ctx->dctx, dst, req->dlen, src, req->slen); 208 if (zstd_is_error(out_len)) 209 return -EINVAL; 210 211 *dlen = out_len; 212 213 return 0; 214 } 215 216 static int zstd_decompress(struct acomp_req *req) 217 { 218 bool stream_initialized = false; 219 struct crypto_acomp_stream *s; 220 unsigned int total_out = 0; 221 unsigned int scur, dcur; 222 zstd_out_buffer outbuf; 223 struct acomp_walk walk; 224 zstd_in_buffer inbuf; 225 struct zstd_ctx *ctx; 226 size_t pending_bytes; 227 int ret; 228 229 s = crypto_acomp_lock_stream_bh(&zstd_streams); 230 ctx = s->ctx; 231 232 ret = acomp_walk_virt(&walk, req, true); 233 if (ret) 234 goto out; 235 236 do { 237 scur = acomp_walk_next_src(&walk); 238 if (scur) { 239 inbuf.pos = 0; 240 inbuf.size = scur; 241 inbuf.src = walk.src.virt.addr; 242 } else { 243 break; 244 } 245 246 do { 247 dcur = acomp_walk_next_dst(&walk); 248 if (dcur == req->dlen && scur == req->slen) { 249 ret = zstd_decompress_one(req, ctx, walk.src.virt.addr, 250 walk.dst.virt.addr, &total_out); 251 acomp_walk_done_dst(&walk, dcur); 252 acomp_walk_done_src(&walk, scur); 253 goto out; 254 } 255 256 if (!dcur) { 257 ret = -ENOSPC; 258 goto out; 259 } 260 261 if (!stream_initialized) { 262 ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, ctx->wksp, 263 ctx->wksp_size); 264 if (!ctx->dctx) { 265 /* Release in the reverse of the map order. */ 266 acomp_walk_done_dst(&walk, 0); 267 acomp_walk_done_src(&walk, 0); 268 ret = -EINVAL; 269 goto out; 270 } 271 stream_initialized = true; 272 } 273 274 outbuf.pos = 0; 275 outbuf.dst = (u8 *)walk.dst.virt.addr; 276 outbuf.size = dcur; 277 278 pending_bytes = zstd_decompress_stream(ctx->dctx, &outbuf, &inbuf); 279 if (ZSTD_isError(pending_bytes)) { 280 ret = -EIO; 281 goto out; 282 } 283 284 total_out += outbuf.pos; 285 286 acomp_walk_done_dst(&walk, outbuf.pos); 287 } while (inbuf.pos != scur); 288 289 acomp_walk_done_src(&walk, scur); 290 } while (ret == 0); 291 292 out: 293 if (ret) 294 req->dlen = 0; 295 else 296 req->dlen = total_out; 297 298 crypto_acomp_unlock_stream_bh(s); 299 300 return ret; 301 } 302 303 static struct acomp_alg zstd_acomp = { 304 .base = { 305 .cra_name = "zstd", 306 .cra_driver_name = "zstd-generic", 307 .cra_flags = CRYPTO_ALG_REQ_VIRT, 308 .cra_module = THIS_MODULE, 309 }, 310 .init = zstd_init, 311 .compress = zstd_compress, 312 .decompress = zstd_decompress, 313 }; 314 315 static int __init zstd_mod_init(void) 316 { 317 return crypto_register_acomp(&zstd_acomp); 318 } 319 320 static void __exit zstd_mod_fini(void) 321 { 322 crypto_unregister_acomp(&zstd_acomp); 323 crypto_acomp_free_streams(&zstd_streams); 324 } 325 326 module_init(zstd_mod_init); 327 module_exit(zstd_mod_fini); 328 329 MODULE_LICENSE("GPL"); 330 MODULE_DESCRIPTION("Zstd Compression Algorithm"); 331 MODULE_ALIAS_CRYPTO("zstd"); 332