1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Synchronous Compression operations 4 * 5 * Copyright 2015 LG Electronics Inc. 6 * Copyright (c) 2016, Intel Corporation 7 * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com> 8 */ 9 10 #include <crypto/internal/scompress.h> 11 #include <crypto/scatterwalk.h> 12 #include <linux/cpumask.h> 13 #include <linux/cryptouser.h> 14 #include <linux/err.h> 15 #include <linux/highmem.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/overflow.h> 19 #include <linux/scatterlist.h> 20 #include <linux/seq_file.h> 21 #include <linux/slab.h> 22 #include <linux/string.h> 23 #include <linux/workqueue.h> 24 #include <net/netlink.h> 25 26 #include "compress.h" 27 28 struct scomp_scratch { 29 spinlock_t lock; 30 union { 31 void *src __guarded_by(&lock); 32 unsigned long saddr __guarded_by(&lock); 33 }; 34 }; 35 36 static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { 37 .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock), 38 }; 39 40 static const struct crypto_type crypto_scomp_type; 41 static DEFINE_MUTEX(scomp_lock); 42 static int scomp_scratch_users __guarded_by(&scomp_lock); 43 44 static cpumask_t scomp_scratch_want; 45 static void scomp_scratch_workfn(struct work_struct *work); 46 static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn); 47 48 static int __maybe_unused crypto_scomp_report( 49 struct sk_buff *skb, struct crypto_alg *alg) 50 { 51 struct crypto_report_comp rscomp; 52 53 memset(&rscomp, 0, sizeof(rscomp)); 54 55 strscpy(rscomp.type, "scomp", sizeof(rscomp.type)); 56 57 return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, 58 sizeof(rscomp), &rscomp); 59 } 60 61 static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 62 __maybe_unused; 63 64 static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 65 { 66 seq_puts(m, "type : scomp\n"); 67 } 68 69 static void crypto_scomp_free_scratches(void) 70 __context_unsafe(/* frees @scratch */) 71 { 72 struct scomp_scratch *scratch; 73 int i; 74 75 for_each_possible_cpu(i) { 76 scratch = per_cpu_ptr(&scomp_scratch, i); 77 78 free_page(scratch->saddr); 79 scratch->src = NULL; 80 } 81 } 82 83 static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu) 84 { 85 int node = cpu_to_node(cpu); 86 struct page *page; 87 88 page = alloc_pages_node(node, GFP_KERNEL, 0); 89 if (!page) 90 return -ENOMEM; 91 spin_lock_bh(&scratch->lock); 92 scratch->src = page_address(page); 93 spin_unlock_bh(&scratch->lock); 94 return 0; 95 } 96 97 static void scomp_scratch_workfn(struct work_struct *work) 98 { 99 int cpu; 100 101 for_each_cpu(cpu, &scomp_scratch_want) { 102 struct scomp_scratch *scratch; 103 104 scratch = per_cpu_ptr(&scomp_scratch, cpu); 105 if (context_unsafe(scratch->src)) 106 continue; 107 if (scomp_alloc_scratch(scratch, cpu)) 108 break; 109 110 cpumask_clear_cpu(cpu, &scomp_scratch_want); 111 } 112 } 113 114 static int crypto_scomp_alloc_scratches(void) 115 __context_unsafe(/* allocates @scratch */) 116 { 117 unsigned int i = cpumask_first(cpu_possible_mask); 118 struct scomp_scratch *scratch; 119 120 scratch = per_cpu_ptr(&scomp_scratch, i); 121 return scomp_alloc_scratch(scratch, i); 122 } 123 124 static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) 125 { 126 struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); 127 int ret = 0; 128 129 mutex_lock(&scomp_lock); 130 ret = crypto_acomp_alloc_streams(&alg->streams); 131 if (ret) 132 goto unlock; 133 if (!scomp_scratch_users++) { 134 ret = crypto_scomp_alloc_scratches(); 135 if (ret) 136 scomp_scratch_users--; 137 } 138 unlock: 139 mutex_unlock(&scomp_lock); 140 141 return ret; 142 } 143 144 #define scomp_lock_scratch(...) __acquire_ret(_scomp_lock_scratch(__VA_ARGS__), &__ret->lock) 145 static struct scomp_scratch *_scomp_lock_scratch(void) __acquires_ret 146 { 147 int cpu = raw_smp_processor_id(); 148 struct scomp_scratch *scratch; 149 150 scratch = per_cpu_ptr(&scomp_scratch, cpu); 151 spin_lock(&scratch->lock); 152 if (likely(scratch->src)) 153 return scratch; 154 spin_unlock(&scratch->lock); 155 156 cpumask_set_cpu(cpu, &scomp_scratch_want); 157 schedule_work(&scomp_scratch_work); 158 159 scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask)); 160 spin_lock(&scratch->lock); 161 return scratch; 162 } 163 164 static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) 165 __releases(&scratch->lock) 166 { 167 spin_unlock(&scratch->lock); 168 } 169 170 static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) 171 { 172 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); 173 struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); 174 bool src_isvirt = acomp_request_src_isvirt(req); 175 bool dst_isvirt = acomp_request_dst_isvirt(req); 176 struct crypto_scomp *scomp = *tfm_ctx; 177 unsigned int slen = req->slen; 178 unsigned int dlen = req->dlen; 179 struct page *spage, *dpage; 180 unsigned int n; 181 const u8 *src; 182 size_t soff; 183 size_t doff; 184 u8 *dst; 185 int ret; 186 187 if (!req->src || !slen) 188 return -EINVAL; 189 190 if (!req->dst || !dlen) 191 return -EINVAL; 192 193 if (dst_isvirt) 194 dst = req->dvirt; 195 else { 196 if (dlen <= req->dst->length) { 197 dpage = sg_page(req->dst); 198 doff = req->dst->offset; 199 } else 200 return -ENOSYS; 201 202 dpage += doff / PAGE_SIZE; 203 doff = offset_in_page(doff); 204 205 n = (dlen - 1) / PAGE_SIZE; 206 n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE; 207 if (PageHighMem(dpage + n) && 208 size_add(doff, dlen) > PAGE_SIZE) 209 return -ENOSYS; 210 dst = kmap_local_page(dpage) + doff; 211 } 212 213 if (src_isvirt) 214 src = req->svirt; 215 else { 216 src = NULL; 217 do { 218 if (slen <= req->src->length) { 219 spage = sg_page(req->src); 220 soff = req->src->offset; 221 } else 222 break; 223 224 spage = spage + soff / PAGE_SIZE; 225 soff = offset_in_page(soff); 226 227 n = (slen - 1) / PAGE_SIZE; 228 n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE; 229 if (PageHighMem(spage + n) && 230 size_add(soff, slen) > PAGE_SIZE) 231 break; 232 src = kmap_local_page(spage) + soff; 233 } while (0); 234 } 235 236 struct crypto_acomp_stream *stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); 237 238 if (!src_isvirt && !src) { 239 struct scomp_scratch *scratch = scomp_lock_scratch(); 240 const u8 *src = scratch->src; 241 242 memcpy_from_sglist(scratch->src, req->src, 0, slen); 243 244 if (dir) 245 ret = crypto_scomp_compress(scomp, src, slen, 246 dst, &dlen, stream->ctx); 247 else 248 ret = crypto_scomp_decompress(scomp, src, slen, 249 dst, &dlen, stream->ctx); 250 251 scomp_unlock_scratch(scratch); 252 } else if (dir) 253 ret = crypto_scomp_compress(scomp, src, slen, 254 dst, &dlen, stream->ctx); 255 else 256 ret = crypto_scomp_decompress(scomp, src, slen, 257 dst, &dlen, stream->ctx); 258 259 crypto_acomp_unlock_stream_bh(stream); 260 261 req->dlen = dlen; 262 263 if (!src_isvirt && src) 264 kunmap_local(src); 265 if (!dst_isvirt) { 266 kunmap_local(dst); 267 dlen += doff; 268 for (;;) { 269 flush_dcache_page(dpage); 270 if (dlen <= PAGE_SIZE) 271 break; 272 dlen -= PAGE_SIZE; 273 dpage++; 274 } 275 } 276 277 return ret; 278 } 279 280 static int scomp_acomp_compress(struct acomp_req *req) 281 { 282 return scomp_acomp_comp_decomp(req, 1); 283 } 284 285 static int scomp_acomp_decompress(struct acomp_req *req) 286 { 287 return scomp_acomp_comp_decomp(req, 0); 288 } 289 290 static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) 291 { 292 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); 293 294 crypto_free_scomp(*ctx); 295 296 flush_work(&scomp_scratch_work); 297 mutex_lock(&scomp_lock); 298 if (!--scomp_scratch_users) 299 crypto_scomp_free_scratches(); 300 mutex_unlock(&scomp_lock); 301 } 302 303 int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) 304 { 305 struct crypto_alg *calg = tfm->__crt_alg; 306 struct crypto_acomp *crt = __crypto_acomp_tfm(tfm); 307 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); 308 struct crypto_scomp *scomp; 309 310 if (!crypto_mod_get(calg)) 311 return -EAGAIN; 312 313 scomp = crypto_create_tfm(calg, &crypto_scomp_type); 314 if (IS_ERR(scomp)) { 315 crypto_mod_put(calg); 316 return PTR_ERR(scomp); 317 } 318 319 *ctx = scomp; 320 tfm->exit = crypto_exit_scomp_ops_async; 321 322 crt->compress = scomp_acomp_compress; 323 crt->decompress = scomp_acomp_decompress; 324 325 return 0; 326 } 327 328 static void crypto_scomp_destroy(struct crypto_alg *alg) 329 { 330 struct scomp_alg *scomp = __crypto_scomp_alg(alg); 331 332 crypto_acomp_free_streams(&scomp->streams); 333 } 334 335 static const struct crypto_type crypto_scomp_type = { 336 .extsize = crypto_alg_extsize, 337 .init_tfm = crypto_scomp_init_tfm, 338 .destroy = crypto_scomp_destroy, 339 #ifdef CONFIG_PROC_FS 340 .show = crypto_scomp_show, 341 #endif 342 #if IS_ENABLED(CONFIG_CRYPTO_USER) 343 .report = crypto_scomp_report, 344 #endif 345 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 346 .maskset = CRYPTO_ALG_TYPE_MASK, 347 .type = CRYPTO_ALG_TYPE_SCOMPRESS, 348 .tfmsize = offsetof(struct crypto_scomp, base), 349 .algsize = offsetof(struct scomp_alg, base), 350 }; 351 352 static void scomp_prepare_alg(struct scomp_alg *alg) 353 { 354 struct crypto_alg *base = &alg->calg.base; 355 356 comp_prepare_alg(&alg->calg); 357 358 base->cra_flags |= CRYPTO_ALG_REQ_VIRT; 359 } 360 361 int crypto_register_scomp(struct scomp_alg *alg) 362 { 363 struct crypto_alg *base = &alg->calg.base; 364 365 scomp_prepare_alg(alg); 366 367 base->cra_type = &crypto_scomp_type; 368 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; 369 370 return crypto_register_alg(base); 371 } 372 EXPORT_SYMBOL_GPL(crypto_register_scomp); 373 374 void crypto_unregister_scomp(struct scomp_alg *alg) 375 { 376 crypto_unregister_alg(&alg->base); 377 } 378 EXPORT_SYMBOL_GPL(crypto_unregister_scomp); 379 380 int crypto_register_scomps(struct scomp_alg *algs, int count) 381 { 382 int i, ret; 383 384 for (i = 0; i < count; i++) { 385 ret = crypto_register_scomp(&algs[i]); 386 if (ret) 387 goto err; 388 } 389 390 return 0; 391 392 err: 393 for (--i; i >= 0; --i) 394 crypto_unregister_scomp(&algs[i]); 395 396 return ret; 397 } 398 EXPORT_SYMBOL_GPL(crypto_register_scomps); 399 400 void crypto_unregister_scomps(struct scomp_alg *algs, int count) 401 { 402 int i; 403 404 for (i = count - 1; i >= 0; --i) 405 crypto_unregister_scomp(&algs[i]); 406 } 407 EXPORT_SYMBOL_GPL(crypto_unregister_scomps); 408 409 MODULE_LICENSE("GPL"); 410 MODULE_DESCRIPTION("Synchronous compression type"); 411