1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Synchronous Compression operations 4 * 5 * Copyright 2015 LG Electronics Inc. 6 * Copyright (c) 2016, Intel Corporation 7 * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com> 8 */ 9 10 #include <crypto/internal/scompress.h> 11 #include <crypto/scatterwalk.h> 12 #include <linux/cpumask.h> 13 #include <linux/cryptouser.h> 14 #include <linux/err.h> 15 #include <linux/highmem.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/overflow.h> 19 #include <linux/scatterlist.h> 20 #include <linux/seq_file.h> 21 #include <linux/slab.h> 22 #include <linux/string.h> 23 #include <linux/workqueue.h> 24 #include <net/netlink.h> 25 26 #include "compress.h" 27 28 struct scomp_scratch { 29 spinlock_t lock; 30 union { 31 void *src __guarded_by(&lock); 32 unsigned long saddr __guarded_by(&lock); 33 }; 34 }; 35 36 static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { 37 .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock), 38 }; 39 40 static const struct crypto_type crypto_scomp_type; 41 static DEFINE_MUTEX(scomp_lock); 42 static int scomp_scratch_users __guarded_by(&scomp_lock); 43 44 static cpumask_t scomp_scratch_want; 45 static void scomp_scratch_workfn(struct work_struct *work); 46 static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn); 47 48 static int __maybe_unused crypto_scomp_report( 49 struct sk_buff *skb, struct crypto_alg *alg) 50 { 51 struct crypto_report_comp rscomp = { 52 .type = "scomp", 53 }; 54 55 return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, 56 sizeof(rscomp), &rscomp); 57 } 58 59 static void __maybe_unused crypto_scomp_show(struct seq_file *m, 60 struct crypto_alg *alg) 61 { 62 seq_puts(m, "type : scomp\n"); 63 } 64 65 static void crypto_scomp_free_scratches(void) 66 __context_unsafe(/* frees @scratch */) 67 { 68 struct scomp_scratch *scratch; 69 int i; 70 71 for_each_possible_cpu(i) { 72 scratch = per_cpu_ptr(&scomp_scratch, i); 73 74 free_page(scratch->saddr); 75 scratch->src = NULL; 76 } 77 } 78 79 static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu) 80 { 81 int node = cpu_to_node(cpu); 82 struct page *page; 83 84 page = alloc_pages_node(node, GFP_KERNEL, 0); 85 if (!page) 86 return -ENOMEM; 87 spin_lock_bh(&scratch->lock); 88 scratch->src = page_address(page); 89 spin_unlock_bh(&scratch->lock); 90 return 0; 91 } 92 93 static void scomp_scratch_workfn(struct work_struct *work) 94 { 95 int cpu; 96 97 for_each_cpu(cpu, &scomp_scratch_want) { 98 struct scomp_scratch *scratch; 99 100 scratch = per_cpu_ptr(&scomp_scratch, cpu); 101 if (context_unsafe(scratch->src)) 102 continue; 103 if (scomp_alloc_scratch(scratch, cpu)) 104 break; 105 106 cpumask_clear_cpu(cpu, &scomp_scratch_want); 107 } 108 } 109 110 static int crypto_scomp_alloc_scratches(void) 111 __context_unsafe(/* allocates @scratch */) 112 { 113 unsigned int i = cpumask_first(cpu_possible_mask); 114 struct scomp_scratch *scratch; 115 116 scratch = per_cpu_ptr(&scomp_scratch, i); 117 return scomp_alloc_scratch(scratch, i); 118 } 119 120 static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) 121 { 122 struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); 123 int ret = 0; 124 125 mutex_lock(&scomp_lock); 126 ret = crypto_acomp_alloc_streams(&alg->streams); 127 if (ret) 128 goto unlock; 129 if (!scomp_scratch_users++) { 130 ret = crypto_scomp_alloc_scratches(); 131 if (ret) 132 scomp_scratch_users--; 133 } 134 unlock: 135 mutex_unlock(&scomp_lock); 136 137 return ret; 138 } 139 140 #define scomp_lock_scratch(...) __acquire_ret(_scomp_lock_scratch(__VA_ARGS__), &__ret->lock) 141 static struct scomp_scratch *_scomp_lock_scratch(void) __acquires_ret 142 { 143 int cpu = raw_smp_processor_id(); 144 struct scomp_scratch *scratch; 145 146 scratch = per_cpu_ptr(&scomp_scratch, cpu); 147 spin_lock(&scratch->lock); 148 if (likely(scratch->src)) 149 return scratch; 150 spin_unlock(&scratch->lock); 151 152 cpumask_set_cpu(cpu, &scomp_scratch_want); 153 schedule_work(&scomp_scratch_work); 154 155 scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask)); 156 spin_lock(&scratch->lock); 157 return scratch; 158 } 159 160 static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) 161 __releases(&scratch->lock) 162 { 163 spin_unlock(&scratch->lock); 164 } 165 166 static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) 167 { 168 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); 169 struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); 170 bool src_isvirt = acomp_request_src_isvirt(req); 171 bool dst_isvirt = acomp_request_dst_isvirt(req); 172 struct crypto_scomp *scomp = *tfm_ctx; 173 unsigned int slen = req->slen; 174 unsigned int dlen = req->dlen; 175 struct page *spage, *dpage; 176 unsigned int n; 177 const u8 *src; 178 size_t soff; 179 size_t doff; 180 u8 *dst; 181 int ret; 182 183 if (!req->src || !slen) 184 return -EINVAL; 185 186 if (!req->dst || !dlen) 187 return -EINVAL; 188 189 if (dst_isvirt) 190 dst = req->dvirt; 191 else { 192 if (dlen <= req->dst->length) { 193 dpage = sg_page(req->dst); 194 doff = req->dst->offset; 195 } else 196 return -ENOSYS; 197 198 dpage += doff / PAGE_SIZE; 199 doff = offset_in_page(doff); 200 201 n = (dlen - 1) / PAGE_SIZE; 202 n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE; 203 if (PageHighMem(dpage + n) && 204 size_add(doff, dlen) > PAGE_SIZE) 205 return -ENOSYS; 206 dst = kmap_local_page(dpage) + doff; 207 } 208 209 if (src_isvirt) 210 src = req->svirt; 211 else { 212 src = NULL; 213 do { 214 if (slen <= req->src->length) { 215 spage = sg_page(req->src); 216 soff = req->src->offset; 217 } else 218 break; 219 220 spage = spage + soff / PAGE_SIZE; 221 soff = offset_in_page(soff); 222 223 n = (slen - 1) / PAGE_SIZE; 224 n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE; 225 if (PageHighMem(spage + n) && 226 size_add(soff, slen) > PAGE_SIZE) 227 break; 228 src = kmap_local_page(spage) + soff; 229 } while (0); 230 } 231 232 struct crypto_acomp_stream *stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); 233 234 if (!src_isvirt && !src) { 235 struct scomp_scratch *scratch = scomp_lock_scratch(); 236 const u8 *src = scratch->src; 237 238 memcpy_from_sglist(scratch->src, req->src, 0, slen); 239 240 if (dir) 241 ret = crypto_scomp_compress(scomp, src, slen, 242 dst, &dlen, stream->ctx); 243 else 244 ret = crypto_scomp_decompress(scomp, src, slen, 245 dst, &dlen, stream->ctx); 246 247 scomp_unlock_scratch(scratch); 248 } else if (dir) 249 ret = crypto_scomp_compress(scomp, src, slen, 250 dst, &dlen, stream->ctx); 251 else 252 ret = crypto_scomp_decompress(scomp, src, slen, 253 dst, &dlen, stream->ctx); 254 255 crypto_acomp_unlock_stream_bh(stream); 256 257 req->dlen = dlen; 258 259 if (!src_isvirt && src) 260 kunmap_local(src); 261 if (!dst_isvirt) { 262 kunmap_local(dst); 263 dlen += doff; 264 for (;;) { 265 flush_dcache_page(dpage); 266 if (dlen <= PAGE_SIZE) 267 break; 268 dlen -= PAGE_SIZE; 269 dpage++; 270 } 271 } 272 273 return ret; 274 } 275 276 static int scomp_acomp_compress(struct acomp_req *req) 277 { 278 return scomp_acomp_comp_decomp(req, 1); 279 } 280 281 static int scomp_acomp_decompress(struct acomp_req *req) 282 { 283 return scomp_acomp_comp_decomp(req, 0); 284 } 285 286 static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) 287 { 288 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); 289 290 crypto_free_scomp(*ctx); 291 292 flush_work(&scomp_scratch_work); 293 mutex_lock(&scomp_lock); 294 if (!--scomp_scratch_users) 295 crypto_scomp_free_scratches(); 296 mutex_unlock(&scomp_lock); 297 } 298 299 int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) 300 { 301 struct crypto_alg *calg = tfm->__crt_alg; 302 struct crypto_acomp *crt = __crypto_acomp_tfm(tfm); 303 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); 304 struct crypto_scomp *scomp; 305 306 if (!crypto_mod_get(calg)) 307 return -EAGAIN; 308 309 scomp = crypto_create_tfm(calg, &crypto_scomp_type); 310 if (IS_ERR(scomp)) { 311 crypto_mod_put(calg); 312 return PTR_ERR(scomp); 313 } 314 315 *ctx = scomp; 316 tfm->exit = crypto_exit_scomp_ops_async; 317 318 crt->compress = scomp_acomp_compress; 319 crt->decompress = scomp_acomp_decompress; 320 321 return 0; 322 } 323 324 static void crypto_scomp_destroy(struct crypto_alg *alg) 325 { 326 struct scomp_alg *scomp = __crypto_scomp_alg(alg); 327 328 crypto_acomp_free_streams(&scomp->streams); 329 } 330 331 static const struct crypto_type crypto_scomp_type = { 332 .extsize = crypto_alg_extsize, 333 .init_tfm = crypto_scomp_init_tfm, 334 .destroy = crypto_scomp_destroy, 335 #ifdef CONFIG_PROC_FS 336 .show = crypto_scomp_show, 337 #endif 338 #if IS_ENABLED(CONFIG_CRYPTO_USER) 339 .report = crypto_scomp_report, 340 #endif 341 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 342 .maskset = CRYPTO_ALG_TYPE_MASK, 343 .type = CRYPTO_ALG_TYPE_SCOMPRESS, 344 .tfmsize = offsetof(struct crypto_scomp, base), 345 .algsize = offsetof(struct scomp_alg, base), 346 }; 347 348 static void scomp_prepare_alg(struct scomp_alg *alg) 349 { 350 struct crypto_alg *base = &alg->calg.base; 351 352 comp_prepare_alg(&alg->calg); 353 354 base->cra_flags |= CRYPTO_ALG_REQ_VIRT; 355 } 356 357 int crypto_register_scomp(struct scomp_alg *alg) 358 { 359 struct crypto_alg *base = &alg->calg.base; 360 361 scomp_prepare_alg(alg); 362 363 base->cra_type = &crypto_scomp_type; 364 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; 365 366 return crypto_register_alg(base); 367 } 368 EXPORT_SYMBOL_GPL(crypto_register_scomp); 369 370 void crypto_unregister_scomp(struct scomp_alg *alg) 371 { 372 crypto_unregister_alg(&alg->base); 373 } 374 EXPORT_SYMBOL_GPL(crypto_unregister_scomp); 375 376 int crypto_register_scomps(struct scomp_alg *algs, int count) 377 { 378 int i, ret; 379 380 for (i = 0; i < count; i++) { 381 ret = crypto_register_scomp(&algs[i]); 382 if (ret) { 383 crypto_unregister_scomps(algs, i); 384 return ret; 385 } 386 } 387 388 return 0; 389 } 390 EXPORT_SYMBOL_GPL(crypto_register_scomps); 391 392 void crypto_unregister_scomps(struct scomp_alg *algs, int count) 393 { 394 int i; 395 396 for (i = count - 1; i >= 0; --i) 397 crypto_unregister_scomp(&algs[i]); 398 } 399 EXPORT_SYMBOL_GPL(crypto_unregister_scomps); 400 401 MODULE_LICENSE("GPL"); 402 MODULE_DESCRIPTION("Synchronous compression type"); 403