1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/crypto.h>
3 #include <linux/kernel.h>
4 #include <linux/module.h>
5 #include <linux/printk.h>
6
7 #include <crypto/aes.h>
8 #include <crypto/skcipher.h>
9 #include <crypto/scatterwalk.h>
10 #include <crypto/ctr.h>
11 #include <crypto/internal/des.h>
12 #include <crypto/xts.h>
13
14 #include "nitrox_dev.h"
15 #include "nitrox_common.h"
16 #include "nitrox_req.h"
17
18 struct nitrox_cipher {
19 const char *name;
20 enum flexi_cipher value;
21 };
22
23 /*
24 * supported cipher list
25 */
26 static const struct nitrox_cipher flexi_cipher_table[] = {
27 { "null", CIPHER_NULL },
28 { "cbc(des3_ede)", CIPHER_3DES_CBC },
29 { "ecb(des3_ede)", CIPHER_3DES_ECB },
30 { "cbc(aes)", CIPHER_AES_CBC },
31 { "ecb(aes)", CIPHER_AES_ECB },
32 { "cfb(aes)", CIPHER_AES_CFB },
33 { "rfc3686(ctr(aes))", CIPHER_AES_CTR },
34 { "xts(aes)", CIPHER_AES_XTS },
35 { "cts(cbc(aes))", CIPHER_AES_CBC_CTS },
36 { NULL, CIPHER_INVALID }
37 };
38
flexi_cipher_type(const char * name)39 static enum flexi_cipher flexi_cipher_type(const char *name)
40 {
41 const struct nitrox_cipher *cipher = flexi_cipher_table;
42
43 while (cipher->name) {
44 if (!strcmp(cipher->name, name))
45 break;
46 cipher++;
47 }
48 return cipher->value;
49 }
50
free_src_sglist(struct skcipher_request * skreq)51 static void free_src_sglist(struct skcipher_request *skreq)
52 {
53 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
54
55 kfree(nkreq->src);
56 }
57
free_dst_sglist(struct skcipher_request * skreq)58 static void free_dst_sglist(struct skcipher_request *skreq)
59 {
60 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
61
62 kfree(nkreq->dst);
63 }
64
nitrox_skcipher_callback(void * arg,int err)65 static void nitrox_skcipher_callback(void *arg, int err)
66 {
67 struct skcipher_request *skreq = arg;
68
69 free_src_sglist(skreq);
70 free_dst_sglist(skreq);
71 if (err) {
72 pr_err_ratelimited("request failed status 0x%0x\n", err);
73 err = -EINVAL;
74 }
75
76 skcipher_request_complete(skreq, err);
77 }
78
nitrox_cbc_cipher_callback(void * arg,int err)79 static void nitrox_cbc_cipher_callback(void *arg, int err)
80 {
81 struct skcipher_request *skreq = arg;
82 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
83 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
84 int ivsize = crypto_skcipher_ivsize(cipher);
85 unsigned int start = skreq->cryptlen - ivsize;
86
87 if (err) {
88 nitrox_skcipher_callback(arg, err);
89 return;
90 }
91
92 if (nkreq->creq.ctrl.s.arg == ENCRYPT) {
93 scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize,
94 0);
95 } else {
96 if (skreq->src != skreq->dst) {
97 scatterwalk_map_and_copy(skreq->iv, skreq->src, start,
98 ivsize, 0);
99 } else {
100 memcpy(skreq->iv, nkreq->iv_out, ivsize);
101 kfree(nkreq->iv_out);
102 }
103 }
104
105 nitrox_skcipher_callback(arg, err);
106 }
107
nitrox_skcipher_init(struct crypto_skcipher * tfm)108 static int nitrox_skcipher_init(struct crypto_skcipher *tfm)
109 {
110 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
111 struct crypto_ctx_hdr *chdr;
112
113 /* get the first device */
114 nctx->ndev = nitrox_get_first_device();
115 if (!nctx->ndev)
116 return -ENODEV;
117
118 /* allocate nitrox crypto context */
119 chdr = crypto_alloc_context(nctx->ndev);
120 if (!chdr) {
121 nitrox_put_device(nctx->ndev);
122 return -ENOMEM;
123 }
124
125 nctx->callback = nitrox_skcipher_callback;
126 nctx->chdr = chdr;
127 nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
128 sizeof(struct ctx_hdr));
129 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +
130 sizeof(struct nitrox_kcrypt_request));
131 return 0;
132 }
133
nitrox_cbc_init(struct crypto_skcipher * tfm)134 static int nitrox_cbc_init(struct crypto_skcipher *tfm)
135 {
136 int err;
137 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
138
139 err = nitrox_skcipher_init(tfm);
140 if (err)
141 return err;
142
143 nctx->callback = nitrox_cbc_cipher_callback;
144 return 0;
145 }
146
nitrox_skcipher_exit(struct crypto_skcipher * tfm)147 static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)
148 {
149 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
150
151 /* free the nitrox crypto context */
152 if (nctx->u.ctx_handle) {
153 struct flexi_crypto_context *fctx = nctx->u.fctx;
154
155 memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
156 memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
157 crypto_free_context((void *)nctx->chdr);
158 }
159 nitrox_put_device(nctx->ndev);
160
161 nctx->u.ctx_handle = 0;
162 nctx->ndev = NULL;
163 }
164
nitrox_skcipher_setkey(struct crypto_skcipher * cipher,int aes_keylen,const u8 * key,unsigned int keylen)165 static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,
166 int aes_keylen, const u8 *key,
167 unsigned int keylen)
168 {
169 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
170 struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
171 struct flexi_crypto_context *fctx;
172 union fc_ctx_flags *flags;
173 enum flexi_cipher cipher_type;
174 const char *name;
175
176 name = crypto_tfm_alg_name(tfm);
177 cipher_type = flexi_cipher_type(name);
178 if (unlikely(cipher_type == CIPHER_INVALID)) {
179 pr_err("unsupported cipher: %s\n", name);
180 return -EINVAL;
181 }
182
183 /* fill crypto context */
184 fctx = nctx->u.fctx;
185 flags = &fctx->flags;
186 flags->f = 0;
187 flags->w0.cipher_type = cipher_type;
188 flags->w0.aes_keylen = aes_keylen;
189 flags->w0.iv_source = IV_FROM_DPTR;
190 flags->f = cpu_to_be64(*(u64 *)&flags->w0);
191 /* copy the key to context */
192 memcpy(fctx->crypto.u.key, key, keylen);
193
194 return 0;
195 }
196
nitrox_aes_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)197 static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
198 unsigned int keylen)
199 {
200 int aes_keylen;
201
202 aes_keylen = flexi_aes_keylen(keylen);
203 if (aes_keylen < 0)
204 return -EINVAL;
205 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
206 }
207
alloc_src_sglist(struct skcipher_request * skreq,int ivsize)208 static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)
209 {
210 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
211 int nents = sg_nents(skreq->src) + 1;
212 int ret;
213
214 /* Allocate buffer to hold IV and input scatterlist array */
215 ret = alloc_src_req_buf(nkreq, nents, ivsize);
216 if (ret)
217 return ret;
218
219 nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize);
220 nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src,
221 skreq->cryptlen);
222
223 return 0;
224 }
225
alloc_dst_sglist(struct skcipher_request * skreq,int ivsize)226 static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)
227 {
228 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
229 int nents = sg_nents(skreq->dst) + 3;
230 int ret;
231
232 /* Allocate buffer to hold ORH, COMPLETION and output scatterlist
233 * array
234 */
235 ret = alloc_dst_req_buf(nkreq, nents);
236 if (ret)
237 return ret;
238
239 nitrox_creq_set_orh(nkreq);
240 nitrox_creq_set_comp(nkreq);
241 nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst,
242 skreq->cryptlen);
243
244 return 0;
245 }
246
nitrox_skcipher_crypt(struct skcipher_request * skreq,bool enc)247 static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)
248 {
249 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
250 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
251 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
252 int ivsize = crypto_skcipher_ivsize(cipher);
253 struct se_crypto_request *creq;
254 int ret;
255
256 creq = &nkreq->creq;
257 creq->flags = skreq->base.flags;
258 creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
259 GFP_KERNEL : GFP_ATOMIC;
260
261 /* fill the request */
262 creq->ctrl.value = 0;
263 creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
264 creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);
265 /* param0: length of the data to be encrypted */
266 creq->gph.param0 = cpu_to_be16(skreq->cryptlen);
267 creq->gph.param1 = 0;
268 /* param2: encryption data offset */
269 creq->gph.param2 = cpu_to_be16(ivsize);
270 creq->gph.param3 = 0;
271
272 creq->ctx_handle = nctx->u.ctx_handle;
273 creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
274
275 ret = alloc_src_sglist(skreq, ivsize);
276 if (ret)
277 return ret;
278
279 ret = alloc_dst_sglist(skreq, ivsize);
280 if (ret) {
281 free_src_sglist(skreq);
282 return ret;
283 }
284
285 /* send the crypto request */
286 return nitrox_process_se_request(nctx->ndev, creq, nctx->callback,
287 skreq);
288 }
289
nitrox_cbc_decrypt(struct skcipher_request * skreq)290 static int nitrox_cbc_decrypt(struct skcipher_request *skreq)
291 {
292 struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
293 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
294 int ivsize = crypto_skcipher_ivsize(cipher);
295 gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
296 GFP_KERNEL : GFP_ATOMIC;
297 unsigned int start = skreq->cryptlen - ivsize;
298
299 if (skreq->src != skreq->dst)
300 return nitrox_skcipher_crypt(skreq, false);
301
302 nkreq->iv_out = kmalloc(ivsize, flags);
303 if (!nkreq->iv_out)
304 return -ENOMEM;
305
306 scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0);
307 return nitrox_skcipher_crypt(skreq, false);
308 }
309
nitrox_aes_encrypt(struct skcipher_request * skreq)310 static int nitrox_aes_encrypt(struct skcipher_request *skreq)
311 {
312 return nitrox_skcipher_crypt(skreq, true);
313 }
314
nitrox_aes_decrypt(struct skcipher_request * skreq)315 static int nitrox_aes_decrypt(struct skcipher_request *skreq)
316 {
317 return nitrox_skcipher_crypt(skreq, false);
318 }
319
nitrox_3des_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)320 static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
321 const u8 *key, unsigned int keylen)
322 {
323 return verify_skcipher_des3_key(cipher, key) ?:
324 nitrox_skcipher_setkey(cipher, 0, key, keylen);
325 }
326
nitrox_3des_encrypt(struct skcipher_request * skreq)327 static int nitrox_3des_encrypt(struct skcipher_request *skreq)
328 {
329 return nitrox_skcipher_crypt(skreq, true);
330 }
331
nitrox_3des_decrypt(struct skcipher_request * skreq)332 static int nitrox_3des_decrypt(struct skcipher_request *skreq)
333 {
334 return nitrox_skcipher_crypt(skreq, false);
335 }
336
nitrox_aes_xts_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)337 static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,
338 const u8 *key, unsigned int keylen)
339 {
340 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
341 struct flexi_crypto_context *fctx;
342 int aes_keylen, ret;
343
344 ret = xts_verify_key(cipher, key, keylen);
345 if (ret)
346 return ret;
347
348 keylen /= 2;
349
350 aes_keylen = flexi_aes_keylen(keylen);
351 if (aes_keylen < 0)
352 return -EINVAL;
353
354 fctx = nctx->u.fctx;
355 /* copy KEY2 */
356 memcpy(fctx->auth.u.key2, (key + keylen), keylen);
357
358 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
359 }
360
nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)361 static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,
362 const u8 *key, unsigned int keylen)
363 {
364 struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
365 struct flexi_crypto_context *fctx;
366 int aes_keylen;
367
368 if (keylen < CTR_RFC3686_NONCE_SIZE)
369 return -EINVAL;
370
371 fctx = nctx->u.fctx;
372
373 memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),
374 CTR_RFC3686_NONCE_SIZE);
375
376 keylen -= CTR_RFC3686_NONCE_SIZE;
377
378 aes_keylen = flexi_aes_keylen(keylen);
379 if (aes_keylen < 0)
380 return -EINVAL;
381 return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
382 }
383
384 static struct skcipher_alg nitrox_skciphers[] = { {
385 .base = {
386 .cra_name = "cbc(aes)",
387 .cra_driver_name = "n5_cbc(aes)",
388 .cra_priority = PRIO,
389 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
390 .cra_blocksize = AES_BLOCK_SIZE,
391 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
392 .cra_alignmask = 0,
393 .cra_module = THIS_MODULE,
394 },
395 .min_keysize = AES_MIN_KEY_SIZE,
396 .max_keysize = AES_MAX_KEY_SIZE,
397 .ivsize = AES_BLOCK_SIZE,
398 .setkey = nitrox_aes_setkey,
399 .encrypt = nitrox_aes_encrypt,
400 .decrypt = nitrox_cbc_decrypt,
401 .init = nitrox_cbc_init,
402 .exit = nitrox_skcipher_exit,
403 }, {
404 .base = {
405 .cra_name = "ecb(aes)",
406 .cra_driver_name = "n5_ecb(aes)",
407 .cra_priority = PRIO,
408 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
409 .cra_blocksize = AES_BLOCK_SIZE,
410 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
411 .cra_alignmask = 0,
412 .cra_module = THIS_MODULE,
413 },
414 .min_keysize = AES_MIN_KEY_SIZE,
415 .max_keysize = AES_MAX_KEY_SIZE,
416 .ivsize = AES_BLOCK_SIZE,
417 .setkey = nitrox_aes_setkey,
418 .encrypt = nitrox_aes_encrypt,
419 .decrypt = nitrox_aes_decrypt,
420 .init = nitrox_skcipher_init,
421 .exit = nitrox_skcipher_exit,
422 }, {
423 .base = {
424 .cra_name = "xts(aes)",
425 .cra_driver_name = "n5_xts(aes)",
426 .cra_priority = PRIO,
427 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
428 .cra_blocksize = AES_BLOCK_SIZE,
429 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
430 .cra_alignmask = 0,
431 .cra_module = THIS_MODULE,
432 },
433 .min_keysize = 2 * AES_MIN_KEY_SIZE,
434 .max_keysize = 2 * AES_MAX_KEY_SIZE,
435 .ivsize = AES_BLOCK_SIZE,
436 .setkey = nitrox_aes_xts_setkey,
437 .encrypt = nitrox_aes_encrypt,
438 .decrypt = nitrox_aes_decrypt,
439 .init = nitrox_skcipher_init,
440 .exit = nitrox_skcipher_exit,
441 }, {
442 .base = {
443 .cra_name = "rfc3686(ctr(aes))",
444 .cra_driver_name = "n5_rfc3686(ctr(aes))",
445 .cra_priority = PRIO,
446 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
447 .cra_blocksize = 1,
448 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
449 .cra_alignmask = 0,
450 .cra_module = THIS_MODULE,
451 },
452 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
453 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
454 .ivsize = CTR_RFC3686_IV_SIZE,
455 .init = nitrox_skcipher_init,
456 .exit = nitrox_skcipher_exit,
457 .setkey = nitrox_aes_ctr_rfc3686_setkey,
458 .encrypt = nitrox_aes_encrypt,
459 .decrypt = nitrox_aes_decrypt,
460 }, {
461 .base = {
462 .cra_name = "cts(cbc(aes))",
463 .cra_driver_name = "n5_cts(cbc(aes))",
464 .cra_priority = PRIO,
465 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
466 .cra_blocksize = AES_BLOCK_SIZE,
467 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
468 .cra_alignmask = 0,
469 .cra_module = THIS_MODULE,
470 },
471 .min_keysize = AES_MIN_KEY_SIZE,
472 .max_keysize = AES_MAX_KEY_SIZE,
473 .ivsize = AES_BLOCK_SIZE,
474 .setkey = nitrox_aes_setkey,
475 .encrypt = nitrox_aes_encrypt,
476 .decrypt = nitrox_aes_decrypt,
477 .init = nitrox_skcipher_init,
478 .exit = nitrox_skcipher_exit,
479 }, {
480 .base = {
481 .cra_name = "cbc(des3_ede)",
482 .cra_driver_name = "n5_cbc(des3_ede)",
483 .cra_priority = PRIO,
484 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
485 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
486 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
487 .cra_alignmask = 0,
488 .cra_module = THIS_MODULE,
489 },
490 .min_keysize = DES3_EDE_KEY_SIZE,
491 .max_keysize = DES3_EDE_KEY_SIZE,
492 .ivsize = DES3_EDE_BLOCK_SIZE,
493 .setkey = nitrox_3des_setkey,
494 .encrypt = nitrox_3des_encrypt,
495 .decrypt = nitrox_cbc_decrypt,
496 .init = nitrox_cbc_init,
497 .exit = nitrox_skcipher_exit,
498 }, {
499 .base = {
500 .cra_name = "ecb(des3_ede)",
501 .cra_driver_name = "n5_ecb(des3_ede)",
502 .cra_priority = PRIO,
503 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
504 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
505 .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
506 .cra_alignmask = 0,
507 .cra_module = THIS_MODULE,
508 },
509 .min_keysize = DES3_EDE_KEY_SIZE,
510 .max_keysize = DES3_EDE_KEY_SIZE,
511 .ivsize = DES3_EDE_BLOCK_SIZE,
512 .setkey = nitrox_3des_setkey,
513 .encrypt = nitrox_3des_encrypt,
514 .decrypt = nitrox_3des_decrypt,
515 .init = nitrox_skcipher_init,
516 .exit = nitrox_skcipher_exit,
517 }
518
519 };
520
nitrox_register_skciphers(void)521 int nitrox_register_skciphers(void)
522 {
523 return crypto_register_skciphers(nitrox_skciphers,
524 ARRAY_SIZE(nitrox_skciphers));
525 }
526
nitrox_unregister_skciphers(void)527 void nitrox_unregister_skciphers(void)
528 {
529 crypto_unregister_skciphers(nitrox_skciphers,
530 ARRAY_SIZE(nitrox_skciphers));
531 }
532