xref: /linux/drivers/crypto/cavium/cpt/cptvf_algs.c (revision e9f0878c4b2004ac19581274c1ae4c61ae3ca70e) 
1 
2 /*
3  * Copyright (C) 2016 Cavium, Inc.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms of version 2 of the GNU General Public License
7  * as published by the Free Software Foundation.
8  */
9 
10 #include <crypto/aes.h>
11 #include <crypto/algapi.h>
12 #include <crypto/authenc.h>
13 #include <crypto/cryptd.h>
14 #include <crypto/crypto_wq.h>
15 #include <crypto/des.h>
16 #include <crypto/xts.h>
17 #include <linux/crypto.h>
18 #include <linux/err.h>
19 #include <linux/list.h>
20 #include <linux/scatterlist.h>
21 
22 #include "cptvf.h"
23 #include "cptvf_algs.h"
24 
25 struct cpt_device_handle {
26 	void *cdev[MAX_DEVICES];
27 	u32 dev_count;
28 };
29 
30 static struct cpt_device_handle dev_handle;
31 
32 static void cvm_callback(u32 status, void *arg)
33 {
34 	struct crypto_async_request *req = (struct crypto_async_request *)arg;
35 
36 	req->complete(req, !status);
37 }
38 
39 static inline void update_input_iv(struct cpt_request_info *req_info,
40 				   u8 *iv, u32 enc_iv_len,
41 				   u32 *argcnt)
42 {
43 	/* Setting the iv information */
44 	req_info->in[*argcnt].vptr = (void *)iv;
45 	req_info->in[*argcnt].size = enc_iv_len;
46 	req_info->req.dlen += enc_iv_len;
47 
48 	++(*argcnt);
49 }
50 
51 static inline void update_output_iv(struct cpt_request_info *req_info,
52 				    u8 *iv, u32 enc_iv_len,
53 				    u32 *argcnt)
54 {
55 	/* Setting the iv information */
56 	req_info->out[*argcnt].vptr = (void *)iv;
57 	req_info->out[*argcnt].size = enc_iv_len;
58 	req_info->rlen += enc_iv_len;
59 
60 	++(*argcnt);
61 }
62 
63 static inline void update_input_data(struct cpt_request_info *req_info,
64 				     struct scatterlist *inp_sg,
65 				     u32 nbytes, u32 *argcnt)
66 {
67 	req_info->req.dlen += nbytes;
68 
69 	while (nbytes) {
70 		u32 len = min(nbytes, inp_sg->length);
71 		u8 *ptr = sg_virt(inp_sg);
72 
73 		req_info->in[*argcnt].vptr = (void *)ptr;
74 		req_info->in[*argcnt].size = len;
75 		nbytes -= len;
76 
77 		++(*argcnt);
78 		++inp_sg;
79 	}
80 }
81 
82 static inline void update_output_data(struct cpt_request_info *req_info,
83 				      struct scatterlist *outp_sg,
84 				      u32 nbytes, u32 *argcnt)
85 {
86 	req_info->rlen += nbytes;
87 
88 	while (nbytes) {
89 		u32 len = min(nbytes, outp_sg->length);
90 		u8 *ptr = sg_virt(outp_sg);
91 
92 		req_info->out[*argcnt].vptr = (void *)ptr;
93 		req_info->out[*argcnt].size = len;
94 		nbytes -= len;
95 		++(*argcnt);
96 		++outp_sg;
97 	}
98 }
99 
100 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
101 				 u32 *argcnt)
102 {
103 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
104 	struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
105 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
106 	struct fc_context *fctx = &rctx->fctx;
107 	u64 *offset_control = &rctx->control_word;
108 	u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
109 	struct cpt_request_info *req_info = &rctx->cpt_req;
110 	u64 *ctrl_flags = NULL;
111 
112 	req_info->ctrl.s.grp = 0;
113 	req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
114 	req_info->ctrl.s.se_req = SE_CORE_REQ;
115 
116 	req_info->req.opcode.s.major = MAJOR_OP_FC |
117 					DMA_MODE_FLAG(DMA_GATHER_SCATTER);
118 	if (enc)
119 		req_info->req.opcode.s.minor = 2;
120 	else
121 		req_info->req.opcode.s.minor = 3;
122 
123 	req_info->req.param1 = req->nbytes; /* Encryption Data length */
124 	req_info->req.param2 = 0; /*Auth data length */
125 
126 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
127 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
128 	fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
129 
130 	if (ctx->cipher_type == AES_XTS)
131 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
132 	else
133 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
134 	ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
135 	*ctrl_flags = cpu_to_be64(*ctrl_flags);
136 
137 	*offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
138 	/* Storing  Packet Data Information in offset
139 	 * Control Word First 8 bytes
140 	 */
141 	req_info->in[*argcnt].vptr = (u8 *)offset_control;
142 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
143 	req_info->req.dlen += CONTROL_WORD_LEN;
144 	++(*argcnt);
145 
146 	req_info->in[*argcnt].vptr = (u8 *)fctx;
147 	req_info->in[*argcnt].size = sizeof(struct fc_context);
148 	req_info->req.dlen += sizeof(struct fc_context);
149 
150 	++(*argcnt);
151 
152 	return 0;
153 }
154 
155 static inline u32 create_input_list(struct ablkcipher_request  *req, u32 enc,
156 				    u32 enc_iv_len)
157 {
158 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
159 	struct cpt_request_info *req_info = &rctx->cpt_req;
160 	u32 argcnt =  0;
161 
162 	create_ctx_hdr(req, enc, &argcnt);
163 	update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
164 	update_input_data(req_info, req->src, req->nbytes, &argcnt);
165 	req_info->incnt = argcnt;
166 
167 	return 0;
168 }
169 
170 static inline void store_cb_info(struct ablkcipher_request *req,
171 				 struct cpt_request_info *req_info)
172 {
173 	req_info->callback = (void *)cvm_callback;
174 	req_info->callback_arg = (void *)&req->base;
175 }
176 
177 static inline void create_output_list(struct ablkcipher_request *req,
178 				      u32 enc_iv_len)
179 {
180 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
181 	struct cpt_request_info *req_info = &rctx->cpt_req;
182 	u32 argcnt = 0;
183 
184 	/* OUTPUT Buffer Processing
185 	 * AES encryption/decryption output would be
186 	 * received in the following format
187 	 *
188 	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
189 	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
190 	 */
191 	/* Reading IV information */
192 	update_output_iv(req_info, req->info, enc_iv_len, &argcnt);
193 	update_output_data(req_info, req->dst, req->nbytes, &argcnt);
194 	req_info->outcnt = argcnt;
195 }
196 
197 static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
198 {
199 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
200 	struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
201 	u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
202 	struct fc_context *fctx = &rctx->fctx;
203 	struct cpt_request_info *req_info = &rctx->cpt_req;
204 	void *cdev = NULL;
205 	int status;
206 
207 	memset(req_info, 0, sizeof(struct cpt_request_info));
208 	memset(fctx, 0, sizeof(struct fc_context));
209 	create_input_list(req, enc, enc_iv_len);
210 	create_output_list(req, enc_iv_len);
211 	store_cb_info(req, req_info);
212 	cdev = dev_handle.cdev[smp_processor_id()];
213 	status = cptvf_do_request(cdev, req_info);
214 	/* We perform an asynchronous send and once
215 	 * the request is completed the driver would
216 	 * intimate through  registered call back functions
217 	 */
218 
219 	if (status)
220 		return status;
221 	else
222 		return -EINPROGRESS;
223 }
224 
225 static int cvm_encrypt(struct ablkcipher_request *req)
226 {
227 	return cvm_enc_dec(req, true);
228 }
229 
230 static int cvm_decrypt(struct ablkcipher_request *req)
231 {
232 	return cvm_enc_dec(req, false);
233 }
234 
235 static int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
236 		   u32 keylen)
237 {
238 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
239 	struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
240 	int err;
241 	const u8 *key1 = key;
242 	const u8 *key2 = key + (keylen / 2);
243 
244 	err = xts_check_key(tfm, key, keylen);
245 	if (err)
246 		return err;
247 	ctx->key_len = keylen;
248 	memcpy(ctx->enc_key, key1, keylen / 2);
249 	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
250 	ctx->cipher_type = AES_XTS;
251 	switch (ctx->key_len) {
252 	case 32:
253 		ctx->key_type = AES_128_BIT;
254 		break;
255 	case 64:
256 		ctx->key_type = AES_256_BIT;
257 		break;
258 	default:
259 		return -EINVAL;
260 	}
261 
262 	return 0;
263 }
264 
265 static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
266 {
267 	if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
268 		ctx->key_len = keylen;
269 		switch (ctx->key_len) {
270 		case 16:
271 			ctx->key_type = AES_128_BIT;
272 			break;
273 		case 24:
274 			ctx->key_type = AES_192_BIT;
275 			break;
276 		case 32:
277 			ctx->key_type = AES_256_BIT;
278 			break;
279 		default:
280 			return -EINVAL;
281 		}
282 
283 		if (ctx->cipher_type == DES3_CBC)
284 			ctx->key_type = 0;
285 
286 		return 0;
287 	}
288 
289 	return -EINVAL;
290 }
291 
292 static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
293 		      u32 keylen, u8 cipher_type)
294 {
295 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
296 	struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
297 
298 	ctx->cipher_type = cipher_type;
299 	if (!cvm_validate_keylen(ctx, keylen)) {
300 		memcpy(ctx->enc_key, key, keylen);
301 		return 0;
302 	} else {
303 		crypto_ablkcipher_set_flags(cipher,
304 					    CRYPTO_TFM_RES_BAD_KEY_LEN);
305 		return -EINVAL;
306 	}
307 }
308 
309 static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
310 			      u32 keylen)
311 {
312 	return cvm_setkey(cipher, key, keylen, AES_CBC);
313 }
314 
315 static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
316 			      u32 keylen)
317 {
318 	return cvm_setkey(cipher, key, keylen, AES_ECB);
319 }
320 
321 static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
322 			      u32 keylen)
323 {
324 	return cvm_setkey(cipher, key, keylen, AES_CFB);
325 }
326 
327 static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
328 			       u32 keylen)
329 {
330 	return cvm_setkey(cipher, key, keylen, DES3_CBC);
331 }
332 
333 static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
334 			       u32 keylen)
335 {
336 	return cvm_setkey(cipher, key, keylen, DES3_ECB);
337 }
338 
339 static int cvm_enc_dec_init(struct crypto_tfm *tfm)
340 {
341 	struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
342 
343 	memset(ctx, 0, sizeof(*ctx));
344 	tfm->crt_ablkcipher.reqsize = sizeof(struct cvm_req_ctx) +
345 					sizeof(struct ablkcipher_request);
346 	/* Additional memory for ablkcipher_request is
347 	 * allocated since the cryptd daemon uses
348 	 * this memory for request_ctx information
349 	 */
350 
351 	return 0;
352 }
353 
354 static struct crypto_alg algs[] = { {
355 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
356 	.cra_blocksize = AES_BLOCK_SIZE,
357 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
358 	.cra_alignmask = 7,
359 	.cra_priority = 4001,
360 	.cra_name = "xts(aes)",
361 	.cra_driver_name = "cavium-xts-aes",
362 	.cra_type = &crypto_ablkcipher_type,
363 	.cra_u = {
364 		.ablkcipher = {
365 			.ivsize = AES_BLOCK_SIZE,
366 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
367 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
368 			.setkey = cvm_xts_setkey,
369 			.encrypt = cvm_encrypt,
370 			.decrypt = cvm_decrypt,
371 		},
372 	},
373 	.cra_init = cvm_enc_dec_init,
374 	.cra_module = THIS_MODULE,
375 }, {
376 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
377 	.cra_blocksize = AES_BLOCK_SIZE,
378 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
379 	.cra_alignmask = 7,
380 	.cra_priority = 4001,
381 	.cra_name = "cbc(aes)",
382 	.cra_driver_name = "cavium-cbc-aes",
383 	.cra_type = &crypto_ablkcipher_type,
384 	.cra_u = {
385 		.ablkcipher = {
386 			.ivsize = AES_BLOCK_SIZE,
387 			.min_keysize = AES_MIN_KEY_SIZE,
388 			.max_keysize = AES_MAX_KEY_SIZE,
389 			.setkey = cvm_cbc_aes_setkey,
390 			.encrypt = cvm_encrypt,
391 			.decrypt = cvm_decrypt,
392 		},
393 	},
394 	.cra_init = cvm_enc_dec_init,
395 	.cra_module = THIS_MODULE,
396 }, {
397 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
398 	.cra_blocksize = AES_BLOCK_SIZE,
399 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
400 	.cra_alignmask = 7,
401 	.cra_priority = 4001,
402 	.cra_name = "ecb(aes)",
403 	.cra_driver_name = "cavium-ecb-aes",
404 	.cra_type = &crypto_ablkcipher_type,
405 	.cra_u = {
406 		.ablkcipher = {
407 			.ivsize = AES_BLOCK_SIZE,
408 			.min_keysize = AES_MIN_KEY_SIZE,
409 			.max_keysize = AES_MAX_KEY_SIZE,
410 			.setkey = cvm_ecb_aes_setkey,
411 			.encrypt = cvm_encrypt,
412 			.decrypt = cvm_decrypt,
413 		},
414 	},
415 	.cra_init = cvm_enc_dec_init,
416 	.cra_module = THIS_MODULE,
417 }, {
418 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
419 	.cra_blocksize = AES_BLOCK_SIZE,
420 	.cra_ctxsize = sizeof(struct cvm_enc_ctx),
421 	.cra_alignmask = 7,
422 	.cra_priority = 4001,
423 	.cra_name = "cfb(aes)",
424 	.cra_driver_name = "cavium-cfb-aes",
425 	.cra_type = &crypto_ablkcipher_type,
426 	.cra_u = {
427 		.ablkcipher = {
428 			.ivsize = AES_BLOCK_SIZE,
429 			.min_keysize = AES_MIN_KEY_SIZE,
430 			.max_keysize = AES_MAX_KEY_SIZE,
431 			.setkey = cvm_cfb_aes_setkey,
432 			.encrypt = cvm_encrypt,
433 			.decrypt = cvm_decrypt,
434 		},
435 	},
436 	.cra_init = cvm_enc_dec_init,
437 	.cra_module = THIS_MODULE,
438 }, {
439 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
440 	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
441 	.cra_ctxsize = sizeof(struct cvm_des3_ctx),
442 	.cra_alignmask = 7,
443 	.cra_priority = 4001,
444 	.cra_name = "cbc(des3_ede)",
445 	.cra_driver_name = "cavium-cbc-des3_ede",
446 	.cra_type = &crypto_ablkcipher_type,
447 	.cra_u = {
448 		.ablkcipher = {
449 			.min_keysize = DES3_EDE_KEY_SIZE,
450 			.max_keysize = DES3_EDE_KEY_SIZE,
451 			.ivsize = DES_BLOCK_SIZE,
452 			.setkey = cvm_cbc_des3_setkey,
453 			.encrypt = cvm_encrypt,
454 			.decrypt = cvm_decrypt,
455 		},
456 	},
457 	.cra_init = cvm_enc_dec_init,
458 	.cra_module = THIS_MODULE,
459 }, {
460 	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
461 	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
462 	.cra_ctxsize = sizeof(struct cvm_des3_ctx),
463 	.cra_alignmask = 7,
464 	.cra_priority = 4001,
465 	.cra_name = "ecb(des3_ede)",
466 	.cra_driver_name = "cavium-ecb-des3_ede",
467 	.cra_type = &crypto_ablkcipher_type,
468 	.cra_u = {
469 		.ablkcipher = {
470 			.min_keysize = DES3_EDE_KEY_SIZE,
471 			.max_keysize = DES3_EDE_KEY_SIZE,
472 			.ivsize = DES_BLOCK_SIZE,
473 			.setkey = cvm_ecb_des3_setkey,
474 			.encrypt = cvm_encrypt,
475 			.decrypt = cvm_decrypt,
476 		},
477 	},
478 	.cra_init = cvm_enc_dec_init,
479 	.cra_module = THIS_MODULE,
480 } };
481 
482 static inline int cav_register_algs(void)
483 {
484 	int err = 0;
485 
486 	err = crypto_register_algs(algs, ARRAY_SIZE(algs));
487 	if (err)
488 		return err;
489 
490 	return 0;
491 }
492 
493 static inline void cav_unregister_algs(void)
494 {
495 	crypto_unregister_algs(algs, ARRAY_SIZE(algs));
496 }
497 
498 int cvm_crypto_init(struct cpt_vf *cptvf)
499 {
500 	struct pci_dev *pdev = cptvf->pdev;
501 	u32 dev_count;
502 
503 	dev_count = dev_handle.dev_count;
504 	dev_handle.cdev[dev_count] = cptvf;
505 	dev_handle.dev_count++;
506 
507 	if (dev_count == 3) {
508 		if (cav_register_algs()) {
509 			dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
510 			return -EINVAL;
511 		}
512 	}
513 
514 	return 0;
515 }
516 
517 void cvm_crypto_exit(void)
518 {
519 	u32 dev_count;
520 
521 	dev_count = --dev_handle.dev_count;
522 	if (!dev_count)
523 		cav_unregister_algs();
524 }
525