xref: /linux/drivers/crypto/bcm/cipher.c (revision 06d07429858317ded2db7986113a9e0129cd599b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2016 Broadcom
4  */
5 
6 #include <linux/err.h>
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/errno.h>
10 #include <linux/kernel.h>
11 #include <linux/interrupt.h>
12 #include <linux/platform_device.h>
13 #include <linux/scatterlist.h>
14 #include <linux/crypto.h>
15 #include <linux/kthread.h>
16 #include <linux/rtnetlink.h>
17 #include <linux/sched.h>
18 #include <linux/of.h>
19 #include <linux/io.h>
20 #include <linux/bitops.h>
21 
22 #include <crypto/algapi.h>
23 #include <crypto/aead.h>
24 #include <crypto/internal/aead.h>
25 #include <crypto/aes.h>
26 #include <crypto/internal/des.h>
27 #include <crypto/hmac.h>
28 #include <crypto/md5.h>
29 #include <crypto/authenc.h>
30 #include <crypto/skcipher.h>
31 #include <crypto/hash.h>
32 #include <crypto/sha1.h>
33 #include <crypto/sha2.h>
34 #include <crypto/sha3.h>
35 
36 #include "util.h"
37 #include "cipher.h"
38 #include "spu.h"
39 #include "spum.h"
40 #include "spu2.h"
41 
42 /* ================= Device Structure ================== */
43 
44 struct bcm_device_private iproc_priv;
45 
46 /* ==================== Parameters ===================== */
47 
48 int flow_debug_logging;
49 module_param(flow_debug_logging, int, 0644);
50 MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
51 
52 int packet_debug_logging;
53 module_param(packet_debug_logging, int, 0644);
54 MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
55 
56 int debug_logging_sleep;
57 module_param(debug_logging_sleep, int, 0644);
58 MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
59 
60 /*
61  * The value of these module parameters is used to set the priority for each
62  * algo type when this driver registers algos with the kernel crypto API.
63  * To use a priority other than the default, set the priority in the insmod or
64  * modprobe. Changing the module priority after init time has no effect.
65  *
66  * The default priorities are chosen to be lower (less preferred) than ARMv8 CE
67  * algos, but more preferred than generic software algos.
68  */
69 static int cipher_pri = 150;
70 module_param(cipher_pri, int, 0644);
71 MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
72 
73 static int hash_pri = 100;
74 module_param(hash_pri, int, 0644);
75 MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
76 
77 static int aead_pri = 150;
78 module_param(aead_pri, int, 0644);
79 MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
80 
81 /* A type 3 BCM header, expected to precede the SPU header for SPU-M.
82  * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
83  * 0x60 - ring 0
84  * 0x68 - ring 1
85  * 0x70 - ring 2
86  * 0x78 - ring 3
87  */
88 static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
89 /*
90  * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
91  * is set dynamically after reading SPU type from device tree.
92  */
93 #define BCM_HDR_LEN  iproc_priv.bcm_hdr_len
94 
95 /* min and max time to sleep before retrying when mbox queue is full. usec */
96 #define MBOX_SLEEP_MIN  800
97 #define MBOX_SLEEP_MAX 1000
98 
99 /**
100  * select_channel() - Select a SPU channel to handle a crypto request. Selects
101  * channel in round robin order.
102  *
103  * Return:  channel index
104  */
select_channel(void)105 static u8 select_channel(void)
106 {
107 	u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
108 
109 	return chan_idx % iproc_priv.spu.num_chan;
110 }
111 
112 /**
113  * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to
114  * receive a SPU response message for an skcipher request. Includes buffers to
115  * catch SPU message headers and the response data.
116  * @mssg:	mailbox message containing the receive sg
117  * @rctx:	crypto request context
118  * @rx_frag_num: number of scatterlist elements required to hold the
119  *		SPU response message
120  * @chunksize:	Number of bytes of response data expected
121  * @stat_pad_len: Number of bytes required to pad the STAT field to
122  *		a 4-byte boundary
123  *
124  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
125  * when the request completes, whether the request is handled successfully or
126  * there is an error.
127  *
128  * Returns:
129  *   0 if successful
130  *   < 0 if an error
131  */
132 static int
spu_skcipher_rx_sg_create(struct brcm_message * mssg,struct iproc_reqctx_s * rctx,u8 rx_frag_num,unsigned int chunksize,u32 stat_pad_len)133 spu_skcipher_rx_sg_create(struct brcm_message *mssg,
134 			    struct iproc_reqctx_s *rctx,
135 			    u8 rx_frag_num,
136 			    unsigned int chunksize, u32 stat_pad_len)
137 {
138 	struct spu_hw *spu = &iproc_priv.spu;
139 	struct scatterlist *sg;	/* used to build sgs in mbox message */
140 	struct iproc_ctx_s *ctx = rctx->ctx;
141 	u32 datalen;		/* Number of bytes of response data expected */
142 
143 	mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
144 				rctx->gfp);
145 	if (!mssg->spu.dst)
146 		return -ENOMEM;
147 
148 	sg = mssg->spu.dst;
149 	sg_init_table(sg, rx_frag_num);
150 	/* Space for SPU message header */
151 	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
152 
153 	/* If XTS tweak in payload, add buffer to receive encrypted tweak */
154 	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
155 	    spu->spu_xts_tweak_in_payload())
156 		sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
157 			   SPU_XTS_TWEAK_SIZE);
158 
159 	/* Copy in each dst sg entry from request, up to chunksize */
160 	datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
161 				 rctx->dst_nents, chunksize);
162 	if (datalen < chunksize) {
163 		pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
164 		       __func__, chunksize, datalen);
165 		return -EFAULT;
166 	}
167 
168 	if (stat_pad_len)
169 		sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
170 
171 	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
172 	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
173 
174 	return 0;
175 }
176 
177 /**
178  * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to
179  * send a SPU request message for an skcipher request. Includes SPU message
180  * headers and the request data.
181  * @mssg:	mailbox message containing the transmit sg
182  * @rctx:	crypto request context
183  * @tx_frag_num: number of scatterlist elements required to construct the
184  *		SPU request message
185  * @chunksize:	Number of bytes of request data
186  * @pad_len:	Number of pad bytes
187  *
188  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
189  * when the request completes, whether the request is handled successfully or
190  * there is an error.
191  *
192  * Returns:
193  *   0 if successful
194  *   < 0 if an error
195  */
196 static int
spu_skcipher_tx_sg_create(struct brcm_message * mssg,struct iproc_reqctx_s * rctx,u8 tx_frag_num,unsigned int chunksize,u32 pad_len)197 spu_skcipher_tx_sg_create(struct brcm_message *mssg,
198 			    struct iproc_reqctx_s *rctx,
199 			    u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
200 {
201 	struct spu_hw *spu = &iproc_priv.spu;
202 	struct scatterlist *sg;	/* used to build sgs in mbox message */
203 	struct iproc_ctx_s *ctx = rctx->ctx;
204 	u32 datalen;		/* Number of bytes of response data expected */
205 	u32 stat_len;
206 
207 	mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
208 				rctx->gfp);
209 	if (unlikely(!mssg->spu.src))
210 		return -ENOMEM;
211 
212 	sg = mssg->spu.src;
213 	sg_init_table(sg, tx_frag_num);
214 
215 	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
216 		   BCM_HDR_LEN + ctx->spu_req_hdr_len);
217 
218 	/* if XTS tweak in payload, copy from IV (where crypto API puts it) */
219 	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
220 	    spu->spu_xts_tweak_in_payload())
221 		sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
222 
223 	/* Copy in each src sg entry from request, up to chunksize */
224 	datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
225 				 rctx->src_nents, chunksize);
226 	if (unlikely(datalen < chunksize)) {
227 		pr_err("%s(): failed to copy src sg to mbox msg",
228 		       __func__);
229 		return -EFAULT;
230 	}
231 
232 	if (pad_len)
233 		sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
234 
235 	stat_len = spu->spu_tx_status_len();
236 	if (stat_len) {
237 		memset(rctx->msg_buf.tx_stat, 0, stat_len);
238 		sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
239 	}
240 	return 0;
241 }
242 
mailbox_send_message(struct brcm_message * mssg,u32 flags,u8 chan_idx)243 static int mailbox_send_message(struct brcm_message *mssg, u32 flags,
244 				u8 chan_idx)
245 {
246 	int err;
247 	int retry_cnt = 0;
248 	struct device *dev = &(iproc_priv.pdev->dev);
249 
250 	err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg);
251 	if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
252 		while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
253 			/*
254 			 * Mailbox queue is full. Since MAY_SLEEP is set, assume
255 			 * not in atomic context and we can wait and try again.
256 			 */
257 			retry_cnt++;
258 			usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
259 			err = mbox_send_message(iproc_priv.mbox[chan_idx],
260 						mssg);
261 			atomic_inc(&iproc_priv.mb_no_spc);
262 		}
263 	}
264 	if (err < 0) {
265 		atomic_inc(&iproc_priv.mb_send_fail);
266 		return err;
267 	}
268 
269 	/* Check error returned by mailbox controller */
270 	err = mssg->error;
271 	if (unlikely(err < 0)) {
272 		dev_err(dev, "message error %d", err);
273 		/* Signal txdone for mailbox channel */
274 	}
275 
276 	/* Signal txdone for mailbox channel */
277 	mbox_client_txdone(iproc_priv.mbox[chan_idx], err);
278 	return err;
279 }
280 
281 /**
282  * handle_skcipher_req() - Submit as much of a block cipher request as fits in
283  * a single SPU request message, starting at the current position in the request
284  * data.
285  * @rctx:	Crypto request context
286  *
287  * This may be called on the crypto API thread, or, when a request is so large
288  * it must be broken into multiple SPU messages, on the thread used to invoke
289  * the response callback. When requests are broken into multiple SPU
290  * messages, we assume subsequent messages depend on previous results, and
291  * thus always wait for previous results before submitting the next message.
292  * Because requests are submitted in lock step like this, there is no need
293  * to synchronize access to request data structures.
294  *
295  * Return: -EINPROGRESS: request has been accepted and result will be returned
296  *			 asynchronously
297  *         Any other value indicates an error
298  */
handle_skcipher_req(struct iproc_reqctx_s * rctx)299 static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
300 {
301 	struct spu_hw *spu = &iproc_priv.spu;
302 	struct crypto_async_request *areq = rctx->parent;
303 	struct skcipher_request *req =
304 	    container_of(areq, struct skcipher_request, base);
305 	struct iproc_ctx_s *ctx = rctx->ctx;
306 	struct spu_cipher_parms cipher_parms;
307 	int err;
308 	unsigned int chunksize;	/* Num bytes of request to submit */
309 	int remaining;	/* Bytes of request still to process */
310 	int chunk_start;	/* Beginning of data for current SPU msg */
311 
312 	/* IV or ctr value to use in this SPU msg */
313 	u8 local_iv_ctr[MAX_IV_SIZE];
314 	u32 stat_pad_len;	/* num bytes to align status field */
315 	u32 pad_len;		/* total length of all padding */
316 	struct brcm_message *mssg;	/* mailbox message */
317 
318 	/* number of entries in src and dst sg in mailbox message. */
319 	u8 rx_frag_num = 2;	/* response header and STATUS */
320 	u8 tx_frag_num = 1;	/* request header */
321 
322 	flow_log("%s\n", __func__);
323 
324 	cipher_parms.alg = ctx->cipher.alg;
325 	cipher_parms.mode = ctx->cipher.mode;
326 	cipher_parms.type = ctx->cipher_type;
327 	cipher_parms.key_len = ctx->enckeylen;
328 	cipher_parms.key_buf = ctx->enckey;
329 	cipher_parms.iv_buf = local_iv_ctr;
330 	cipher_parms.iv_len = rctx->iv_ctr_len;
331 
332 	mssg = &rctx->mb_mssg;
333 	chunk_start = rctx->src_sent;
334 	remaining = rctx->total_todo - chunk_start;
335 
336 	/* determine the chunk we are breaking off and update the indexes */
337 	if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
338 	    (remaining > ctx->max_payload))
339 		chunksize = ctx->max_payload;
340 	else
341 		chunksize = remaining;
342 
343 	rctx->src_sent += chunksize;
344 	rctx->total_sent = rctx->src_sent;
345 
346 	/* Count number of sg entries to be included in this request */
347 	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
348 	rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
349 
350 	if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
351 	    rctx->is_encrypt && chunk_start)
352 		/*
353 		 * Encrypting non-first first chunk. Copy last block of
354 		 * previous result to IV for this chunk.
355 		 */
356 		sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
357 				    rctx->iv_ctr_len,
358 				    chunk_start - rctx->iv_ctr_len);
359 
360 	if (rctx->iv_ctr_len) {
361 		/* get our local copy of the iv */
362 		__builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
363 				 rctx->iv_ctr_len);
364 
365 		/* generate the next IV if possible */
366 		if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
367 		    !rctx->is_encrypt) {
368 			/*
369 			 * CBC Decrypt: next IV is the last ciphertext block in
370 			 * this chunk
371 			 */
372 			sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
373 					    rctx->iv_ctr_len,
374 					    rctx->src_sent - rctx->iv_ctr_len);
375 		} else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
376 			/*
377 			 * The SPU hardware increments the counter once for
378 			 * each AES block of 16 bytes. So update the counter
379 			 * for the next chunk, if there is one. Note that for
380 			 * this chunk, the counter has already been copied to
381 			 * local_iv_ctr. We can assume a block size of 16,
382 			 * because we only support CTR mode for AES, not for
383 			 * any other cipher alg.
384 			 */
385 			add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
386 		}
387 	}
388 
389 	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
390 		flow_log("max_payload infinite\n");
391 	else
392 		flow_log("max_payload %u\n", ctx->max_payload);
393 
394 	flow_log("sent:%u start:%u remains:%u size:%u\n",
395 		 rctx->src_sent, chunk_start, remaining, chunksize);
396 
397 	/* Copy SPU header template created at setkey time */
398 	memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
399 	       sizeof(rctx->msg_buf.bcm_spu_req_hdr));
400 
401 	spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
402 				   ctx->spu_req_hdr_len, !(rctx->is_encrypt),
403 				   &cipher_parms, chunksize);
404 
405 	atomic64_add(chunksize, &iproc_priv.bytes_out);
406 
407 	stat_pad_len = spu->spu_wordalign_padlen(chunksize);
408 	if (stat_pad_len)
409 		rx_frag_num++;
410 	pad_len = stat_pad_len;
411 	if (pad_len) {
412 		tx_frag_num++;
413 		spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
414 				     0, ctx->auth.alg, ctx->auth.mode,
415 				     rctx->total_sent, stat_pad_len);
416 	}
417 
418 	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
419 			      ctx->spu_req_hdr_len);
420 	packet_log("payload:\n");
421 	dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
422 	packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
423 
424 	/*
425 	 * Build mailbox message containing SPU request msg and rx buffers
426 	 * to catch response message
427 	 */
428 	memset(mssg, 0, sizeof(*mssg));
429 	mssg->type = BRCM_MESSAGE_SPU;
430 	mssg->ctx = rctx;	/* Will be returned in response */
431 
432 	/* Create rx scatterlist to catch result */
433 	rx_frag_num += rctx->dst_nents;
434 
435 	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
436 	    spu->spu_xts_tweak_in_payload())
437 		rx_frag_num++;	/* extra sg to insert tweak */
438 
439 	err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
440 					  stat_pad_len);
441 	if (err)
442 		return err;
443 
444 	/* Create tx scatterlist containing SPU request message */
445 	tx_frag_num += rctx->src_nents;
446 	if (spu->spu_tx_status_len())
447 		tx_frag_num++;
448 
449 	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
450 	    spu->spu_xts_tweak_in_payload())
451 		tx_frag_num++;	/* extra sg to insert tweak */
452 
453 	err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
454 					  pad_len);
455 	if (err)
456 		return err;
457 
458 	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
459 	if (unlikely(err < 0))
460 		return err;
461 
462 	return -EINPROGRESS;
463 }
464 
465 /**
466  * handle_skcipher_resp() - Process a block cipher SPU response. Updates the
467  * total received count for the request and updates global stats.
468  * @rctx:	Crypto request context
469  */
handle_skcipher_resp(struct iproc_reqctx_s * rctx)470 static void handle_skcipher_resp(struct iproc_reqctx_s *rctx)
471 {
472 	struct spu_hw *spu = &iproc_priv.spu;
473 	struct crypto_async_request *areq = rctx->parent;
474 	struct skcipher_request *req = skcipher_request_cast(areq);
475 	struct iproc_ctx_s *ctx = rctx->ctx;
476 	u32 payload_len;
477 
478 	/* See how much data was returned */
479 	payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
480 
481 	/*
482 	 * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
483 	 * encrypted tweak ("i") value; we don't count those.
484 	 */
485 	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
486 	    spu->spu_xts_tweak_in_payload() &&
487 	    (payload_len >= SPU_XTS_TWEAK_SIZE))
488 		payload_len -= SPU_XTS_TWEAK_SIZE;
489 
490 	atomic64_add(payload_len, &iproc_priv.bytes_in);
491 
492 	flow_log("%s() offset: %u, bd_len: %u BD:\n",
493 		 __func__, rctx->total_received, payload_len);
494 
495 	dump_sg(req->dst, rctx->total_received, payload_len);
496 
497 	rctx->total_received += payload_len;
498 	if (rctx->total_received == rctx->total_todo) {
499 		atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
500 		atomic_inc(
501 		   &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
502 	}
503 }
504 
505 /**
506  * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
507  * receive a SPU response message for an ahash request.
508  * @mssg:	mailbox message containing the receive sg
509  * @rctx:	crypto request context
510  * @rx_frag_num: number of scatterlist elements required to hold the
511  *		SPU response message
512  * @digestsize: length of hash digest, in bytes
513  * @stat_pad_len: Number of bytes required to pad the STAT field to
514  *		a 4-byte boundary
515  *
516  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
517  * when the request completes, whether the request is handled successfully or
518  * there is an error.
519  *
520  * Return:
521  *   0 if successful
522  *   < 0 if an error
523  */
524 static int
spu_ahash_rx_sg_create(struct brcm_message * mssg,struct iproc_reqctx_s * rctx,u8 rx_frag_num,unsigned int digestsize,u32 stat_pad_len)525 spu_ahash_rx_sg_create(struct brcm_message *mssg,
526 		       struct iproc_reqctx_s *rctx,
527 		       u8 rx_frag_num, unsigned int digestsize,
528 		       u32 stat_pad_len)
529 {
530 	struct spu_hw *spu = &iproc_priv.spu;
531 	struct scatterlist *sg;	/* used to build sgs in mbox message */
532 	struct iproc_ctx_s *ctx = rctx->ctx;
533 
534 	mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
535 				rctx->gfp);
536 	if (!mssg->spu.dst)
537 		return -ENOMEM;
538 
539 	sg = mssg->spu.dst;
540 	sg_init_table(sg, rx_frag_num);
541 	/* Space for SPU message header */
542 	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
543 
544 	/* Space for digest */
545 	sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
546 
547 	if (stat_pad_len)
548 		sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
549 
550 	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
551 	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
552 	return 0;
553 }
554 
555 /**
556  * spu_ahash_tx_sg_create() -  Build up the scatterlist of buffers used to send
557  * a SPU request message for an ahash request. Includes SPU message headers and
558  * the request data.
559  * @mssg:	mailbox message containing the transmit sg
560  * @rctx:	crypto request context
561  * @tx_frag_num: number of scatterlist elements required to construct the
562  *		SPU request message
563  * @spu_hdr_len: length in bytes of SPU message header
564  * @hash_carry_len: Number of bytes of data carried over from previous req
565  * @new_data_len: Number of bytes of new request data
566  * @pad_len:	Number of pad bytes
567  *
568  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
569  * when the request completes, whether the request is handled successfully or
570  * there is an error.
571  *
572  * Return:
573  *   0 if successful
574  *   < 0 if an error
575  */
576 static int
spu_ahash_tx_sg_create(struct brcm_message * mssg,struct iproc_reqctx_s * rctx,u8 tx_frag_num,u32 spu_hdr_len,unsigned int hash_carry_len,unsigned int new_data_len,u32 pad_len)577 spu_ahash_tx_sg_create(struct brcm_message *mssg,
578 		       struct iproc_reqctx_s *rctx,
579 		       u8 tx_frag_num,
580 		       u32 spu_hdr_len,
581 		       unsigned int hash_carry_len,
582 		       unsigned int new_data_len, u32 pad_len)
583 {
584 	struct spu_hw *spu = &iproc_priv.spu;
585 	struct scatterlist *sg;	/* used to build sgs in mbox message */
586 	u32 datalen;		/* Number of bytes of response data expected */
587 	u32 stat_len;
588 
589 	mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
590 				rctx->gfp);
591 	if (!mssg->spu.src)
592 		return -ENOMEM;
593 
594 	sg = mssg->spu.src;
595 	sg_init_table(sg, tx_frag_num);
596 
597 	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
598 		   BCM_HDR_LEN + spu_hdr_len);
599 
600 	if (hash_carry_len)
601 		sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
602 
603 	if (new_data_len) {
604 		/* Copy in each src sg entry from request, up to chunksize */
605 		datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
606 					 rctx->src_nents, new_data_len);
607 		if (datalen < new_data_len) {
608 			pr_err("%s(): failed to copy src sg to mbox msg",
609 			       __func__);
610 			return -EFAULT;
611 		}
612 	}
613 
614 	if (pad_len)
615 		sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
616 
617 	stat_len = spu->spu_tx_status_len();
618 	if (stat_len) {
619 		memset(rctx->msg_buf.tx_stat, 0, stat_len);
620 		sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
621 	}
622 
623 	return 0;
624 }
625 
626 /**
627  * handle_ahash_req() - Process an asynchronous hash request from the crypto
628  * API.
629  * @rctx:  Crypto request context
630  *
631  * Builds a SPU request message embedded in a mailbox message and submits the
632  * mailbox message on a selected mailbox channel. The SPU request message is
633  * constructed as a scatterlist, including entries from the crypto API's
634  * src scatterlist to avoid copying the data to be hashed. This function is
635  * called either on the thread from the crypto API, or, in the case that the
636  * crypto API request is too large to fit in a single SPU request message,
637  * on the thread that invokes the receive callback with a response message.
638  * Because some operations require the response from one chunk before the next
639  * chunk can be submitted, we always wait for the response for the previous
640  * chunk before submitting the next chunk. Because requests are submitted in
641  * lock step like this, there is no need to synchronize access to request data
642  * structures.
643  *
644  * Return:
645  *   -EINPROGRESS: request has been submitted to SPU and response will be
646  *		   returned asynchronously
647  *   -EAGAIN:      non-final request included a small amount of data, which for
648  *		   efficiency we did not submit to the SPU, but instead stored
649  *		   to be submitted to the SPU with the next part of the request
650  *   other:        an error code
651  */
handle_ahash_req(struct iproc_reqctx_s * rctx)652 static int handle_ahash_req(struct iproc_reqctx_s *rctx)
653 {
654 	struct spu_hw *spu = &iproc_priv.spu;
655 	struct crypto_async_request *areq = rctx->parent;
656 	struct ahash_request *req = ahash_request_cast(areq);
657 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
658 	struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
659 	unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
660 	struct iproc_ctx_s *ctx = rctx->ctx;
661 
662 	/* number of bytes still to be hashed in this req */
663 	unsigned int nbytes_to_hash = 0;
664 	int err;
665 	unsigned int chunksize = 0;	/* length of hash carry + new data */
666 	/*
667 	 * length of new data, not from hash carry, to be submitted in
668 	 * this hw request
669 	 */
670 	unsigned int new_data_len;
671 
672 	unsigned int __maybe_unused chunk_start = 0;
673 	u32 db_size;	 /* Length of data field, incl gcm and hash padding */
674 	int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
675 	u32 data_pad_len = 0;	/* length of GCM/CCM padding */
676 	u32 stat_pad_len = 0;	/* length of padding to align STATUS word */
677 	struct brcm_message *mssg;	/* mailbox message */
678 	struct spu_request_opts req_opts;
679 	struct spu_cipher_parms cipher_parms;
680 	struct spu_hash_parms hash_parms;
681 	struct spu_aead_parms aead_parms;
682 	unsigned int local_nbuf;
683 	u32 spu_hdr_len;
684 	unsigned int digestsize;
685 	u16 rem = 0;
686 
687 	/*
688 	 * number of entries in src and dst sg. Always includes SPU msg header.
689 	 * rx always includes a buffer to catch digest and STATUS.
690 	 */
691 	u8 rx_frag_num = 3;
692 	u8 tx_frag_num = 1;
693 
694 	flow_log("total_todo %u, total_sent %u\n",
695 		 rctx->total_todo, rctx->total_sent);
696 
697 	memset(&req_opts, 0, sizeof(req_opts));
698 	memset(&cipher_parms, 0, sizeof(cipher_parms));
699 	memset(&hash_parms, 0, sizeof(hash_parms));
700 	memset(&aead_parms, 0, sizeof(aead_parms));
701 
702 	req_opts.bd_suppress = true;
703 	hash_parms.alg = ctx->auth.alg;
704 	hash_parms.mode = ctx->auth.mode;
705 	hash_parms.type = HASH_TYPE_NONE;
706 	hash_parms.key_buf = (u8 *)ctx->authkey;
707 	hash_parms.key_len = ctx->authkeylen;
708 
709 	/*
710 	 * For hash algorithms below assignment looks bit odd but
711 	 * it's needed for AES-XCBC and AES-CMAC hash algorithms
712 	 * to differentiate between 128, 192, 256 bit key values.
713 	 * Based on the key values, hash algorithm is selected.
714 	 * For example for 128 bit key, hash algorithm is AES-128.
715 	 */
716 	cipher_parms.type = ctx->cipher_type;
717 
718 	mssg = &rctx->mb_mssg;
719 	chunk_start = rctx->src_sent;
720 
721 	/*
722 	 * Compute the amount remaining to hash. This may include data
723 	 * carried over from previous requests.
724 	 */
725 	nbytes_to_hash = rctx->total_todo - rctx->total_sent;
726 	chunksize = nbytes_to_hash;
727 	if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
728 	    (chunksize > ctx->max_payload))
729 		chunksize = ctx->max_payload;
730 
731 	/*
732 	 * If this is not a final request and the request data is not a multiple
733 	 * of a full block, then simply park the extra data and prefix it to the
734 	 * data for the next request.
735 	 */
736 	if (!rctx->is_final) {
737 		u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
738 		u16 new_len;  /* len of data to add to hash carry */
739 
740 		rem = chunksize % blocksize;   /* remainder */
741 		if (rem) {
742 			/* chunksize not a multiple of blocksize */
743 			chunksize -= rem;
744 			if (chunksize == 0) {
745 				/* Don't have a full block to submit to hw */
746 				new_len = rem - rctx->hash_carry_len;
747 				sg_copy_part_to_buf(req->src, dest, new_len,
748 						    rctx->src_sent);
749 				rctx->hash_carry_len = rem;
750 				flow_log("Exiting with hash carry len: %u\n",
751 					 rctx->hash_carry_len);
752 				packet_dump("  buf: ",
753 					    rctx->hash_carry,
754 					    rctx->hash_carry_len);
755 				return -EAGAIN;
756 			}
757 		}
758 	}
759 
760 	/* if we have hash carry, then prefix it to the data in this request */
761 	local_nbuf = rctx->hash_carry_len;
762 	rctx->hash_carry_len = 0;
763 	if (local_nbuf)
764 		tx_frag_num++;
765 	new_data_len = chunksize - local_nbuf;
766 
767 	/* Count number of sg entries to be used in this request */
768 	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
769 				       new_data_len);
770 
771 	/* AES hashing keeps key size in type field, so need to copy it here */
772 	if (hash_parms.alg == HASH_ALG_AES)
773 		hash_parms.type = (enum hash_type)cipher_parms.type;
774 	else
775 		hash_parms.type = spu->spu_hash_type(rctx->total_sent);
776 
777 	digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
778 					  hash_parms.type);
779 	hash_parms.digestsize =	digestsize;
780 
781 	/* update the indexes */
782 	rctx->total_sent += chunksize;
783 	/* if you sent a prebuf then that wasn't from this req->src */
784 	rctx->src_sent += new_data_len;
785 
786 	if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
787 		hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
788 							   hash_parms.mode,
789 							   chunksize,
790 							   blocksize);
791 
792 	/*
793 	 * If a non-first chunk, then include the digest returned from the
794 	 * previous chunk so that hw can add to it (except for AES types).
795 	 */
796 	if ((hash_parms.type == HASH_TYPE_UPDT) &&
797 	    (hash_parms.alg != HASH_ALG_AES)) {
798 		hash_parms.key_buf = rctx->incr_hash;
799 		hash_parms.key_len = digestsize;
800 	}
801 
802 	atomic64_add(chunksize, &iproc_priv.bytes_out);
803 
804 	flow_log("%s() final: %u nbuf: %u ",
805 		 __func__, rctx->is_final, local_nbuf);
806 
807 	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
808 		flow_log("max_payload infinite\n");
809 	else
810 		flow_log("max_payload %u\n", ctx->max_payload);
811 
812 	flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
813 
814 	/* Prepend SPU header with type 3 BCM header */
815 	memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
816 
817 	hash_parms.prebuf_len = local_nbuf;
818 	spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
819 					      BCM_HDR_LEN,
820 					      &req_opts, &cipher_parms,
821 					      &hash_parms, &aead_parms,
822 					      new_data_len);
823 
824 	if (spu_hdr_len == 0) {
825 		pr_err("Failed to create SPU request header\n");
826 		return -EFAULT;
827 	}
828 
829 	/*
830 	 * Determine total length of padding required. Put all padding in one
831 	 * buffer.
832 	 */
833 	data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
834 	db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
835 				   0, 0, hash_parms.pad_len);
836 	if (spu->spu_tx_status_len())
837 		stat_pad_len = spu->spu_wordalign_padlen(db_size);
838 	if (stat_pad_len)
839 		rx_frag_num++;
840 	pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
841 	if (pad_len) {
842 		tx_frag_num++;
843 		spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
844 				     hash_parms.pad_len, ctx->auth.alg,
845 				     ctx->auth.mode, rctx->total_sent,
846 				     stat_pad_len);
847 	}
848 
849 	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
850 			      spu_hdr_len);
851 	packet_dump("    prebuf: ", rctx->hash_carry, local_nbuf);
852 	flow_log("Data:\n");
853 	dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
854 	packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
855 
856 	/*
857 	 * Build mailbox message containing SPU request msg and rx buffers
858 	 * to catch response message
859 	 */
860 	memset(mssg, 0, sizeof(*mssg));
861 	mssg->type = BRCM_MESSAGE_SPU;
862 	mssg->ctx = rctx;	/* Will be returned in response */
863 
864 	/* Create rx scatterlist to catch result */
865 	err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
866 				     stat_pad_len);
867 	if (err)
868 		return err;
869 
870 	/* Create tx scatterlist containing SPU request message */
871 	tx_frag_num += rctx->src_nents;
872 	if (spu->spu_tx_status_len())
873 		tx_frag_num++;
874 	err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
875 				     local_nbuf, new_data_len, pad_len);
876 	if (err)
877 		return err;
878 
879 	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
880 	if (unlikely(err < 0))
881 		return err;
882 
883 	return -EINPROGRESS;
884 }
885 
886 /**
887  * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
888  * for an HMAC request.
889  * @req:  The HMAC request from the crypto API
890  * @ctx:  The session context
891  *
892  * Return: 0 if synchronous hash operation successful
893  *         -EINVAL if the hash algo is unrecognized
894  *         any other value indicates an error
895  */
spu_hmac_outer_hash(struct ahash_request * req,struct iproc_ctx_s * ctx)896 static int spu_hmac_outer_hash(struct ahash_request *req,
897 			       struct iproc_ctx_s *ctx)
898 {
899 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
900 	unsigned int blocksize =
901 		crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
902 	int rc;
903 
904 	switch (ctx->auth.alg) {
905 	case HASH_ALG_MD5:
906 		rc = do_shash("md5", req->result, ctx->opad, blocksize,
907 			      req->result, ctx->digestsize, NULL, 0);
908 		break;
909 	case HASH_ALG_SHA1:
910 		rc = do_shash("sha1", req->result, ctx->opad, blocksize,
911 			      req->result, ctx->digestsize, NULL, 0);
912 		break;
913 	case HASH_ALG_SHA224:
914 		rc = do_shash("sha224", req->result, ctx->opad, blocksize,
915 			      req->result, ctx->digestsize, NULL, 0);
916 		break;
917 	case HASH_ALG_SHA256:
918 		rc = do_shash("sha256", req->result, ctx->opad, blocksize,
919 			      req->result, ctx->digestsize, NULL, 0);
920 		break;
921 	case HASH_ALG_SHA384:
922 		rc = do_shash("sha384", req->result, ctx->opad, blocksize,
923 			      req->result, ctx->digestsize, NULL, 0);
924 		break;
925 	case HASH_ALG_SHA512:
926 		rc = do_shash("sha512", req->result, ctx->opad, blocksize,
927 			      req->result, ctx->digestsize, NULL, 0);
928 		break;
929 	default:
930 		pr_err("%s() Error : unknown hmac type\n", __func__);
931 		rc = -EINVAL;
932 	}
933 	return rc;
934 }
935 
936 /**
937  * ahash_req_done() - Process a hash result from the SPU hardware.
938  * @rctx: Crypto request context
939  *
940  * Return: 0 if successful
941  *         < 0 if an error
942  */
ahash_req_done(struct iproc_reqctx_s * rctx)943 static int ahash_req_done(struct iproc_reqctx_s *rctx)
944 {
945 	struct spu_hw *spu = &iproc_priv.spu;
946 	struct crypto_async_request *areq = rctx->parent;
947 	struct ahash_request *req = ahash_request_cast(areq);
948 	struct iproc_ctx_s *ctx = rctx->ctx;
949 	int err;
950 
951 	memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
952 
953 	if (spu->spu_type == SPU_TYPE_SPUM) {
954 		/* byte swap the output from the UPDT function to network byte
955 		 * order
956 		 */
957 		if (ctx->auth.alg == HASH_ALG_MD5) {
958 			__swab32s((u32 *)req->result);
959 			__swab32s(((u32 *)req->result) + 1);
960 			__swab32s(((u32 *)req->result) + 2);
961 			__swab32s(((u32 *)req->result) + 3);
962 			__swab32s(((u32 *)req->result) + 4);
963 		}
964 	}
965 
966 	flow_dump("  digest ", req->result, ctx->digestsize);
967 
968 	/* if this an HMAC then do the outer hash */
969 	if (rctx->is_sw_hmac) {
970 		err = spu_hmac_outer_hash(req, ctx);
971 		if (err < 0)
972 			return err;
973 		flow_dump("  hmac: ", req->result, ctx->digestsize);
974 	}
975 
976 	if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) {
977 		atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
978 		atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
979 	} else {
980 		atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
981 		atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
982 	}
983 
984 	return 0;
985 }
986 
987 /**
988  * handle_ahash_resp() - Process a SPU response message for a hash request.
989  * Checks if the entire crypto API request has been processed, and if so,
990  * invokes post processing on the result.
991  * @rctx: Crypto request context
992  */
handle_ahash_resp(struct iproc_reqctx_s * rctx)993 static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
994 {
995 	struct iproc_ctx_s *ctx = rctx->ctx;
996 	struct crypto_async_request *areq = rctx->parent;
997 	struct ahash_request *req = ahash_request_cast(areq);
998 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
999 	unsigned int blocksize =
1000 		crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
1001 	/*
1002 	 * Save hash to use as input to next op if incremental. Might be copying
1003 	 * too much, but that's easier than figuring out actual digest size here
1004 	 */
1005 	memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1006 
1007 	flow_log("%s() blocksize:%u digestsize:%u\n",
1008 		 __func__, blocksize, ctx->digestsize);
1009 
1010 	atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1011 
1012 	if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1013 		ahash_req_done(rctx);
1014 }
1015 
1016 /**
1017  * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1018  * a SPU response message for an AEAD request. Includes buffers to catch SPU
1019  * message headers and the response data.
1020  * @mssg:	mailbox message containing the receive sg
1021  * @req:	Crypto API request
1022  * @rctx:	crypto request context
1023  * @rx_frag_num: number of scatterlist elements required to hold the
1024  *		SPU response message
1025  * @assoc_len:	Length of associated data included in the crypto request
1026  * @ret_iv_len: Length of IV returned in response
1027  * @resp_len:	Number of bytes of response data expected to be written to
1028  *              dst buffer from crypto API
1029  * @digestsize: Length of hash digest, in bytes
1030  * @stat_pad_len: Number of bytes required to pad the STAT field to
1031  *		a 4-byte boundary
1032  *
1033  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1034  * when the request completes, whether the request is handled successfully or
1035  * there is an error.
1036  *
1037  * Returns:
1038  *   0 if successful
1039  *   < 0 if an error
1040  */
spu_aead_rx_sg_create(struct brcm_message * mssg,struct aead_request * req,struct iproc_reqctx_s * rctx,u8 rx_frag_num,unsigned int assoc_len,u32 ret_iv_len,unsigned int resp_len,unsigned int digestsize,u32 stat_pad_len)1041 static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1042 				 struct aead_request *req,
1043 				 struct iproc_reqctx_s *rctx,
1044 				 u8 rx_frag_num,
1045 				 unsigned int assoc_len,
1046 				 u32 ret_iv_len, unsigned int resp_len,
1047 				 unsigned int digestsize, u32 stat_pad_len)
1048 {
1049 	struct spu_hw *spu = &iproc_priv.spu;
1050 	struct scatterlist *sg;	/* used to build sgs in mbox message */
1051 	struct iproc_ctx_s *ctx = rctx->ctx;
1052 	u32 datalen;		/* Number of bytes of response data expected */
1053 	u32 assoc_buf_len;
1054 	u8 data_padlen = 0;
1055 
1056 	if (ctx->is_rfc4543) {
1057 		/* RFC4543: only pad after data, not after AAD */
1058 		data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1059 							  assoc_len + resp_len);
1060 		assoc_buf_len = assoc_len;
1061 	} else {
1062 		data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1063 							  resp_len);
1064 		assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1065 						assoc_len, ret_iv_len,
1066 						rctx->is_encrypt);
1067 	}
1068 
1069 	if (ctx->cipher.mode == CIPHER_MODE_CCM)
1070 		/* ICV (after data) must be in the next 32-bit word for CCM */
1071 		data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1072 							 resp_len +
1073 							 data_padlen);
1074 
1075 	if (data_padlen)
1076 		/* have to catch gcm pad in separate buffer */
1077 		rx_frag_num++;
1078 
1079 	mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
1080 				rctx->gfp);
1081 	if (!mssg->spu.dst)
1082 		return -ENOMEM;
1083 
1084 	sg = mssg->spu.dst;
1085 	sg_init_table(sg, rx_frag_num);
1086 
1087 	/* Space for SPU message header */
1088 	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1089 
1090 	if (assoc_buf_len) {
1091 		/*
1092 		 * Don't write directly to req->dst, because SPU may pad the
1093 		 * assoc data in the response
1094 		 */
1095 		memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1096 		sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1097 	}
1098 
1099 	if (resp_len) {
1100 		/*
1101 		 * Copy in each dst sg entry from request, up to chunksize.
1102 		 * dst sg catches just the data. digest caught in separate buf.
1103 		 */
1104 		datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1105 					 rctx->dst_nents, resp_len);
1106 		if (datalen < (resp_len)) {
1107 			pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1108 			       __func__, resp_len, datalen);
1109 			return -EFAULT;
1110 		}
1111 	}
1112 
1113 	/* If GCM/CCM data is padded, catch padding in separate buffer */
1114 	if (data_padlen) {
1115 		memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1116 		sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1117 	}
1118 
1119 	/* Always catch ICV in separate buffer */
1120 	sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1121 
1122 	flow_log("stat_pad_len %u\n", stat_pad_len);
1123 	if (stat_pad_len) {
1124 		memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1125 		sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1126 	}
1127 
1128 	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1129 	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1130 
1131 	return 0;
1132 }
1133 
1134 /**
1135  * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1136  * SPU request message for an AEAD request. Includes SPU message headers and the
1137  * request data.
1138  * @mssg:	mailbox message containing the transmit sg
1139  * @rctx:	crypto request context
1140  * @tx_frag_num: number of scatterlist elements required to construct the
1141  *		SPU request message
1142  * @spu_hdr_len: length of SPU message header in bytes
1143  * @assoc:	crypto API associated data scatterlist
1144  * @assoc_len:	length of associated data
1145  * @assoc_nents: number of scatterlist entries containing assoc data
1146  * @aead_iv_len: length of AEAD IV, if included
1147  * @chunksize:	Number of bytes of request data
1148  * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1149  * @pad_len:	Number of pad bytes
1150  * @incl_icv:	If true, write separate ICV buffer after data and
1151  *              any padding
1152  *
1153  * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1154  * when the request completes, whether the request is handled successfully or
1155  * there is an error.
1156  *
1157  * Return:
1158  *   0 if successful
1159  *   < 0 if an error
1160  */
spu_aead_tx_sg_create(struct brcm_message * mssg,struct iproc_reqctx_s * rctx,u8 tx_frag_num,u32 spu_hdr_len,struct scatterlist * assoc,unsigned int assoc_len,int assoc_nents,unsigned int aead_iv_len,unsigned int chunksize,u32 aad_pad_len,u32 pad_len,bool incl_icv)1161 static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1162 				 struct iproc_reqctx_s *rctx,
1163 				 u8 tx_frag_num,
1164 				 u32 spu_hdr_len,
1165 				 struct scatterlist *assoc,
1166 				 unsigned int assoc_len,
1167 				 int assoc_nents,
1168 				 unsigned int aead_iv_len,
1169 				 unsigned int chunksize,
1170 				 u32 aad_pad_len, u32 pad_len, bool incl_icv)
1171 {
1172 	struct spu_hw *spu = &iproc_priv.spu;
1173 	struct scatterlist *sg;	/* used to build sgs in mbox message */
1174 	struct scatterlist *assoc_sg = assoc;
1175 	struct iproc_ctx_s *ctx = rctx->ctx;
1176 	u32 datalen;		/* Number of bytes of data to write */
1177 	u32 written;		/* Number of bytes of data written */
1178 	u32 assoc_offset = 0;
1179 	u32 stat_len;
1180 
1181 	mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
1182 				rctx->gfp);
1183 	if (!mssg->spu.src)
1184 		return -ENOMEM;
1185 
1186 	sg = mssg->spu.src;
1187 	sg_init_table(sg, tx_frag_num);
1188 
1189 	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1190 		   BCM_HDR_LEN + spu_hdr_len);
1191 
1192 	if (assoc_len) {
1193 		/* Copy in each associated data sg entry from request */
1194 		written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1195 					 assoc_nents, assoc_len);
1196 		if (written < assoc_len) {
1197 			pr_err("%s(): failed to copy assoc sg to mbox msg",
1198 			       __func__);
1199 			return -EFAULT;
1200 		}
1201 	}
1202 
1203 	if (aead_iv_len)
1204 		sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1205 
1206 	if (aad_pad_len) {
1207 		memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1208 		sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1209 	}
1210 
1211 	datalen = chunksize;
1212 	if ((chunksize > ctx->digestsize) && incl_icv)
1213 		datalen -= ctx->digestsize;
1214 	if (datalen) {
1215 		/* For aead, a single msg should consume the entire src sg */
1216 		written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1217 					 rctx->src_nents, datalen);
1218 		if (written < datalen) {
1219 			pr_err("%s(): failed to copy src sg to mbox msg",
1220 			       __func__);
1221 			return -EFAULT;
1222 		}
1223 	}
1224 
1225 	if (pad_len) {
1226 		memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1227 		sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1228 	}
1229 
1230 	if (incl_icv)
1231 		sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1232 
1233 	stat_len = spu->spu_tx_status_len();
1234 	if (stat_len) {
1235 		memset(rctx->msg_buf.tx_stat, 0, stat_len);
1236 		sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1237 	}
1238 	return 0;
1239 }
1240 
1241 /**
1242  * handle_aead_req() - Submit a SPU request message for the next chunk of the
1243  * current AEAD request.
1244  * @rctx:  Crypto request context
1245  *
1246  * Unlike other operation types, we assume the length of the request fits in
1247  * a single SPU request message. aead_enqueue() makes sure this is true.
1248  * Comments for other op types regarding threads applies here as well.
1249  *
1250  * Unlike incremental hash ops, where the spu returns the entire hash for
1251  * truncated algs like sha-224, the SPU returns just the truncated hash in
1252  * response to aead requests. So digestsize is always ctx->digestsize here.
1253  *
1254  * Return: -EINPROGRESS: crypto request has been accepted and result will be
1255  *			 returned asynchronously
1256  *         Any other value indicates an error
1257  */
handle_aead_req(struct iproc_reqctx_s * rctx)1258 static int handle_aead_req(struct iproc_reqctx_s *rctx)
1259 {
1260 	struct spu_hw *spu = &iproc_priv.spu;
1261 	struct crypto_async_request *areq = rctx->parent;
1262 	struct aead_request *req = container_of(areq,
1263 						struct aead_request, base);
1264 	struct iproc_ctx_s *ctx = rctx->ctx;
1265 	int err;
1266 	unsigned int chunksize;
1267 	unsigned int resp_len;
1268 	u32 spu_hdr_len;
1269 	u32 db_size;
1270 	u32 stat_pad_len;
1271 	u32 pad_len;
1272 	struct brcm_message *mssg;	/* mailbox message */
1273 	struct spu_request_opts req_opts;
1274 	struct spu_cipher_parms cipher_parms;
1275 	struct spu_hash_parms hash_parms;
1276 	struct spu_aead_parms aead_parms;
1277 	int assoc_nents = 0;
1278 	bool incl_icv = false;
1279 	unsigned int digestsize = ctx->digestsize;
1280 
1281 	/* number of entries in src and dst sg. Always includes SPU msg header.
1282 	 */
1283 	u8 rx_frag_num = 2;	/* and STATUS */
1284 	u8 tx_frag_num = 1;
1285 
1286 	/* doing the whole thing at once */
1287 	chunksize = rctx->total_todo;
1288 
1289 	flow_log("%s: chunksize %u\n", __func__, chunksize);
1290 
1291 	memset(&req_opts, 0, sizeof(req_opts));
1292 	memset(&hash_parms, 0, sizeof(hash_parms));
1293 	memset(&aead_parms, 0, sizeof(aead_parms));
1294 
1295 	req_opts.is_inbound = !(rctx->is_encrypt);
1296 	req_opts.auth_first = ctx->auth_first;
1297 	req_opts.is_aead = true;
1298 	req_opts.is_esp = ctx->is_esp;
1299 
1300 	cipher_parms.alg = ctx->cipher.alg;
1301 	cipher_parms.mode = ctx->cipher.mode;
1302 	cipher_parms.type = ctx->cipher_type;
1303 	cipher_parms.key_buf = ctx->enckey;
1304 	cipher_parms.key_len = ctx->enckeylen;
1305 	cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1306 	cipher_parms.iv_len = rctx->iv_ctr_len;
1307 
1308 	hash_parms.alg = ctx->auth.alg;
1309 	hash_parms.mode = ctx->auth.mode;
1310 	hash_parms.type = HASH_TYPE_NONE;
1311 	hash_parms.key_buf = (u8 *)ctx->authkey;
1312 	hash_parms.key_len = ctx->authkeylen;
1313 	hash_parms.digestsize = digestsize;
1314 
1315 	if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1316 	    (ctx->authkeylen < SHA224_DIGEST_SIZE))
1317 		hash_parms.key_len = SHA224_DIGEST_SIZE;
1318 
1319 	aead_parms.assoc_size = req->assoclen;
1320 	if (ctx->is_esp && !ctx->is_rfc4543) {
1321 		/*
1322 		 * 8-byte IV is included assoc data in request. SPU2
1323 		 * expects AAD to include just SPI and seqno. So
1324 		 * subtract off the IV len.
1325 		 */
1326 		aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE;
1327 
1328 		if (rctx->is_encrypt) {
1329 			aead_parms.return_iv = true;
1330 			aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE;
1331 			aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1332 		}
1333 	} else {
1334 		aead_parms.ret_iv_len = 0;
1335 	}
1336 
1337 	/*
1338 	 * Count number of sg entries from the crypto API request that are to
1339 	 * be included in this mailbox message. For dst sg, don't count space
1340 	 * for digest. Digest gets caught in a separate buffer and copied back
1341 	 * to dst sg when processing response.
1342 	 */
1343 	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1344 	rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1345 	if (aead_parms.assoc_size)
1346 		assoc_nents = spu_sg_count(rctx->assoc, 0,
1347 					   aead_parms.assoc_size);
1348 
1349 	mssg = &rctx->mb_mssg;
1350 
1351 	rctx->total_sent = chunksize;
1352 	rctx->src_sent = chunksize;
1353 	if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1354 				    aead_parms.assoc_size,
1355 				    aead_parms.ret_iv_len,
1356 				    rctx->is_encrypt))
1357 		rx_frag_num++;
1358 
1359 	aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1360 						rctx->iv_ctr_len);
1361 
1362 	if (ctx->auth.alg == HASH_ALG_AES)
1363 		hash_parms.type = (enum hash_type)ctx->cipher_type;
1364 
1365 	/* General case AAD padding (CCM and RFC4543 special cases below) */
1366 	aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1367 						 aead_parms.assoc_size);
1368 
1369 	/* General case data padding (CCM decrypt special case below) */
1370 	aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1371 							   chunksize);
1372 
1373 	if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1374 		/*
1375 		 * for CCM, AAD len + 2 (rather than AAD len) needs to be
1376 		 * 128-bit aligned
1377 		 */
1378 		aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1379 					 ctx->cipher.mode,
1380 					 aead_parms.assoc_size + 2);
1381 
1382 		/*
1383 		 * And when decrypting CCM, need to pad without including
1384 		 * size of ICV which is tacked on to end of chunk
1385 		 */
1386 		if (!rctx->is_encrypt)
1387 			aead_parms.data_pad_len =
1388 				spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1389 							chunksize - digestsize);
1390 
1391 		/* CCM also requires software to rewrite portions of IV: */
1392 		spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1393 				       chunksize, rctx->is_encrypt,
1394 				       ctx->is_esp);
1395 	}
1396 
1397 	if (ctx->is_rfc4543) {
1398 		/*
1399 		 * RFC4543: data is included in AAD, so don't pad after AAD
1400 		 * and pad data based on both AAD + data size
1401 		 */
1402 		aead_parms.aad_pad_len = 0;
1403 		if (!rctx->is_encrypt)
1404 			aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1405 					ctx->cipher.mode,
1406 					aead_parms.assoc_size + chunksize -
1407 					digestsize);
1408 		else
1409 			aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1410 					ctx->cipher.mode,
1411 					aead_parms.assoc_size + chunksize);
1412 
1413 		req_opts.is_rfc4543 = true;
1414 	}
1415 
1416 	if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1417 		incl_icv = true;
1418 		tx_frag_num++;
1419 		/* Copy ICV from end of src scatterlist to digest buf */
1420 		sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1421 				    req->assoclen + rctx->total_sent -
1422 				    digestsize);
1423 	}
1424 
1425 	atomic64_add(chunksize, &iproc_priv.bytes_out);
1426 
1427 	flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1428 
1429 	/* Prepend SPU header with type 3 BCM header */
1430 	memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1431 
1432 	spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1433 					      BCM_HDR_LEN, &req_opts,
1434 					      &cipher_parms, &hash_parms,
1435 					      &aead_parms, chunksize);
1436 
1437 	/* Determine total length of padding. Put all padding in one buffer. */
1438 	db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1439 				   chunksize, aead_parms.aad_pad_len,
1440 				   aead_parms.data_pad_len, 0);
1441 
1442 	stat_pad_len = spu->spu_wordalign_padlen(db_size);
1443 
1444 	if (stat_pad_len)
1445 		rx_frag_num++;
1446 	pad_len = aead_parms.data_pad_len + stat_pad_len;
1447 	if (pad_len) {
1448 		tx_frag_num++;
1449 		spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1450 				     aead_parms.data_pad_len, 0,
1451 				     ctx->auth.alg, ctx->auth.mode,
1452 				     rctx->total_sent, stat_pad_len);
1453 	}
1454 
1455 	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1456 			      spu_hdr_len);
1457 	dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1458 	packet_dump("    aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1459 	packet_log("BD:\n");
1460 	dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1461 	packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1462 
1463 	/*
1464 	 * Build mailbox message containing SPU request msg and rx buffers
1465 	 * to catch response message
1466 	 */
1467 	memset(mssg, 0, sizeof(*mssg));
1468 	mssg->type = BRCM_MESSAGE_SPU;
1469 	mssg->ctx = rctx;	/* Will be returned in response */
1470 
1471 	/* Create rx scatterlist to catch result */
1472 	rx_frag_num += rctx->dst_nents;
1473 	resp_len = chunksize;
1474 
1475 	/*
1476 	 * Always catch ICV in separate buffer. Have to for GCM/CCM because of
1477 	 * padding. Have to for SHA-224 and other truncated SHAs because SPU
1478 	 * sends entire digest back.
1479 	 */
1480 	rx_frag_num++;
1481 
1482 	if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
1483 	     (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1484 		/*
1485 		 * Input is ciphertxt plus ICV, but ICV not incl
1486 		 * in output.
1487 		 */
1488 		resp_len -= ctx->digestsize;
1489 		if (resp_len == 0)
1490 			/* no rx frags to catch output data */
1491 			rx_frag_num -= rctx->dst_nents;
1492 	}
1493 
1494 	err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1495 				    aead_parms.assoc_size,
1496 				    aead_parms.ret_iv_len, resp_len, digestsize,
1497 				    stat_pad_len);
1498 	if (err)
1499 		return err;
1500 
1501 	/* Create tx scatterlist containing SPU request message */
1502 	tx_frag_num += rctx->src_nents;
1503 	tx_frag_num += assoc_nents;
1504 	if (aead_parms.aad_pad_len)
1505 		tx_frag_num++;
1506 	if (aead_parms.iv_len)
1507 		tx_frag_num++;
1508 	if (spu->spu_tx_status_len())
1509 		tx_frag_num++;
1510 	err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1511 				    rctx->assoc, aead_parms.assoc_size,
1512 				    assoc_nents, aead_parms.iv_len, chunksize,
1513 				    aead_parms.aad_pad_len, pad_len, incl_icv);
1514 	if (err)
1515 		return err;
1516 
1517 	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
1518 	if (unlikely(err < 0))
1519 		return err;
1520 
1521 	return -EINPROGRESS;
1522 }
1523 
1524 /**
1525  * handle_aead_resp() - Process a SPU response message for an AEAD request.
1526  * @rctx:  Crypto request context
1527  */
handle_aead_resp(struct iproc_reqctx_s * rctx)1528 static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1529 {
1530 	struct spu_hw *spu = &iproc_priv.spu;
1531 	struct crypto_async_request *areq = rctx->parent;
1532 	struct aead_request *req = container_of(areq,
1533 						struct aead_request, base);
1534 	struct iproc_ctx_s *ctx = rctx->ctx;
1535 	u32 payload_len;
1536 	unsigned int icv_offset;
1537 	u32 result_len;
1538 
1539 	/* See how much data was returned */
1540 	payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1541 	flow_log("payload_len %u\n", payload_len);
1542 
1543 	/* only count payload */
1544 	atomic64_add(payload_len, &iproc_priv.bytes_in);
1545 
1546 	if (req->assoclen)
1547 		packet_dump("  assoc_data ", rctx->msg_buf.a.resp_aad,
1548 			    req->assoclen);
1549 
1550 	/*
1551 	 * Copy the ICV back to the destination
1552 	 * buffer. In decrypt case, SPU gives us back the digest, but crypto
1553 	 * API doesn't expect ICV in dst buffer.
1554 	 */
1555 	result_len = req->cryptlen;
1556 	if (rctx->is_encrypt) {
1557 		icv_offset = req->assoclen + rctx->total_sent;
1558 		packet_dump("  ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1559 		flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1560 		sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1561 				      ctx->digestsize, icv_offset);
1562 		result_len += ctx->digestsize;
1563 	}
1564 
1565 	packet_log("response data:  ");
1566 	dump_sg(req->dst, req->assoclen, result_len);
1567 
1568 	atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1569 	if (ctx->cipher.alg == CIPHER_ALG_AES) {
1570 		if (ctx->cipher.mode == CIPHER_MODE_CCM)
1571 			atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1572 		else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1573 			atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1574 		else
1575 			atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1576 	} else {
1577 		atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1578 	}
1579 }
1580 
1581 /**
1582  * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1583  * @rctx:  request context
1584  *
1585  * Mailbox scatterlists are allocated for each chunk. So free them after
1586  * processing each chunk.
1587  */
spu_chunk_cleanup(struct iproc_reqctx_s * rctx)1588 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1589 {
1590 	/* mailbox message used to tx request */
1591 	struct brcm_message *mssg = &rctx->mb_mssg;
1592 
1593 	kfree(mssg->spu.src);
1594 	kfree(mssg->spu.dst);
1595 	memset(mssg, 0, sizeof(struct brcm_message));
1596 }
1597 
1598 /**
1599  * finish_req() - Used to invoke the complete callback from the requester when
1600  * a request has been handled asynchronously.
1601  * @rctx:  Request context
1602  * @err:   Indicates whether the request was successful or not
1603  *
1604  * Ensures that cleanup has been done for request
1605  */
finish_req(struct iproc_reqctx_s * rctx,int err)1606 static void finish_req(struct iproc_reqctx_s *rctx, int err)
1607 {
1608 	struct crypto_async_request *areq = rctx->parent;
1609 
1610 	flow_log("%s() err:%d\n\n", __func__, err);
1611 
1612 	/* No harm done if already called */
1613 	spu_chunk_cleanup(rctx);
1614 
1615 	if (areq)
1616 		crypto_request_complete(areq, err);
1617 }
1618 
1619 /**
1620  * spu_rx_callback() - Callback from mailbox framework with a SPU response.
1621  * @cl:		mailbox client structure for SPU driver
1622  * @msg:	mailbox message containing SPU response
1623  */
spu_rx_callback(struct mbox_client * cl,void * msg)1624 static void spu_rx_callback(struct mbox_client *cl, void *msg)
1625 {
1626 	struct spu_hw *spu = &iproc_priv.spu;
1627 	struct brcm_message *mssg = msg;
1628 	struct iproc_reqctx_s *rctx;
1629 	int err;
1630 
1631 	rctx = mssg->ctx;
1632 	if (unlikely(!rctx)) {
1633 		/* This is fatal */
1634 		pr_err("%s(): no request context", __func__);
1635 		err = -EFAULT;
1636 		goto cb_finish;
1637 	}
1638 
1639 	/* process the SPU status */
1640 	err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1641 	if (err != 0) {
1642 		if (err == SPU_INVALID_ICV)
1643 			atomic_inc(&iproc_priv.bad_icv);
1644 		err = -EBADMSG;
1645 		goto cb_finish;
1646 	}
1647 
1648 	/* Process the SPU response message */
1649 	switch (rctx->ctx->alg->type) {
1650 	case CRYPTO_ALG_TYPE_SKCIPHER:
1651 		handle_skcipher_resp(rctx);
1652 		break;
1653 	case CRYPTO_ALG_TYPE_AHASH:
1654 		handle_ahash_resp(rctx);
1655 		break;
1656 	case CRYPTO_ALG_TYPE_AEAD:
1657 		handle_aead_resp(rctx);
1658 		break;
1659 	default:
1660 		err = -EINVAL;
1661 		goto cb_finish;
1662 	}
1663 
1664 	/*
1665 	 * If this response does not complete the request, then send the next
1666 	 * request chunk.
1667 	 */
1668 	if (rctx->total_sent < rctx->total_todo) {
1669 		/* Deallocate anything specific to previous chunk */
1670 		spu_chunk_cleanup(rctx);
1671 
1672 		switch (rctx->ctx->alg->type) {
1673 		case CRYPTO_ALG_TYPE_SKCIPHER:
1674 			err = handle_skcipher_req(rctx);
1675 			break;
1676 		case CRYPTO_ALG_TYPE_AHASH:
1677 			err = handle_ahash_req(rctx);
1678 			if (err == -EAGAIN)
1679 				/*
1680 				 * we saved data in hash carry, but tell crypto
1681 				 * API we successfully completed request.
1682 				 */
1683 				err = 0;
1684 			break;
1685 		case CRYPTO_ALG_TYPE_AEAD:
1686 			err = handle_aead_req(rctx);
1687 			break;
1688 		default:
1689 			err = -EINVAL;
1690 		}
1691 
1692 		if (err == -EINPROGRESS)
1693 			/* Successfully submitted request for next chunk */
1694 			return;
1695 	}
1696 
1697 cb_finish:
1698 	finish_req(rctx, err);
1699 }
1700 
1701 /* ==================== Kernel Cryptographic API ==================== */
1702 
1703 /**
1704  * skcipher_enqueue() - Handle skcipher encrypt or decrypt request.
1705  * @req:	Crypto API request
1706  * @encrypt:	true if encrypting; false if decrypting
1707  *
1708  * Return: -EINPROGRESS if request accepted and result will be returned
1709  *			asynchronously
1710  *	   < 0 if an error
1711  */
skcipher_enqueue(struct skcipher_request * req,bool encrypt)1712 static int skcipher_enqueue(struct skcipher_request *req, bool encrypt)
1713 {
1714 	struct iproc_reqctx_s *rctx = skcipher_request_ctx(req);
1715 	struct iproc_ctx_s *ctx =
1716 	    crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1717 	int err;
1718 
1719 	flow_log("%s() enc:%u\n", __func__, encrypt);
1720 
1721 	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1722 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1723 	rctx->parent = &req->base;
1724 	rctx->is_encrypt = encrypt;
1725 	rctx->bd_suppress = false;
1726 	rctx->total_todo = req->cryptlen;
1727 	rctx->src_sent = 0;
1728 	rctx->total_sent = 0;
1729 	rctx->total_received = 0;
1730 	rctx->ctx = ctx;
1731 
1732 	/* Initialize current position in src and dst scatterlists */
1733 	rctx->src_sg = req->src;
1734 	rctx->src_nents = 0;
1735 	rctx->src_skip = 0;
1736 	rctx->dst_sg = req->dst;
1737 	rctx->dst_nents = 0;
1738 	rctx->dst_skip = 0;
1739 
1740 	if (ctx->cipher.mode == CIPHER_MODE_CBC ||
1741 	    ctx->cipher.mode == CIPHER_MODE_CTR ||
1742 	    ctx->cipher.mode == CIPHER_MODE_OFB ||
1743 	    ctx->cipher.mode == CIPHER_MODE_XTS ||
1744 	    ctx->cipher.mode == CIPHER_MODE_GCM ||
1745 	    ctx->cipher.mode == CIPHER_MODE_CCM) {
1746 		rctx->iv_ctr_len =
1747 		    crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1748 		memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len);
1749 	} else {
1750 		rctx->iv_ctr_len = 0;
1751 	}
1752 
1753 	/* Choose a SPU to process this request */
1754 	rctx->chan_idx = select_channel();
1755 	err = handle_skcipher_req(rctx);
1756 	if (err != -EINPROGRESS)
1757 		/* synchronous result */
1758 		spu_chunk_cleanup(rctx);
1759 
1760 	return err;
1761 }
1762 
des_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)1763 static int des_setkey(struct crypto_skcipher *cipher, const u8 *key,
1764 		      unsigned int keylen)
1765 {
1766 	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1767 	int err;
1768 
1769 	err = verify_skcipher_des_key(cipher, key);
1770 	if (err)
1771 		return err;
1772 
1773 	ctx->cipher_type = CIPHER_TYPE_DES;
1774 	return 0;
1775 }
1776 
threedes_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)1777 static int threedes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1778 			   unsigned int keylen)
1779 {
1780 	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1781 	int err;
1782 
1783 	err = verify_skcipher_des3_key(cipher, key);
1784 	if (err)
1785 		return err;
1786 
1787 	ctx->cipher_type = CIPHER_TYPE_3DES;
1788 	return 0;
1789 }
1790 
aes_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)1791 static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1792 		      unsigned int keylen)
1793 {
1794 	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1795 
1796 	if (ctx->cipher.mode == CIPHER_MODE_XTS)
1797 		/* XTS includes two keys of equal length */
1798 		keylen = keylen / 2;
1799 
1800 	switch (keylen) {
1801 	case AES_KEYSIZE_128:
1802 		ctx->cipher_type = CIPHER_TYPE_AES128;
1803 		break;
1804 	case AES_KEYSIZE_192:
1805 		ctx->cipher_type = CIPHER_TYPE_AES192;
1806 		break;
1807 	case AES_KEYSIZE_256:
1808 		ctx->cipher_type = CIPHER_TYPE_AES256;
1809 		break;
1810 	default:
1811 		return -EINVAL;
1812 	}
1813 	WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1814 		((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1815 	return 0;
1816 }
1817 
skcipher_setkey(struct crypto_skcipher * cipher,const u8 * key,unsigned int keylen)1818 static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key,
1819 			     unsigned int keylen)
1820 {
1821 	struct spu_hw *spu = &iproc_priv.spu;
1822 	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1823 	struct spu_cipher_parms cipher_parms;
1824 	u32 alloc_len = 0;
1825 	int err;
1826 
1827 	flow_log("skcipher_setkey() keylen: %d\n", keylen);
1828 	flow_dump("  key: ", key, keylen);
1829 
1830 	switch (ctx->cipher.alg) {
1831 	case CIPHER_ALG_DES:
1832 		err = des_setkey(cipher, key, keylen);
1833 		break;
1834 	case CIPHER_ALG_3DES:
1835 		err = threedes_setkey(cipher, key, keylen);
1836 		break;
1837 	case CIPHER_ALG_AES:
1838 		err = aes_setkey(cipher, key, keylen);
1839 		break;
1840 	default:
1841 		pr_err("%s() Error: unknown cipher alg\n", __func__);
1842 		err = -EINVAL;
1843 	}
1844 	if (err)
1845 		return err;
1846 
1847 	memcpy(ctx->enckey, key, keylen);
1848 	ctx->enckeylen = keylen;
1849 
1850 	/* SPU needs XTS keys in the reverse order the crypto API presents */
1851 	if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1852 	    (ctx->cipher.mode == CIPHER_MODE_XTS)) {
1853 		unsigned int xts_keylen = keylen / 2;
1854 
1855 		memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1856 		memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1857 	}
1858 
1859 	if (spu->spu_type == SPU_TYPE_SPUM)
1860 		alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1861 	else if (spu->spu_type == SPU_TYPE_SPU2)
1862 		alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1863 	memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1864 	cipher_parms.iv_buf = NULL;
1865 	cipher_parms.iv_len = crypto_skcipher_ivsize(cipher);
1866 	flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1867 
1868 	cipher_parms.alg = ctx->cipher.alg;
1869 	cipher_parms.mode = ctx->cipher.mode;
1870 	cipher_parms.type = ctx->cipher_type;
1871 	cipher_parms.key_buf = ctx->enckey;
1872 	cipher_parms.key_len = ctx->enckeylen;
1873 
1874 	/* Prepend SPU request message with BCM header */
1875 	memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1876 	ctx->spu_req_hdr_len =
1877 	    spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1878 				     &cipher_parms);
1879 
1880 	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1881 							  ctx->enckeylen,
1882 							  false);
1883 
1884 	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1885 
1886 	return 0;
1887 }
1888 
skcipher_encrypt(struct skcipher_request * req)1889 static int skcipher_encrypt(struct skcipher_request *req)
1890 {
1891 	flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen);
1892 
1893 	return skcipher_enqueue(req, true);
1894 }
1895 
skcipher_decrypt(struct skcipher_request * req)1896 static int skcipher_decrypt(struct skcipher_request *req)
1897 {
1898 	flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen);
1899 	return skcipher_enqueue(req, false);
1900 }
1901 
ahash_enqueue(struct ahash_request * req)1902 static int ahash_enqueue(struct ahash_request *req)
1903 {
1904 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1905 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1906 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1907 	int err;
1908 	const char *alg_name;
1909 
1910 	flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
1911 
1912 	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1913 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1914 	rctx->parent = &req->base;
1915 	rctx->ctx = ctx;
1916 	rctx->bd_suppress = true;
1917 	memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
1918 
1919 	/* Initialize position in src scatterlist */
1920 	rctx->src_sg = req->src;
1921 	rctx->src_skip = 0;
1922 	rctx->src_nents = 0;
1923 	rctx->dst_sg = NULL;
1924 	rctx->dst_skip = 0;
1925 	rctx->dst_nents = 0;
1926 
1927 	/* SPU2 hardware does not compute hash of zero length data */
1928 	if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
1929 	    (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
1930 		alg_name = crypto_ahash_alg_name(tfm);
1931 		flow_log("Doing %sfinal %s zero-len hash request in software\n",
1932 			 rctx->is_final ? "" : "non-", alg_name);
1933 		err = do_shash((unsigned char *)alg_name, req->result,
1934 			       NULL, 0, NULL, 0, ctx->authkey,
1935 			       ctx->authkeylen);
1936 		if (err < 0)
1937 			flow_log("Hash request failed with error %d\n", err);
1938 		return err;
1939 	}
1940 	/* Choose a SPU to process this request */
1941 	rctx->chan_idx = select_channel();
1942 
1943 	err = handle_ahash_req(rctx);
1944 	if (err != -EINPROGRESS)
1945 		/* synchronous result */
1946 		spu_chunk_cleanup(rctx);
1947 
1948 	if (err == -EAGAIN)
1949 		/*
1950 		 * we saved data in hash carry, but tell crypto API
1951 		 * we successfully completed request.
1952 		 */
1953 		err = 0;
1954 
1955 	return err;
1956 }
1957 
__ahash_init(struct ahash_request * req)1958 static int __ahash_init(struct ahash_request *req)
1959 {
1960 	struct spu_hw *spu = &iproc_priv.spu;
1961 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1962 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1963 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1964 
1965 	flow_log("%s()\n", __func__);
1966 
1967 	/* Initialize the context */
1968 	rctx->hash_carry_len = 0;
1969 	rctx->is_final = 0;
1970 
1971 	rctx->total_todo = 0;
1972 	rctx->src_sent = 0;
1973 	rctx->total_sent = 0;
1974 	rctx->total_received = 0;
1975 
1976 	ctx->digestsize = crypto_ahash_digestsize(tfm);
1977 	/* If we add a hash whose digest is larger, catch it here. */
1978 	WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
1979 
1980 	rctx->is_sw_hmac = false;
1981 
1982 	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
1983 							  true);
1984 
1985 	return 0;
1986 }
1987 
1988 /**
1989  * spu_no_incr_hash() - Determine whether incremental hashing is supported.
1990  * @ctx:  Crypto session context
1991  *
1992  * SPU-2 does not support incremental hashing (we'll have to revisit and
1993  * condition based on chip revision or device tree entry if future versions do
1994  * support incremental hash)
1995  *
1996  * SPU-M also doesn't support incremental hashing of AES-XCBC
1997  *
1998  * Return: true if incremental hashing is not supported
1999  *         false otherwise
2000  */
spu_no_incr_hash(struct iproc_ctx_s * ctx)2001 static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
2002 {
2003 	struct spu_hw *spu = &iproc_priv.spu;
2004 
2005 	if (spu->spu_type == SPU_TYPE_SPU2)
2006 		return true;
2007 
2008 	if ((ctx->auth.alg == HASH_ALG_AES) &&
2009 	    (ctx->auth.mode == HASH_MODE_XCBC))
2010 		return true;
2011 
2012 	/* Otherwise, incremental hashing is supported */
2013 	return false;
2014 }
2015 
ahash_init(struct ahash_request * req)2016 static int ahash_init(struct ahash_request *req)
2017 {
2018 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2019 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2020 	const char *alg_name;
2021 	struct crypto_shash *hash;
2022 	int ret;
2023 	gfp_t gfp;
2024 
2025 	if (spu_no_incr_hash(ctx)) {
2026 		/*
2027 		 * If we get an incremental hashing request and it's not
2028 		 * supported by the hardware, we need to handle it in software
2029 		 * by calling synchronous hash functions.
2030 		 */
2031 		alg_name = crypto_ahash_alg_name(tfm);
2032 		hash = crypto_alloc_shash(alg_name, 0, 0);
2033 		if (IS_ERR(hash)) {
2034 			ret = PTR_ERR(hash);
2035 			goto err;
2036 		}
2037 
2038 		gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2039 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2040 		ctx->shash = kmalloc(sizeof(*ctx->shash) +
2041 				     crypto_shash_descsize(hash), gfp);
2042 		if (!ctx->shash) {
2043 			ret = -ENOMEM;
2044 			goto err_hash;
2045 		}
2046 		ctx->shash->tfm = hash;
2047 
2048 		/* Set the key using data we already have from setkey */
2049 		if (ctx->authkeylen > 0) {
2050 			ret = crypto_shash_setkey(hash, ctx->authkey,
2051 						  ctx->authkeylen);
2052 			if (ret)
2053 				goto err_shash;
2054 		}
2055 
2056 		/* Initialize hash w/ this key and other params */
2057 		ret = crypto_shash_init(ctx->shash);
2058 		if (ret)
2059 			goto err_shash;
2060 	} else {
2061 		/* Otherwise call the internal function which uses SPU hw */
2062 		ret = __ahash_init(req);
2063 	}
2064 
2065 	return ret;
2066 
2067 err_shash:
2068 	kfree(ctx->shash);
2069 err_hash:
2070 	crypto_free_shash(hash);
2071 err:
2072 	return ret;
2073 }
2074 
__ahash_update(struct ahash_request * req)2075 static int __ahash_update(struct ahash_request *req)
2076 {
2077 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2078 
2079 	flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2080 
2081 	if (!req->nbytes)
2082 		return 0;
2083 	rctx->total_todo += req->nbytes;
2084 	rctx->src_sent = 0;
2085 
2086 	return ahash_enqueue(req);
2087 }
2088 
ahash_update(struct ahash_request * req)2089 static int ahash_update(struct ahash_request *req)
2090 {
2091 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2092 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2093 	u8 *tmpbuf;
2094 	int ret;
2095 	int nents;
2096 	gfp_t gfp;
2097 
2098 	if (spu_no_incr_hash(ctx)) {
2099 		/*
2100 		 * If we get an incremental hashing request and it's not
2101 		 * supported by the hardware, we need to handle it in software
2102 		 * by calling synchronous hash functions.
2103 		 */
2104 		if (req->src)
2105 			nents = sg_nents(req->src);
2106 		else
2107 			return -EINVAL;
2108 
2109 		/* Copy data from req scatterlist to tmp buffer */
2110 		gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2111 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2112 		tmpbuf = kmalloc(req->nbytes, gfp);
2113 		if (!tmpbuf)
2114 			return -ENOMEM;
2115 
2116 		if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2117 				req->nbytes) {
2118 			kfree(tmpbuf);
2119 			return -EINVAL;
2120 		}
2121 
2122 		/* Call synchronous update */
2123 		ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2124 		kfree(tmpbuf);
2125 	} else {
2126 		/* Otherwise call the internal function which uses SPU hw */
2127 		ret = __ahash_update(req);
2128 	}
2129 
2130 	return ret;
2131 }
2132 
__ahash_final(struct ahash_request * req)2133 static int __ahash_final(struct ahash_request *req)
2134 {
2135 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2136 
2137 	flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2138 
2139 	rctx->is_final = 1;
2140 
2141 	return ahash_enqueue(req);
2142 }
2143 
ahash_final(struct ahash_request * req)2144 static int ahash_final(struct ahash_request *req)
2145 {
2146 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2147 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2148 	int ret;
2149 
2150 	if (spu_no_incr_hash(ctx)) {
2151 		/*
2152 		 * If we get an incremental hashing request and it's not
2153 		 * supported by the hardware, we need to handle it in software
2154 		 * by calling synchronous hash functions.
2155 		 */
2156 		ret = crypto_shash_final(ctx->shash, req->result);
2157 
2158 		/* Done with hash, can deallocate it now */
2159 		crypto_free_shash(ctx->shash->tfm);
2160 		kfree(ctx->shash);
2161 
2162 	} else {
2163 		/* Otherwise call the internal function which uses SPU hw */
2164 		ret = __ahash_final(req);
2165 	}
2166 
2167 	return ret;
2168 }
2169 
__ahash_finup(struct ahash_request * req)2170 static int __ahash_finup(struct ahash_request *req)
2171 {
2172 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2173 
2174 	flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2175 
2176 	rctx->total_todo += req->nbytes;
2177 	rctx->src_sent = 0;
2178 	rctx->is_final = 1;
2179 
2180 	return ahash_enqueue(req);
2181 }
2182 
ahash_finup(struct ahash_request * req)2183 static int ahash_finup(struct ahash_request *req)
2184 {
2185 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2186 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2187 	u8 *tmpbuf;
2188 	int ret;
2189 	int nents;
2190 	gfp_t gfp;
2191 
2192 	if (spu_no_incr_hash(ctx)) {
2193 		/*
2194 		 * If we get an incremental hashing request and it's not
2195 		 * supported by the hardware, we need to handle it in software
2196 		 * by calling synchronous hash functions.
2197 		 */
2198 		if (req->src) {
2199 			nents = sg_nents(req->src);
2200 		} else {
2201 			ret = -EINVAL;
2202 			goto ahash_finup_exit;
2203 		}
2204 
2205 		/* Copy data from req scatterlist to tmp buffer */
2206 		gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2207 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2208 		tmpbuf = kmalloc(req->nbytes, gfp);
2209 		if (!tmpbuf) {
2210 			ret = -ENOMEM;
2211 			goto ahash_finup_exit;
2212 		}
2213 
2214 		if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2215 				req->nbytes) {
2216 			ret = -EINVAL;
2217 			goto ahash_finup_free;
2218 		}
2219 
2220 		/* Call synchronous update */
2221 		ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2222 					 req->result);
2223 	} else {
2224 		/* Otherwise call the internal function which uses SPU hw */
2225 		return __ahash_finup(req);
2226 	}
2227 ahash_finup_free:
2228 	kfree(tmpbuf);
2229 
2230 ahash_finup_exit:
2231 	/* Done with hash, can deallocate it now */
2232 	crypto_free_shash(ctx->shash->tfm);
2233 	kfree(ctx->shash);
2234 	return ret;
2235 }
2236 
ahash_digest(struct ahash_request * req)2237 static int ahash_digest(struct ahash_request *req)
2238 {
2239 	int err;
2240 
2241 	flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2242 
2243 	/* whole thing at once */
2244 	err = __ahash_init(req);
2245 	if (!err)
2246 		err = __ahash_finup(req);
2247 
2248 	return err;
2249 }
2250 
ahash_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)2251 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
2252 			unsigned int keylen)
2253 {
2254 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2255 
2256 	flow_log("%s() ahash:%p key:%p keylen:%u\n",
2257 		 __func__, ahash, key, keylen);
2258 	flow_dump("  key: ", key, keylen);
2259 
2260 	if (ctx->auth.alg == HASH_ALG_AES) {
2261 		switch (keylen) {
2262 		case AES_KEYSIZE_128:
2263 			ctx->cipher_type = CIPHER_TYPE_AES128;
2264 			break;
2265 		case AES_KEYSIZE_192:
2266 			ctx->cipher_type = CIPHER_TYPE_AES192;
2267 			break;
2268 		case AES_KEYSIZE_256:
2269 			ctx->cipher_type = CIPHER_TYPE_AES256;
2270 			break;
2271 		default:
2272 			pr_err("%s() Error: Invalid key length\n", __func__);
2273 			return -EINVAL;
2274 		}
2275 	} else {
2276 		pr_err("%s() Error: unknown hash alg\n", __func__);
2277 		return -EINVAL;
2278 	}
2279 	memcpy(ctx->authkey, key, keylen);
2280 	ctx->authkeylen = keylen;
2281 
2282 	return 0;
2283 }
2284 
ahash_export(struct ahash_request * req,void * out)2285 static int ahash_export(struct ahash_request *req, void *out)
2286 {
2287 	const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2288 	struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out;
2289 
2290 	spu_exp->total_todo = rctx->total_todo;
2291 	spu_exp->total_sent = rctx->total_sent;
2292 	spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2293 	memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2294 	spu_exp->hash_carry_len = rctx->hash_carry_len;
2295 	memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2296 
2297 	return 0;
2298 }
2299 
ahash_import(struct ahash_request * req,const void * in)2300 static int ahash_import(struct ahash_request *req, const void *in)
2301 {
2302 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2303 	struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in;
2304 
2305 	rctx->total_todo = spu_exp->total_todo;
2306 	rctx->total_sent = spu_exp->total_sent;
2307 	rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2308 	memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2309 	rctx->hash_carry_len = spu_exp->hash_carry_len;
2310 	memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2311 
2312 	return 0;
2313 }
2314 
ahash_hmac_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)2315 static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
2316 			     unsigned int keylen)
2317 {
2318 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2319 	unsigned int blocksize =
2320 		crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2321 	unsigned int digestsize = crypto_ahash_digestsize(ahash);
2322 	unsigned int index;
2323 	int rc;
2324 
2325 	flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2326 		 __func__, ahash, key, keylen, blocksize, digestsize);
2327 	flow_dump("  key: ", key, keylen);
2328 
2329 	if (keylen > blocksize) {
2330 		switch (ctx->auth.alg) {
2331 		case HASH_ALG_MD5:
2332 			rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2333 				      0, NULL, 0);
2334 			break;
2335 		case HASH_ALG_SHA1:
2336 			rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2337 				      0, NULL, 0);
2338 			break;
2339 		case HASH_ALG_SHA224:
2340 			rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2341 				      0, NULL, 0);
2342 			break;
2343 		case HASH_ALG_SHA256:
2344 			rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2345 				      0, NULL, 0);
2346 			break;
2347 		case HASH_ALG_SHA384:
2348 			rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2349 				      0, NULL, 0);
2350 			break;
2351 		case HASH_ALG_SHA512:
2352 			rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2353 				      0, NULL, 0);
2354 			break;
2355 		case HASH_ALG_SHA3_224:
2356 			rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2357 				      NULL, 0, NULL, 0);
2358 			break;
2359 		case HASH_ALG_SHA3_256:
2360 			rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2361 				      NULL, 0, NULL, 0);
2362 			break;
2363 		case HASH_ALG_SHA3_384:
2364 			rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2365 				      NULL, 0, NULL, 0);
2366 			break;
2367 		case HASH_ALG_SHA3_512:
2368 			rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2369 				      NULL, 0, NULL, 0);
2370 			break;
2371 		default:
2372 			pr_err("%s() Error: unknown hash alg\n", __func__);
2373 			return -EINVAL;
2374 		}
2375 		if (rc < 0) {
2376 			pr_err("%s() Error %d computing shash for %s\n",
2377 			       __func__, rc, hash_alg_name[ctx->auth.alg]);
2378 			return rc;
2379 		}
2380 		ctx->authkeylen = digestsize;
2381 
2382 		flow_log("  keylen > digestsize... hashed\n");
2383 		flow_dump("  newkey: ", ctx->authkey, ctx->authkeylen);
2384 	} else {
2385 		memcpy(ctx->authkey, key, keylen);
2386 		ctx->authkeylen = keylen;
2387 	}
2388 
2389 	/*
2390 	 * Full HMAC operation in SPUM is not verified,
2391 	 * So keeping the generation of IPAD, OPAD and
2392 	 * outer hashing in software.
2393 	 */
2394 	if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2395 		memcpy(ctx->ipad, ctx->authkey, ctx->authkeylen);
2396 		memset(ctx->ipad + ctx->authkeylen, 0,
2397 		       blocksize - ctx->authkeylen);
2398 		ctx->authkeylen = 0;
2399 		unsafe_memcpy(ctx->opad, ctx->ipad, blocksize,
2400 			      "fortified memcpy causes -Wrestrict warning");
2401 
2402 		for (index = 0; index < blocksize; index++) {
2403 			ctx->ipad[index] ^= HMAC_IPAD_VALUE;
2404 			ctx->opad[index] ^= HMAC_OPAD_VALUE;
2405 		}
2406 
2407 		flow_dump("  ipad: ", ctx->ipad, blocksize);
2408 		flow_dump("  opad: ", ctx->opad, blocksize);
2409 	}
2410 	ctx->digestsize = digestsize;
2411 	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2412 
2413 	return 0;
2414 }
2415 
ahash_hmac_init(struct ahash_request * req)2416 static int ahash_hmac_init(struct ahash_request *req)
2417 {
2418 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2419 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2420 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2421 	unsigned int blocksize =
2422 			crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2423 
2424 	flow_log("ahash_hmac_init()\n");
2425 
2426 	/* init the context as a hash */
2427 	ahash_init(req);
2428 
2429 	if (!spu_no_incr_hash(ctx)) {
2430 		/* SPU-M can do incr hashing but needs sw for outer HMAC */
2431 		rctx->is_sw_hmac = true;
2432 		ctx->auth.mode = HASH_MODE_HASH;
2433 		/* start with a prepended ipad */
2434 		memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2435 		rctx->hash_carry_len = blocksize;
2436 		rctx->total_todo += blocksize;
2437 	}
2438 
2439 	return 0;
2440 }
2441 
ahash_hmac_update(struct ahash_request * req)2442 static int ahash_hmac_update(struct ahash_request *req)
2443 {
2444 	flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2445 
2446 	if (!req->nbytes)
2447 		return 0;
2448 
2449 	return ahash_update(req);
2450 }
2451 
ahash_hmac_final(struct ahash_request * req)2452 static int ahash_hmac_final(struct ahash_request *req)
2453 {
2454 	flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2455 
2456 	return ahash_final(req);
2457 }
2458 
ahash_hmac_finup(struct ahash_request * req)2459 static int ahash_hmac_finup(struct ahash_request *req)
2460 {
2461 	flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2462 
2463 	return ahash_finup(req);
2464 }
2465 
ahash_hmac_digest(struct ahash_request * req)2466 static int ahash_hmac_digest(struct ahash_request *req)
2467 {
2468 	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2469 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2470 	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2471 	unsigned int blocksize =
2472 			crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2473 
2474 	flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2475 
2476 	/* Perform initialization and then call finup */
2477 	__ahash_init(req);
2478 
2479 	if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2480 		/*
2481 		 * SPU2 supports full HMAC implementation in the
2482 		 * hardware, need not to generate IPAD, OPAD and
2483 		 * outer hash in software.
2484 		 * Only for hash key len > hash block size, SPU2
2485 		 * expects to perform hashing on the key, shorten
2486 		 * it to digest size and feed it as hash key.
2487 		 */
2488 		rctx->is_sw_hmac = false;
2489 		ctx->auth.mode = HASH_MODE_HMAC;
2490 	} else {
2491 		rctx->is_sw_hmac = true;
2492 		ctx->auth.mode = HASH_MODE_HASH;
2493 		/* start with a prepended ipad */
2494 		memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2495 		rctx->hash_carry_len = blocksize;
2496 		rctx->total_todo += blocksize;
2497 	}
2498 
2499 	return __ahash_finup(req);
2500 }
2501 
2502 /* aead helpers */
2503 
aead_need_fallback(struct aead_request * req)2504 static int aead_need_fallback(struct aead_request *req)
2505 {
2506 	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2507 	struct spu_hw *spu = &iproc_priv.spu;
2508 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2509 	struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2510 	u32 payload_len;
2511 
2512 	/*
2513 	 * SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2514 	 * and AAD are both 0 bytes long. So use fallback in this case.
2515 	 */
2516 	if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
2517 	     (ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2518 	    (req->assoclen == 0)) {
2519 		if ((rctx->is_encrypt && (req->cryptlen == 0)) ||
2520 		    (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2521 			flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2522 			return 1;
2523 		}
2524 	}
2525 
2526 	/* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2527 	if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2528 	    (spu->spu_type == SPU_TYPE_SPUM) &&
2529 	    (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2530 	    (ctx->digestsize != 16)) {
2531 		flow_log("%s() AES CCM needs fallback for digest size %d\n",
2532 			 __func__, ctx->digestsize);
2533 		return 1;
2534 	}
2535 
2536 	/*
2537 	 * SPU-M on NSP has an issue where AES-CCM hash is not correct
2538 	 * when AAD size is 0
2539 	 */
2540 	if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2541 	    (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2542 	    (req->assoclen == 0)) {
2543 		flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2544 			 __func__);
2545 		return 1;
2546 	}
2547 
2548 	/*
2549 	 * RFC4106 and RFC4543 cannot handle the case where AAD is other than
2550 	 * 16 or 20 bytes long. So use fallback in this case.
2551 	 */
2552 	if (ctx->cipher.mode == CIPHER_MODE_GCM &&
2553 	    ctx->cipher.alg == CIPHER_ALG_AES &&
2554 	    rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
2555 	    req->assoclen != 16 && req->assoclen != 20) {
2556 		flow_log("RFC4106/RFC4543 needs fallback for assoclen"
2557 			 " other than 16 or 20 bytes\n");
2558 		return 1;
2559 	}
2560 
2561 	payload_len = req->cryptlen;
2562 	if (spu->spu_type == SPU_TYPE_SPUM)
2563 		payload_len += req->assoclen;
2564 
2565 	flow_log("%s() payload len: %u\n", __func__, payload_len);
2566 
2567 	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2568 		return 0;
2569 	else
2570 		return payload_len > ctx->max_payload;
2571 }
2572 
aead_do_fallback(struct aead_request * req,bool is_encrypt)2573 static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2574 {
2575 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2576 	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2577 	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2578 	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2579 	struct aead_request *subreq;
2580 
2581 	flow_log("%s() enc:%u\n", __func__, is_encrypt);
2582 
2583 	if (!ctx->fallback_cipher)
2584 		return -EINVAL;
2585 
2586 	subreq = &rctx->req;
2587 	aead_request_set_tfm(subreq, ctx->fallback_cipher);
2588 	aead_request_set_callback(subreq, aead_request_flags(req),
2589 				  req->base.complete, req->base.data);
2590 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2591 			       req->iv);
2592 	aead_request_set_ad(subreq, req->assoclen);
2593 
2594 	return is_encrypt ? crypto_aead_encrypt(req) :
2595 			    crypto_aead_decrypt(req);
2596 }
2597 
aead_enqueue(struct aead_request * req,bool is_encrypt)2598 static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2599 {
2600 	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2601 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2602 	struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2603 	int err;
2604 
2605 	flow_log("%s() enc:%u\n", __func__, is_encrypt);
2606 
2607 	if (req->assoclen > MAX_ASSOC_SIZE) {
2608 		pr_err
2609 		    ("%s() Error: associated data too long. (%u > %u bytes)\n",
2610 		     __func__, req->assoclen, MAX_ASSOC_SIZE);
2611 		return -EINVAL;
2612 	}
2613 
2614 	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2615 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2616 	rctx->parent = &req->base;
2617 	rctx->is_encrypt = is_encrypt;
2618 	rctx->bd_suppress = false;
2619 	rctx->total_todo = req->cryptlen;
2620 	rctx->src_sent = 0;
2621 	rctx->total_sent = 0;
2622 	rctx->total_received = 0;
2623 	rctx->is_sw_hmac = false;
2624 	rctx->ctx = ctx;
2625 	memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2626 
2627 	/* assoc data is at start of src sg */
2628 	rctx->assoc = req->src;
2629 
2630 	/*
2631 	 * Init current position in src scatterlist to be after assoc data.
2632 	 * src_skip set to buffer offset where data begins. (Assoc data could
2633 	 * end in the middle of a buffer.)
2634 	 */
2635 	if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2636 			     &rctx->src_skip) < 0) {
2637 		pr_err("%s() Error: Unable to find start of src data\n",
2638 		       __func__);
2639 		return -EINVAL;
2640 	}
2641 
2642 	rctx->src_nents = 0;
2643 	rctx->dst_nents = 0;
2644 	if (req->dst == req->src) {
2645 		rctx->dst_sg = rctx->src_sg;
2646 		rctx->dst_skip = rctx->src_skip;
2647 	} else {
2648 		/*
2649 		 * Expect req->dst to have room for assoc data followed by
2650 		 * output data and ICV, if encrypt. So initialize dst_sg
2651 		 * to point beyond assoc len offset.
2652 		 */
2653 		if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2654 				     &rctx->dst_skip) < 0) {
2655 			pr_err("%s() Error: Unable to find start of dst data\n",
2656 			       __func__);
2657 			return -EINVAL;
2658 		}
2659 	}
2660 
2661 	if (ctx->cipher.mode == CIPHER_MODE_CBC ||
2662 	    ctx->cipher.mode == CIPHER_MODE_CTR ||
2663 	    ctx->cipher.mode == CIPHER_MODE_OFB ||
2664 	    ctx->cipher.mode == CIPHER_MODE_XTS ||
2665 	    ctx->cipher.mode == CIPHER_MODE_GCM) {
2666 		rctx->iv_ctr_len =
2667 			ctx->salt_len +
2668 			crypto_aead_ivsize(crypto_aead_reqtfm(req));
2669 	} else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2670 		rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2671 	} else {
2672 		rctx->iv_ctr_len = 0;
2673 	}
2674 
2675 	rctx->hash_carry_len = 0;
2676 
2677 	flow_log("  src sg: %p\n", req->src);
2678 	flow_log("  rctx->src_sg: %p, src_skip %u\n",
2679 		 rctx->src_sg, rctx->src_skip);
2680 	flow_log("  assoc:  %p, assoclen %u\n", rctx->assoc, req->assoclen);
2681 	flow_log("  dst sg: %p\n", req->dst);
2682 	flow_log("  rctx->dst_sg: %p, dst_skip %u\n",
2683 		 rctx->dst_sg, rctx->dst_skip);
2684 	flow_log("  iv_ctr_len:%u\n", rctx->iv_ctr_len);
2685 	flow_dump("  iv: ", req->iv, rctx->iv_ctr_len);
2686 	flow_log("  authkeylen:%u\n", ctx->authkeylen);
2687 	flow_log("  is_esp: %s\n", ctx->is_esp ? "yes" : "no");
2688 
2689 	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2690 		flow_log("  max_payload infinite");
2691 	else
2692 		flow_log("  max_payload: %u\n", ctx->max_payload);
2693 
2694 	if (unlikely(aead_need_fallback(req)))
2695 		return aead_do_fallback(req, is_encrypt);
2696 
2697 	/*
2698 	 * Do memory allocations for request after fallback check, because if we
2699 	 * do fallback, we won't call finish_req() to dealloc.
2700 	 */
2701 	if (rctx->iv_ctr_len) {
2702 		if (ctx->salt_len)
2703 			memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2704 			       ctx->salt, ctx->salt_len);
2705 		memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2706 		       req->iv,
2707 		       rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2708 	}
2709 
2710 	rctx->chan_idx = select_channel();
2711 	err = handle_aead_req(rctx);
2712 	if (err != -EINPROGRESS)
2713 		/* synchronous result */
2714 		spu_chunk_cleanup(rctx);
2715 
2716 	return err;
2717 }
2718 
aead_authenc_setkey(struct crypto_aead * cipher,const u8 * key,unsigned int keylen)2719 static int aead_authenc_setkey(struct crypto_aead *cipher,
2720 			       const u8 *key, unsigned int keylen)
2721 {
2722 	struct spu_hw *spu = &iproc_priv.spu;
2723 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2724 	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2725 	struct crypto_authenc_keys keys;
2726 	int ret;
2727 
2728 	flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2729 		 keylen);
2730 	flow_dump("  key: ", key, keylen);
2731 
2732 	ret = crypto_authenc_extractkeys(&keys, key, keylen);
2733 	if (ret)
2734 		goto badkey;
2735 
2736 	if (keys.enckeylen > MAX_KEY_SIZE ||
2737 	    keys.authkeylen > MAX_KEY_SIZE)
2738 		goto badkey;
2739 
2740 	ctx->enckeylen = keys.enckeylen;
2741 	ctx->authkeylen = keys.authkeylen;
2742 
2743 	memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
2744 	/* May end up padding auth key. So make sure it's zeroed. */
2745 	memset(ctx->authkey, 0, sizeof(ctx->authkey));
2746 	memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
2747 
2748 	switch (ctx->alg->cipher_info.alg) {
2749 	case CIPHER_ALG_DES:
2750 		if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
2751 			return -EINVAL;
2752 
2753 		ctx->cipher_type = CIPHER_TYPE_DES;
2754 		break;
2755 	case CIPHER_ALG_3DES:
2756 		if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
2757 			return -EINVAL;
2758 
2759 		ctx->cipher_type = CIPHER_TYPE_3DES;
2760 		break;
2761 	case CIPHER_ALG_AES:
2762 		switch (ctx->enckeylen) {
2763 		case AES_KEYSIZE_128:
2764 			ctx->cipher_type = CIPHER_TYPE_AES128;
2765 			break;
2766 		case AES_KEYSIZE_192:
2767 			ctx->cipher_type = CIPHER_TYPE_AES192;
2768 			break;
2769 		case AES_KEYSIZE_256:
2770 			ctx->cipher_type = CIPHER_TYPE_AES256;
2771 			break;
2772 		default:
2773 			goto badkey;
2774 		}
2775 		break;
2776 	default:
2777 		pr_err("%s() Error: Unknown cipher alg\n", __func__);
2778 		return -EINVAL;
2779 	}
2780 
2781 	flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2782 		 ctx->authkeylen);
2783 	flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2784 	flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2785 
2786 	/* setkey the fallback just in case we needto use it */
2787 	if (ctx->fallback_cipher) {
2788 		flow_log("  running fallback setkey()\n");
2789 
2790 		ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2791 		ctx->fallback_cipher->base.crt_flags |=
2792 		    tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2793 		ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen);
2794 		if (ret)
2795 			flow_log("  fallback setkey() returned:%d\n", ret);
2796 	}
2797 
2798 	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2799 							  ctx->enckeylen,
2800 							  false);
2801 
2802 	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2803 
2804 	return ret;
2805 
2806 badkey:
2807 	ctx->enckeylen = 0;
2808 	ctx->authkeylen = 0;
2809 	ctx->digestsize = 0;
2810 
2811 	return -EINVAL;
2812 }
2813 
aead_gcm_ccm_setkey(struct crypto_aead * cipher,const u8 * key,unsigned int keylen)2814 static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
2815 			       const u8 *key, unsigned int keylen)
2816 {
2817 	struct spu_hw *spu = &iproc_priv.spu;
2818 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2819 	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2820 
2821 	int ret = 0;
2822 
2823 	flow_log("%s() keylen:%u\n", __func__, keylen);
2824 	flow_dump("  key: ", key, keylen);
2825 
2826 	if (!ctx->is_esp)
2827 		ctx->digestsize = keylen;
2828 
2829 	ctx->enckeylen = keylen;
2830 	ctx->authkeylen = 0;
2831 
2832 	switch (ctx->enckeylen) {
2833 	case AES_KEYSIZE_128:
2834 		ctx->cipher_type = CIPHER_TYPE_AES128;
2835 		break;
2836 	case AES_KEYSIZE_192:
2837 		ctx->cipher_type = CIPHER_TYPE_AES192;
2838 		break;
2839 	case AES_KEYSIZE_256:
2840 		ctx->cipher_type = CIPHER_TYPE_AES256;
2841 		break;
2842 	default:
2843 		goto badkey;
2844 	}
2845 
2846 	memcpy(ctx->enckey, key, ctx->enckeylen);
2847 
2848 	flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2849 		 ctx->authkeylen);
2850 	flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2851 	flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2852 
2853 	/* setkey the fallback just in case we need to use it */
2854 	if (ctx->fallback_cipher) {
2855 		flow_log("  running fallback setkey()\n");
2856 
2857 		ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2858 		ctx->fallback_cipher->base.crt_flags |=
2859 		    tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2860 		ret = crypto_aead_setkey(ctx->fallback_cipher, key,
2861 					 keylen + ctx->salt_len);
2862 		if (ret)
2863 			flow_log("  fallback setkey() returned:%d\n", ret);
2864 	}
2865 
2866 	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2867 							  ctx->enckeylen,
2868 							  false);
2869 
2870 	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2871 
2872 	flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2873 		 ctx->authkeylen);
2874 
2875 	return ret;
2876 
2877 badkey:
2878 	ctx->enckeylen = 0;
2879 	ctx->authkeylen = 0;
2880 	ctx->digestsize = 0;
2881 
2882 	return -EINVAL;
2883 }
2884 
2885 /**
2886  * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
2887  * @cipher: AEAD structure
2888  * @key:    Key followed by 4 bytes of salt
2889  * @keylen: Length of key plus salt, in bytes
2890  *
2891  * Extracts salt from key and stores it to be prepended to IV on each request.
2892  * Digest is always 16 bytes
2893  *
2894  * Return: Value from generic gcm setkey.
2895  */
aead_gcm_esp_setkey(struct crypto_aead * cipher,const u8 * key,unsigned int keylen)2896 static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
2897 			       const u8 *key, unsigned int keylen)
2898 {
2899 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2900 
2901 	flow_log("%s\n", __func__);
2902 
2903 	if (keylen < GCM_ESP_SALT_SIZE)
2904 		return -EINVAL;
2905 
2906 	ctx->salt_len = GCM_ESP_SALT_SIZE;
2907 	ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2908 	memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2909 	keylen -= GCM_ESP_SALT_SIZE;
2910 	ctx->digestsize = GCM_ESP_DIGESTSIZE;
2911 	ctx->is_esp = true;
2912 	flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2913 
2914 	return aead_gcm_ccm_setkey(cipher, key, keylen);
2915 }
2916 
2917 /**
2918  * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
2919  * @cipher: AEAD structure
2920  * @key:    Key followed by 4 bytes of salt
2921  * @keylen: Length of key plus salt, in bytes
2922  *
2923  * Extracts salt from key and stores it to be prepended to IV on each request.
2924  * Digest is always 16 bytes
2925  *
2926  * Return: Value from generic gcm setkey.
2927  */
rfc4543_gcm_esp_setkey(struct crypto_aead * cipher,const u8 * key,unsigned int keylen)2928 static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
2929 				  const u8 *key, unsigned int keylen)
2930 {
2931 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2932 
2933 	flow_log("%s\n", __func__);
2934 
2935 	if (keylen < GCM_ESP_SALT_SIZE)
2936 		return -EINVAL;
2937 
2938 	ctx->salt_len = GCM_ESP_SALT_SIZE;
2939 	ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2940 	memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2941 	keylen -= GCM_ESP_SALT_SIZE;
2942 	ctx->digestsize = GCM_ESP_DIGESTSIZE;
2943 	ctx->is_esp = true;
2944 	ctx->is_rfc4543 = true;
2945 	flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2946 
2947 	return aead_gcm_ccm_setkey(cipher, key, keylen);
2948 }
2949 
2950 /**
2951  * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
2952  * @cipher: AEAD structure
2953  * @key:    Key followed by 4 bytes of salt
2954  * @keylen: Length of key plus salt, in bytes
2955  *
2956  * Extracts salt from key and stores it to be prepended to IV on each request.
2957  * Digest is always 16 bytes
2958  *
2959  * Return: Value from generic ccm setkey.
2960  */
aead_ccm_esp_setkey(struct crypto_aead * cipher,const u8 * key,unsigned int keylen)2961 static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
2962 			       const u8 *key, unsigned int keylen)
2963 {
2964 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2965 
2966 	flow_log("%s\n", __func__);
2967 
2968 	if (keylen < CCM_ESP_SALT_SIZE)
2969 		return -EINVAL;
2970 
2971 	ctx->salt_len = CCM_ESP_SALT_SIZE;
2972 	ctx->salt_offset = CCM_ESP_SALT_OFFSET;
2973 	memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
2974 	keylen -= CCM_ESP_SALT_SIZE;
2975 	ctx->is_esp = true;
2976 	flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
2977 
2978 	return aead_gcm_ccm_setkey(cipher, key, keylen);
2979 }
2980 
aead_setauthsize(struct crypto_aead * cipher,unsigned int authsize)2981 static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
2982 {
2983 	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2984 	int ret = 0;
2985 
2986 	flow_log("%s() authkeylen:%u authsize:%u\n",
2987 		 __func__, ctx->authkeylen, authsize);
2988 
2989 	ctx->digestsize = authsize;
2990 
2991 	/* setkey the fallback just in case we needto use it */
2992 	if (ctx->fallback_cipher) {
2993 		flow_log("  running fallback setauth()\n");
2994 
2995 		ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
2996 		if (ret)
2997 			flow_log("  fallback setauth() returned:%d\n", ret);
2998 	}
2999 
3000 	return ret;
3001 }
3002 
aead_encrypt(struct aead_request * req)3003 static int aead_encrypt(struct aead_request *req)
3004 {
3005 	flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3006 		 req->cryptlen);
3007 	dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3008 	flow_log("  assoc_len:%u\n", req->assoclen);
3009 
3010 	return aead_enqueue(req, true);
3011 }
3012 
aead_decrypt(struct aead_request * req)3013 static int aead_decrypt(struct aead_request *req)
3014 {
3015 	flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3016 	dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3017 	flow_log("  assoc_len:%u\n", req->assoclen);
3018 
3019 	return aead_enqueue(req, false);
3020 }
3021 
3022 /* ==================== Supported Cipher Algorithms ==================== */
3023 
3024 static struct iproc_alg_s driver_algs[] = {
3025 	{
3026 	 .type = CRYPTO_ALG_TYPE_AEAD,
3027 	 .alg.aead = {
3028 		 .base = {
3029 			.cra_name = "gcm(aes)",
3030 			.cra_driver_name = "gcm-aes-iproc",
3031 			.cra_blocksize = AES_BLOCK_SIZE,
3032 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3033 		 },
3034 		 .setkey = aead_gcm_ccm_setkey,
3035 		 .ivsize = GCM_AES_IV_SIZE,
3036 		.maxauthsize = AES_BLOCK_SIZE,
3037 	 },
3038 	 .cipher_info = {
3039 			 .alg = CIPHER_ALG_AES,
3040 			 .mode = CIPHER_MODE_GCM,
3041 			 },
3042 	 .auth_info = {
3043 		       .alg = HASH_ALG_AES,
3044 		       .mode = HASH_MODE_GCM,
3045 		       },
3046 	 .auth_first = 0,
3047 	 },
3048 	{
3049 	 .type = CRYPTO_ALG_TYPE_AEAD,
3050 	 .alg.aead = {
3051 		 .base = {
3052 			.cra_name = "ccm(aes)",
3053 			.cra_driver_name = "ccm-aes-iproc",
3054 			.cra_blocksize = AES_BLOCK_SIZE,
3055 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3056 		 },
3057 		 .setkey = aead_gcm_ccm_setkey,
3058 		 .ivsize = CCM_AES_IV_SIZE,
3059 		.maxauthsize = AES_BLOCK_SIZE,
3060 	 },
3061 	 .cipher_info = {
3062 			 .alg = CIPHER_ALG_AES,
3063 			 .mode = CIPHER_MODE_CCM,
3064 			 },
3065 	 .auth_info = {
3066 		       .alg = HASH_ALG_AES,
3067 		       .mode = HASH_MODE_CCM,
3068 		       },
3069 	 .auth_first = 0,
3070 	 },
3071 	{
3072 	 .type = CRYPTO_ALG_TYPE_AEAD,
3073 	 .alg.aead = {
3074 		 .base = {
3075 			.cra_name = "rfc4106(gcm(aes))",
3076 			.cra_driver_name = "gcm-aes-esp-iproc",
3077 			.cra_blocksize = AES_BLOCK_SIZE,
3078 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3079 		 },
3080 		 .setkey = aead_gcm_esp_setkey,
3081 		 .ivsize = GCM_RFC4106_IV_SIZE,
3082 		 .maxauthsize = AES_BLOCK_SIZE,
3083 	 },
3084 	 .cipher_info = {
3085 			 .alg = CIPHER_ALG_AES,
3086 			 .mode = CIPHER_MODE_GCM,
3087 			 },
3088 	 .auth_info = {
3089 		       .alg = HASH_ALG_AES,
3090 		       .mode = HASH_MODE_GCM,
3091 		       },
3092 	 .auth_first = 0,
3093 	 },
3094 	{
3095 	 .type = CRYPTO_ALG_TYPE_AEAD,
3096 	 .alg.aead = {
3097 		 .base = {
3098 			.cra_name = "rfc4309(ccm(aes))",
3099 			.cra_driver_name = "ccm-aes-esp-iproc",
3100 			.cra_blocksize = AES_BLOCK_SIZE,
3101 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3102 		 },
3103 		 .setkey = aead_ccm_esp_setkey,
3104 		 .ivsize = CCM_AES_IV_SIZE,
3105 		 .maxauthsize = AES_BLOCK_SIZE,
3106 	 },
3107 	 .cipher_info = {
3108 			 .alg = CIPHER_ALG_AES,
3109 			 .mode = CIPHER_MODE_CCM,
3110 			 },
3111 	 .auth_info = {
3112 		       .alg = HASH_ALG_AES,
3113 		       .mode = HASH_MODE_CCM,
3114 		       },
3115 	 .auth_first = 0,
3116 	 },
3117 	{
3118 	 .type = CRYPTO_ALG_TYPE_AEAD,
3119 	 .alg.aead = {
3120 		 .base = {
3121 			.cra_name = "rfc4543(gcm(aes))",
3122 			.cra_driver_name = "gmac-aes-esp-iproc",
3123 			.cra_blocksize = AES_BLOCK_SIZE,
3124 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3125 		 },
3126 		 .setkey = rfc4543_gcm_esp_setkey,
3127 		 .ivsize = GCM_RFC4106_IV_SIZE,
3128 		 .maxauthsize = AES_BLOCK_SIZE,
3129 	 },
3130 	 .cipher_info = {
3131 			 .alg = CIPHER_ALG_AES,
3132 			 .mode = CIPHER_MODE_GCM,
3133 			 },
3134 	 .auth_info = {
3135 		       .alg = HASH_ALG_AES,
3136 		       .mode = HASH_MODE_GCM,
3137 		       },
3138 	 .auth_first = 0,
3139 	 },
3140 	{
3141 	 .type = CRYPTO_ALG_TYPE_AEAD,
3142 	 .alg.aead = {
3143 		 .base = {
3144 			.cra_name = "authenc(hmac(md5),cbc(aes))",
3145 			.cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3146 			.cra_blocksize = AES_BLOCK_SIZE,
3147 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3148 				     CRYPTO_ALG_ASYNC |
3149 				     CRYPTO_ALG_ALLOCATES_MEMORY
3150 		 },
3151 		 .setkey = aead_authenc_setkey,
3152 		.ivsize = AES_BLOCK_SIZE,
3153 		.maxauthsize = MD5_DIGEST_SIZE,
3154 	 },
3155 	 .cipher_info = {
3156 			 .alg = CIPHER_ALG_AES,
3157 			 .mode = CIPHER_MODE_CBC,
3158 			 },
3159 	 .auth_info = {
3160 		       .alg = HASH_ALG_MD5,
3161 		       .mode = HASH_MODE_HMAC,
3162 		       },
3163 	 .auth_first = 0,
3164 	 },
3165 	{
3166 	 .type = CRYPTO_ALG_TYPE_AEAD,
3167 	 .alg.aead = {
3168 		 .base = {
3169 			.cra_name = "authenc(hmac(sha1),cbc(aes))",
3170 			.cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3171 			.cra_blocksize = AES_BLOCK_SIZE,
3172 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3173 				     CRYPTO_ALG_ASYNC |
3174 				     CRYPTO_ALG_ALLOCATES_MEMORY
3175 		 },
3176 		 .setkey = aead_authenc_setkey,
3177 		 .ivsize = AES_BLOCK_SIZE,
3178 		 .maxauthsize = SHA1_DIGEST_SIZE,
3179 	 },
3180 	 .cipher_info = {
3181 			 .alg = CIPHER_ALG_AES,
3182 			 .mode = CIPHER_MODE_CBC,
3183 			 },
3184 	 .auth_info = {
3185 		       .alg = HASH_ALG_SHA1,
3186 		       .mode = HASH_MODE_HMAC,
3187 		       },
3188 	 .auth_first = 0,
3189 	 },
3190 	{
3191 	 .type = CRYPTO_ALG_TYPE_AEAD,
3192 	 .alg.aead = {
3193 		 .base = {
3194 			.cra_name = "authenc(hmac(sha256),cbc(aes))",
3195 			.cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3196 			.cra_blocksize = AES_BLOCK_SIZE,
3197 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3198 				     CRYPTO_ALG_ASYNC |
3199 				     CRYPTO_ALG_ALLOCATES_MEMORY
3200 		 },
3201 		 .setkey = aead_authenc_setkey,
3202 		 .ivsize = AES_BLOCK_SIZE,
3203 		 .maxauthsize = SHA256_DIGEST_SIZE,
3204 	 },
3205 	 .cipher_info = {
3206 			 .alg = CIPHER_ALG_AES,
3207 			 .mode = CIPHER_MODE_CBC,
3208 			 },
3209 	 .auth_info = {
3210 		       .alg = HASH_ALG_SHA256,
3211 		       .mode = HASH_MODE_HMAC,
3212 		       },
3213 	 .auth_first = 0,
3214 	 },
3215 	{
3216 	 .type = CRYPTO_ALG_TYPE_AEAD,
3217 	 .alg.aead = {
3218 		 .base = {
3219 			.cra_name = "authenc(hmac(md5),cbc(des))",
3220 			.cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3221 			.cra_blocksize = DES_BLOCK_SIZE,
3222 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3223 				     CRYPTO_ALG_ASYNC |
3224 				     CRYPTO_ALG_ALLOCATES_MEMORY
3225 		 },
3226 		 .setkey = aead_authenc_setkey,
3227 		 .ivsize = DES_BLOCK_SIZE,
3228 		 .maxauthsize = MD5_DIGEST_SIZE,
3229 	 },
3230 	 .cipher_info = {
3231 			 .alg = CIPHER_ALG_DES,
3232 			 .mode = CIPHER_MODE_CBC,
3233 			 },
3234 	 .auth_info = {
3235 		       .alg = HASH_ALG_MD5,
3236 		       .mode = HASH_MODE_HMAC,
3237 		       },
3238 	 .auth_first = 0,
3239 	 },
3240 	{
3241 	 .type = CRYPTO_ALG_TYPE_AEAD,
3242 	 .alg.aead = {
3243 		 .base = {
3244 			.cra_name = "authenc(hmac(sha1),cbc(des))",
3245 			.cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3246 			.cra_blocksize = DES_BLOCK_SIZE,
3247 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3248 				     CRYPTO_ALG_ASYNC |
3249 				     CRYPTO_ALG_ALLOCATES_MEMORY
3250 		 },
3251 		 .setkey = aead_authenc_setkey,
3252 		 .ivsize = DES_BLOCK_SIZE,
3253 		 .maxauthsize = SHA1_DIGEST_SIZE,
3254 	 },
3255 	 .cipher_info = {
3256 			 .alg = CIPHER_ALG_DES,
3257 			 .mode = CIPHER_MODE_CBC,
3258 			 },
3259 	 .auth_info = {
3260 		       .alg = HASH_ALG_SHA1,
3261 		       .mode = HASH_MODE_HMAC,
3262 		       },
3263 	 .auth_first = 0,
3264 	 },
3265 	{
3266 	 .type = CRYPTO_ALG_TYPE_AEAD,
3267 	 .alg.aead = {
3268 		 .base = {
3269 			.cra_name = "authenc(hmac(sha224),cbc(des))",
3270 			.cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3271 			.cra_blocksize = DES_BLOCK_SIZE,
3272 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3273 				     CRYPTO_ALG_ASYNC |
3274 				     CRYPTO_ALG_ALLOCATES_MEMORY
3275 		 },
3276 		 .setkey = aead_authenc_setkey,
3277 		 .ivsize = DES_BLOCK_SIZE,
3278 		 .maxauthsize = SHA224_DIGEST_SIZE,
3279 	 },
3280 	 .cipher_info = {
3281 			 .alg = CIPHER_ALG_DES,
3282 			 .mode = CIPHER_MODE_CBC,
3283 			 },
3284 	 .auth_info = {
3285 		       .alg = HASH_ALG_SHA224,
3286 		       .mode = HASH_MODE_HMAC,
3287 		       },
3288 	 .auth_first = 0,
3289 	 },
3290 	{
3291 	 .type = CRYPTO_ALG_TYPE_AEAD,
3292 	 .alg.aead = {
3293 		 .base = {
3294 			.cra_name = "authenc(hmac(sha256),cbc(des))",
3295 			.cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3296 			.cra_blocksize = DES_BLOCK_SIZE,
3297 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3298 				     CRYPTO_ALG_ASYNC |
3299 				     CRYPTO_ALG_ALLOCATES_MEMORY
3300 		 },
3301 		 .setkey = aead_authenc_setkey,
3302 		 .ivsize = DES_BLOCK_SIZE,
3303 		 .maxauthsize = SHA256_DIGEST_SIZE,
3304 	 },
3305 	 .cipher_info = {
3306 			 .alg = CIPHER_ALG_DES,
3307 			 .mode = CIPHER_MODE_CBC,
3308 			 },
3309 	 .auth_info = {
3310 		       .alg = HASH_ALG_SHA256,
3311 		       .mode = HASH_MODE_HMAC,
3312 		       },
3313 	 .auth_first = 0,
3314 	 },
3315 	{
3316 	 .type = CRYPTO_ALG_TYPE_AEAD,
3317 	 .alg.aead = {
3318 		 .base = {
3319 			.cra_name = "authenc(hmac(sha384),cbc(des))",
3320 			.cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3321 			.cra_blocksize = DES_BLOCK_SIZE,
3322 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3323 				     CRYPTO_ALG_ASYNC |
3324 				     CRYPTO_ALG_ALLOCATES_MEMORY
3325 		 },
3326 		 .setkey = aead_authenc_setkey,
3327 		 .ivsize = DES_BLOCK_SIZE,
3328 		 .maxauthsize = SHA384_DIGEST_SIZE,
3329 	 },
3330 	 .cipher_info = {
3331 			 .alg = CIPHER_ALG_DES,
3332 			 .mode = CIPHER_MODE_CBC,
3333 			 },
3334 	 .auth_info = {
3335 		       .alg = HASH_ALG_SHA384,
3336 		       .mode = HASH_MODE_HMAC,
3337 		       },
3338 	 .auth_first = 0,
3339 	 },
3340 	{
3341 	 .type = CRYPTO_ALG_TYPE_AEAD,
3342 	 .alg.aead = {
3343 		 .base = {
3344 			.cra_name = "authenc(hmac(sha512),cbc(des))",
3345 			.cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3346 			.cra_blocksize = DES_BLOCK_SIZE,
3347 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3348 				     CRYPTO_ALG_ASYNC |
3349 				     CRYPTO_ALG_ALLOCATES_MEMORY
3350 		 },
3351 		 .setkey = aead_authenc_setkey,
3352 		 .ivsize = DES_BLOCK_SIZE,
3353 		 .maxauthsize = SHA512_DIGEST_SIZE,
3354 	 },
3355 	 .cipher_info = {
3356 			 .alg = CIPHER_ALG_DES,
3357 			 .mode = CIPHER_MODE_CBC,
3358 			 },
3359 	 .auth_info = {
3360 		       .alg = HASH_ALG_SHA512,
3361 		       .mode = HASH_MODE_HMAC,
3362 		       },
3363 	 .auth_first = 0,
3364 	 },
3365 	{
3366 	 .type = CRYPTO_ALG_TYPE_AEAD,
3367 	 .alg.aead = {
3368 		 .base = {
3369 			.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3370 			.cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3371 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3372 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3373 				     CRYPTO_ALG_ASYNC |
3374 				     CRYPTO_ALG_ALLOCATES_MEMORY
3375 		 },
3376 		 .setkey = aead_authenc_setkey,
3377 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3378 		 .maxauthsize = MD5_DIGEST_SIZE,
3379 	 },
3380 	 .cipher_info = {
3381 			 .alg = CIPHER_ALG_3DES,
3382 			 .mode = CIPHER_MODE_CBC,
3383 			 },
3384 	 .auth_info = {
3385 		       .alg = HASH_ALG_MD5,
3386 		       .mode = HASH_MODE_HMAC,
3387 		       },
3388 	 .auth_first = 0,
3389 	 },
3390 	{
3391 	 .type = CRYPTO_ALG_TYPE_AEAD,
3392 	 .alg.aead = {
3393 		 .base = {
3394 			.cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3395 			.cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3396 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3397 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3398 				     CRYPTO_ALG_ASYNC |
3399 				     CRYPTO_ALG_ALLOCATES_MEMORY
3400 		 },
3401 		 .setkey = aead_authenc_setkey,
3402 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3403 		 .maxauthsize = SHA1_DIGEST_SIZE,
3404 	 },
3405 	 .cipher_info = {
3406 			 .alg = CIPHER_ALG_3DES,
3407 			 .mode = CIPHER_MODE_CBC,
3408 			 },
3409 	 .auth_info = {
3410 		       .alg = HASH_ALG_SHA1,
3411 		       .mode = HASH_MODE_HMAC,
3412 		       },
3413 	 .auth_first = 0,
3414 	 },
3415 	{
3416 	 .type = CRYPTO_ALG_TYPE_AEAD,
3417 	 .alg.aead = {
3418 		 .base = {
3419 			.cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3420 			.cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3421 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3422 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3423 				     CRYPTO_ALG_ASYNC |
3424 				     CRYPTO_ALG_ALLOCATES_MEMORY
3425 		 },
3426 		 .setkey = aead_authenc_setkey,
3427 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3428 		 .maxauthsize = SHA224_DIGEST_SIZE,
3429 	 },
3430 	 .cipher_info = {
3431 			 .alg = CIPHER_ALG_3DES,
3432 			 .mode = CIPHER_MODE_CBC,
3433 			 },
3434 	 .auth_info = {
3435 		       .alg = HASH_ALG_SHA224,
3436 		       .mode = HASH_MODE_HMAC,
3437 		       },
3438 	 .auth_first = 0,
3439 	 },
3440 	{
3441 	 .type = CRYPTO_ALG_TYPE_AEAD,
3442 	 .alg.aead = {
3443 		 .base = {
3444 			.cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3445 			.cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3446 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3447 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3448 				     CRYPTO_ALG_ASYNC |
3449 				     CRYPTO_ALG_ALLOCATES_MEMORY
3450 		 },
3451 		 .setkey = aead_authenc_setkey,
3452 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3453 		 .maxauthsize = SHA256_DIGEST_SIZE,
3454 	 },
3455 	 .cipher_info = {
3456 			 .alg = CIPHER_ALG_3DES,
3457 			 .mode = CIPHER_MODE_CBC,
3458 			 },
3459 	 .auth_info = {
3460 		       .alg = HASH_ALG_SHA256,
3461 		       .mode = HASH_MODE_HMAC,
3462 		       },
3463 	 .auth_first = 0,
3464 	 },
3465 	{
3466 	 .type = CRYPTO_ALG_TYPE_AEAD,
3467 	 .alg.aead = {
3468 		 .base = {
3469 			.cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3470 			.cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3471 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3472 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3473 				     CRYPTO_ALG_ASYNC |
3474 				     CRYPTO_ALG_ALLOCATES_MEMORY
3475 		 },
3476 		 .setkey = aead_authenc_setkey,
3477 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3478 		 .maxauthsize = SHA384_DIGEST_SIZE,
3479 	 },
3480 	 .cipher_info = {
3481 			 .alg = CIPHER_ALG_3DES,
3482 			 .mode = CIPHER_MODE_CBC,
3483 			 },
3484 	 .auth_info = {
3485 		       .alg = HASH_ALG_SHA384,
3486 		       .mode = HASH_MODE_HMAC,
3487 		       },
3488 	 .auth_first = 0,
3489 	 },
3490 	{
3491 	 .type = CRYPTO_ALG_TYPE_AEAD,
3492 	 .alg.aead = {
3493 		 .base = {
3494 			.cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3495 			.cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3496 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3497 			.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3498 				     CRYPTO_ALG_ASYNC |
3499 				     CRYPTO_ALG_ALLOCATES_MEMORY
3500 		 },
3501 		 .setkey = aead_authenc_setkey,
3502 		 .ivsize = DES3_EDE_BLOCK_SIZE,
3503 		 .maxauthsize = SHA512_DIGEST_SIZE,
3504 	 },
3505 	 .cipher_info = {
3506 			 .alg = CIPHER_ALG_3DES,
3507 			 .mode = CIPHER_MODE_CBC,
3508 			 },
3509 	 .auth_info = {
3510 		       .alg = HASH_ALG_SHA512,
3511 		       .mode = HASH_MODE_HMAC,
3512 		       },
3513 	 .auth_first = 0,
3514 	 },
3515 
3516 /* SKCIPHER algorithms. */
3517 	{
3518 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3519 	 .alg.skcipher = {
3520 			.base.cra_name = "cbc(des)",
3521 			.base.cra_driver_name = "cbc-des-iproc",
3522 			.base.cra_blocksize = DES_BLOCK_SIZE,
3523 			.min_keysize = DES_KEY_SIZE,
3524 			.max_keysize = DES_KEY_SIZE,
3525 			.ivsize = DES_BLOCK_SIZE,
3526 			},
3527 	 .cipher_info = {
3528 			 .alg = CIPHER_ALG_DES,
3529 			 .mode = CIPHER_MODE_CBC,
3530 			 },
3531 	 .auth_info = {
3532 		       .alg = HASH_ALG_NONE,
3533 		       .mode = HASH_MODE_NONE,
3534 		       },
3535 	 },
3536 	{
3537 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3538 	 .alg.skcipher = {
3539 			.base.cra_name = "ecb(des)",
3540 			.base.cra_driver_name = "ecb-des-iproc",
3541 			.base.cra_blocksize = DES_BLOCK_SIZE,
3542 			.min_keysize = DES_KEY_SIZE,
3543 			.max_keysize = DES_KEY_SIZE,
3544 			.ivsize = 0,
3545 			},
3546 	 .cipher_info = {
3547 			 .alg = CIPHER_ALG_DES,
3548 			 .mode = CIPHER_MODE_ECB,
3549 			 },
3550 	 .auth_info = {
3551 		       .alg = HASH_ALG_NONE,
3552 		       .mode = HASH_MODE_NONE,
3553 		       },
3554 	 },
3555 	{
3556 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3557 	 .alg.skcipher = {
3558 			.base.cra_name = "cbc(des3_ede)",
3559 			.base.cra_driver_name = "cbc-des3-iproc",
3560 			.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3561 			.min_keysize = DES3_EDE_KEY_SIZE,
3562 			.max_keysize = DES3_EDE_KEY_SIZE,
3563 			.ivsize = DES3_EDE_BLOCK_SIZE,
3564 			},
3565 	 .cipher_info = {
3566 			 .alg = CIPHER_ALG_3DES,
3567 			 .mode = CIPHER_MODE_CBC,
3568 			 },
3569 	 .auth_info = {
3570 		       .alg = HASH_ALG_NONE,
3571 		       .mode = HASH_MODE_NONE,
3572 		       },
3573 	 },
3574 	{
3575 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3576 	 .alg.skcipher = {
3577 			.base.cra_name = "ecb(des3_ede)",
3578 			.base.cra_driver_name = "ecb-des3-iproc",
3579 			.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3580 			.min_keysize = DES3_EDE_KEY_SIZE,
3581 			.max_keysize = DES3_EDE_KEY_SIZE,
3582 			.ivsize = 0,
3583 			},
3584 	 .cipher_info = {
3585 			 .alg = CIPHER_ALG_3DES,
3586 			 .mode = CIPHER_MODE_ECB,
3587 			 },
3588 	 .auth_info = {
3589 		       .alg = HASH_ALG_NONE,
3590 		       .mode = HASH_MODE_NONE,
3591 		       },
3592 	 },
3593 	{
3594 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3595 	 .alg.skcipher = {
3596 			.base.cra_name = "cbc(aes)",
3597 			.base.cra_driver_name = "cbc-aes-iproc",
3598 			.base.cra_blocksize = AES_BLOCK_SIZE,
3599 			.min_keysize = AES_MIN_KEY_SIZE,
3600 			.max_keysize = AES_MAX_KEY_SIZE,
3601 			.ivsize = AES_BLOCK_SIZE,
3602 			},
3603 	 .cipher_info = {
3604 			 .alg = CIPHER_ALG_AES,
3605 			 .mode = CIPHER_MODE_CBC,
3606 			 },
3607 	 .auth_info = {
3608 		       .alg = HASH_ALG_NONE,
3609 		       .mode = HASH_MODE_NONE,
3610 		       },
3611 	 },
3612 	{
3613 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3614 	 .alg.skcipher = {
3615 			.base.cra_name = "ecb(aes)",
3616 			.base.cra_driver_name = "ecb-aes-iproc",
3617 			.base.cra_blocksize = AES_BLOCK_SIZE,
3618 			.min_keysize = AES_MIN_KEY_SIZE,
3619 			.max_keysize = AES_MAX_KEY_SIZE,
3620 			.ivsize = 0,
3621 			},
3622 	 .cipher_info = {
3623 			 .alg = CIPHER_ALG_AES,
3624 			 .mode = CIPHER_MODE_ECB,
3625 			 },
3626 	 .auth_info = {
3627 		       .alg = HASH_ALG_NONE,
3628 		       .mode = HASH_MODE_NONE,
3629 		       },
3630 	 },
3631 	{
3632 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3633 	 .alg.skcipher = {
3634 			.base.cra_name = "ctr(aes)",
3635 			.base.cra_driver_name = "ctr-aes-iproc",
3636 			.base.cra_blocksize = AES_BLOCK_SIZE,
3637 			.min_keysize = AES_MIN_KEY_SIZE,
3638 			.max_keysize = AES_MAX_KEY_SIZE,
3639 			.ivsize = AES_BLOCK_SIZE,
3640 			},
3641 	 .cipher_info = {
3642 			 .alg = CIPHER_ALG_AES,
3643 			 .mode = CIPHER_MODE_CTR,
3644 			 },
3645 	 .auth_info = {
3646 		       .alg = HASH_ALG_NONE,
3647 		       .mode = HASH_MODE_NONE,
3648 		       },
3649 	 },
3650 {
3651 	 .type = CRYPTO_ALG_TYPE_SKCIPHER,
3652 	 .alg.skcipher = {
3653 			.base.cra_name = "xts(aes)",
3654 			.base.cra_driver_name = "xts-aes-iproc",
3655 			.base.cra_blocksize = AES_BLOCK_SIZE,
3656 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
3657 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
3658 			.ivsize = AES_BLOCK_SIZE,
3659 			},
3660 	 .cipher_info = {
3661 			 .alg = CIPHER_ALG_AES,
3662 			 .mode = CIPHER_MODE_XTS,
3663 			 },
3664 	 .auth_info = {
3665 		       .alg = HASH_ALG_NONE,
3666 		       .mode = HASH_MODE_NONE,
3667 		       },
3668 	 },
3669 
3670 /* AHASH algorithms. */
3671 	{
3672 	 .type = CRYPTO_ALG_TYPE_AHASH,
3673 	 .alg.hash = {
3674 		      .halg.digestsize = MD5_DIGEST_SIZE,
3675 		      .halg.base = {
3676 				    .cra_name = "md5",
3677 				    .cra_driver_name = "md5-iproc",
3678 				    .cra_blocksize = MD5_BLOCK_WORDS * 4,
3679 				    .cra_flags = CRYPTO_ALG_ASYNC |
3680 						 CRYPTO_ALG_ALLOCATES_MEMORY,
3681 				}
3682 		      },
3683 	 .cipher_info = {
3684 			 .alg = CIPHER_ALG_NONE,
3685 			 .mode = CIPHER_MODE_NONE,
3686 			 },
3687 	 .auth_info = {
3688 		       .alg = HASH_ALG_MD5,
3689 		       .mode = HASH_MODE_HASH,
3690 		       },
3691 	 },
3692 	{
3693 	 .type = CRYPTO_ALG_TYPE_AHASH,
3694 	 .alg.hash = {
3695 		      .halg.digestsize = MD5_DIGEST_SIZE,
3696 		      .halg.base = {
3697 				    .cra_name = "hmac(md5)",
3698 				    .cra_driver_name = "hmac-md5-iproc",
3699 				    .cra_blocksize = MD5_BLOCK_WORDS * 4,
3700 				}
3701 		      },
3702 	 .cipher_info = {
3703 			 .alg = CIPHER_ALG_NONE,
3704 			 .mode = CIPHER_MODE_NONE,
3705 			 },
3706 	 .auth_info = {
3707 		       .alg = HASH_ALG_MD5,
3708 		       .mode = HASH_MODE_HMAC,
3709 		       },
3710 	 },
3711 	{.type = CRYPTO_ALG_TYPE_AHASH,
3712 	 .alg.hash = {
3713 		      .halg.digestsize = SHA1_DIGEST_SIZE,
3714 		      .halg.base = {
3715 				    .cra_name = "sha1",
3716 				    .cra_driver_name = "sha1-iproc",
3717 				    .cra_blocksize = SHA1_BLOCK_SIZE,
3718 				}
3719 		      },
3720 	 .cipher_info = {
3721 			 .alg = CIPHER_ALG_NONE,
3722 			 .mode = CIPHER_MODE_NONE,
3723 			 },
3724 	 .auth_info = {
3725 		       .alg = HASH_ALG_SHA1,
3726 		       .mode = HASH_MODE_HASH,
3727 		       },
3728 	 },
3729 	{.type = CRYPTO_ALG_TYPE_AHASH,
3730 	 .alg.hash = {
3731 		      .halg.digestsize = SHA1_DIGEST_SIZE,
3732 		      .halg.base = {
3733 				    .cra_name = "hmac(sha1)",
3734 				    .cra_driver_name = "hmac-sha1-iproc",
3735 				    .cra_blocksize = SHA1_BLOCK_SIZE,
3736 				}
3737 		      },
3738 	 .cipher_info = {
3739 			 .alg = CIPHER_ALG_NONE,
3740 			 .mode = CIPHER_MODE_NONE,
3741 			 },
3742 	 .auth_info = {
3743 		       .alg = HASH_ALG_SHA1,
3744 		       .mode = HASH_MODE_HMAC,
3745 		       },
3746 	 },
3747 	{.type = CRYPTO_ALG_TYPE_AHASH,
3748 	 .alg.hash = {
3749 			.halg.digestsize = SHA224_DIGEST_SIZE,
3750 			.halg.base = {
3751 				    .cra_name = "sha224",
3752 				    .cra_driver_name = "sha224-iproc",
3753 				    .cra_blocksize = SHA224_BLOCK_SIZE,
3754 			}
3755 		      },
3756 	 .cipher_info = {
3757 			 .alg = CIPHER_ALG_NONE,
3758 			 .mode = CIPHER_MODE_NONE,
3759 			 },
3760 	 .auth_info = {
3761 		       .alg = HASH_ALG_SHA224,
3762 		       .mode = HASH_MODE_HASH,
3763 		       },
3764 	 },
3765 	{.type = CRYPTO_ALG_TYPE_AHASH,
3766 	 .alg.hash = {
3767 		      .halg.digestsize = SHA224_DIGEST_SIZE,
3768 		      .halg.base = {
3769 				    .cra_name = "hmac(sha224)",
3770 				    .cra_driver_name = "hmac-sha224-iproc",
3771 				    .cra_blocksize = SHA224_BLOCK_SIZE,
3772 				}
3773 		      },
3774 	 .cipher_info = {
3775 			 .alg = CIPHER_ALG_NONE,
3776 			 .mode = CIPHER_MODE_NONE,
3777 			 },
3778 	 .auth_info = {
3779 		       .alg = HASH_ALG_SHA224,
3780 		       .mode = HASH_MODE_HMAC,
3781 		       },
3782 	 },
3783 	{.type = CRYPTO_ALG_TYPE_AHASH,
3784 	 .alg.hash = {
3785 		      .halg.digestsize = SHA256_DIGEST_SIZE,
3786 		      .halg.base = {
3787 				    .cra_name = "sha256",
3788 				    .cra_driver_name = "sha256-iproc",
3789 				    .cra_blocksize = SHA256_BLOCK_SIZE,
3790 				}
3791 		      },
3792 	 .cipher_info = {
3793 			 .alg = CIPHER_ALG_NONE,
3794 			 .mode = CIPHER_MODE_NONE,
3795 			 },
3796 	 .auth_info = {
3797 		       .alg = HASH_ALG_SHA256,
3798 		       .mode = HASH_MODE_HASH,
3799 		       },
3800 	 },
3801 	{.type = CRYPTO_ALG_TYPE_AHASH,
3802 	 .alg.hash = {
3803 		      .halg.digestsize = SHA256_DIGEST_SIZE,
3804 		      .halg.base = {
3805 				    .cra_name = "hmac(sha256)",
3806 				    .cra_driver_name = "hmac-sha256-iproc",
3807 				    .cra_blocksize = SHA256_BLOCK_SIZE,
3808 				}
3809 		      },
3810 	 .cipher_info = {
3811 			 .alg = CIPHER_ALG_NONE,
3812 			 .mode = CIPHER_MODE_NONE,
3813 			 },
3814 	 .auth_info = {
3815 		       .alg = HASH_ALG_SHA256,
3816 		       .mode = HASH_MODE_HMAC,
3817 		       },
3818 	 },
3819 	{
3820 	.type = CRYPTO_ALG_TYPE_AHASH,
3821 	 .alg.hash = {
3822 		      .halg.digestsize = SHA384_DIGEST_SIZE,
3823 		      .halg.base = {
3824 				    .cra_name = "sha384",
3825 				    .cra_driver_name = "sha384-iproc",
3826 				    .cra_blocksize = SHA384_BLOCK_SIZE,
3827 				}
3828 		      },
3829 	 .cipher_info = {
3830 			 .alg = CIPHER_ALG_NONE,
3831 			 .mode = CIPHER_MODE_NONE,
3832 			 },
3833 	 .auth_info = {
3834 		       .alg = HASH_ALG_SHA384,
3835 		       .mode = HASH_MODE_HASH,
3836 		       },
3837 	 },
3838 	{
3839 	 .type = CRYPTO_ALG_TYPE_AHASH,
3840 	 .alg.hash = {
3841 		      .halg.digestsize = SHA384_DIGEST_SIZE,
3842 		      .halg.base = {
3843 				    .cra_name = "hmac(sha384)",
3844 				    .cra_driver_name = "hmac-sha384-iproc",
3845 				    .cra_blocksize = SHA384_BLOCK_SIZE,
3846 				}
3847 		      },
3848 	 .cipher_info = {
3849 			 .alg = CIPHER_ALG_NONE,
3850 			 .mode = CIPHER_MODE_NONE,
3851 			 },
3852 	 .auth_info = {
3853 		       .alg = HASH_ALG_SHA384,
3854 		       .mode = HASH_MODE_HMAC,
3855 		       },
3856 	 },
3857 	{
3858 	 .type = CRYPTO_ALG_TYPE_AHASH,
3859 	 .alg.hash = {
3860 		      .halg.digestsize = SHA512_DIGEST_SIZE,
3861 		      .halg.base = {
3862 				    .cra_name = "sha512",
3863 				    .cra_driver_name = "sha512-iproc",
3864 				    .cra_blocksize = SHA512_BLOCK_SIZE,
3865 				}
3866 		      },
3867 	 .cipher_info = {
3868 			 .alg = CIPHER_ALG_NONE,
3869 			 .mode = CIPHER_MODE_NONE,
3870 			 },
3871 	 .auth_info = {
3872 		       .alg = HASH_ALG_SHA512,
3873 		       .mode = HASH_MODE_HASH,
3874 		       },
3875 	 },
3876 	{
3877 	 .type = CRYPTO_ALG_TYPE_AHASH,
3878 	 .alg.hash = {
3879 		      .halg.digestsize = SHA512_DIGEST_SIZE,
3880 		      .halg.base = {
3881 				    .cra_name = "hmac(sha512)",
3882 				    .cra_driver_name = "hmac-sha512-iproc",
3883 				    .cra_blocksize = SHA512_BLOCK_SIZE,
3884 				}
3885 		      },
3886 	 .cipher_info = {
3887 			 .alg = CIPHER_ALG_NONE,
3888 			 .mode = CIPHER_MODE_NONE,
3889 			 },
3890 	 .auth_info = {
3891 		       .alg = HASH_ALG_SHA512,
3892 		       .mode = HASH_MODE_HMAC,
3893 		       },
3894 	 },
3895 	{
3896 	 .type = CRYPTO_ALG_TYPE_AHASH,
3897 	 .alg.hash = {
3898 		      .halg.digestsize = SHA3_224_DIGEST_SIZE,
3899 		      .halg.base = {
3900 				    .cra_name = "sha3-224",
3901 				    .cra_driver_name = "sha3-224-iproc",
3902 				    .cra_blocksize = SHA3_224_BLOCK_SIZE,
3903 				}
3904 		      },
3905 	 .cipher_info = {
3906 			 .alg = CIPHER_ALG_NONE,
3907 			 .mode = CIPHER_MODE_NONE,
3908 			 },
3909 	 .auth_info = {
3910 		       .alg = HASH_ALG_SHA3_224,
3911 		       .mode = HASH_MODE_HASH,
3912 		       },
3913 	 },
3914 	{
3915 	 .type = CRYPTO_ALG_TYPE_AHASH,
3916 	 .alg.hash = {
3917 		      .halg.digestsize = SHA3_224_DIGEST_SIZE,
3918 		      .halg.base = {
3919 				    .cra_name = "hmac(sha3-224)",
3920 				    .cra_driver_name = "hmac-sha3-224-iproc",
3921 				    .cra_blocksize = SHA3_224_BLOCK_SIZE,
3922 				}
3923 		      },
3924 	 .cipher_info = {
3925 			 .alg = CIPHER_ALG_NONE,
3926 			 .mode = CIPHER_MODE_NONE,
3927 			 },
3928 	 .auth_info = {
3929 		       .alg = HASH_ALG_SHA3_224,
3930 		       .mode = HASH_MODE_HMAC
3931 		       },
3932 	 },
3933 	{
3934 	 .type = CRYPTO_ALG_TYPE_AHASH,
3935 	 .alg.hash = {
3936 		      .halg.digestsize = SHA3_256_DIGEST_SIZE,
3937 		      .halg.base = {
3938 				    .cra_name = "sha3-256",
3939 				    .cra_driver_name = "sha3-256-iproc",
3940 				    .cra_blocksize = SHA3_256_BLOCK_SIZE,
3941 				}
3942 		      },
3943 	 .cipher_info = {
3944 			 .alg = CIPHER_ALG_NONE,
3945 			 .mode = CIPHER_MODE_NONE,
3946 			 },
3947 	 .auth_info = {
3948 		       .alg = HASH_ALG_SHA3_256,
3949 		       .mode = HASH_MODE_HASH,
3950 		       },
3951 	 },
3952 	{
3953 	 .type = CRYPTO_ALG_TYPE_AHASH,
3954 	 .alg.hash = {
3955 		      .halg.digestsize = SHA3_256_DIGEST_SIZE,
3956 		      .halg.base = {
3957 				    .cra_name = "hmac(sha3-256)",
3958 				    .cra_driver_name = "hmac-sha3-256-iproc",
3959 				    .cra_blocksize = SHA3_256_BLOCK_SIZE,
3960 				}
3961 		      },
3962 	 .cipher_info = {
3963 			 .alg = CIPHER_ALG_NONE,
3964 			 .mode = CIPHER_MODE_NONE,
3965 			 },
3966 	 .auth_info = {
3967 		       .alg = HASH_ALG_SHA3_256,
3968 		       .mode = HASH_MODE_HMAC,
3969 		       },
3970 	 },
3971 	{
3972 	 .type = CRYPTO_ALG_TYPE_AHASH,
3973 	 .alg.hash = {
3974 		      .halg.digestsize = SHA3_384_DIGEST_SIZE,
3975 		      .halg.base = {
3976 				    .cra_name = "sha3-384",
3977 				    .cra_driver_name = "sha3-384-iproc",
3978 				    .cra_blocksize = SHA3_224_BLOCK_SIZE,
3979 				}
3980 		      },
3981 	 .cipher_info = {
3982 			 .alg = CIPHER_ALG_NONE,
3983 			 .mode = CIPHER_MODE_NONE,
3984 			 },
3985 	 .auth_info = {
3986 		       .alg = HASH_ALG_SHA3_384,
3987 		       .mode = HASH_MODE_HASH,
3988 		       },
3989 	 },
3990 	{
3991 	 .type = CRYPTO_ALG_TYPE_AHASH,
3992 	 .alg.hash = {
3993 		      .halg.digestsize = SHA3_384_DIGEST_SIZE,
3994 		      .halg.base = {
3995 				    .cra_name = "hmac(sha3-384)",
3996 				    .cra_driver_name = "hmac-sha3-384-iproc",
3997 				    .cra_blocksize = SHA3_384_BLOCK_SIZE,
3998 				}
3999 		      },
4000 	 .cipher_info = {
4001 			 .alg = CIPHER_ALG_NONE,
4002 			 .mode = CIPHER_MODE_NONE,
4003 			 },
4004 	 .auth_info = {
4005 		       .alg = HASH_ALG_SHA3_384,
4006 		       .mode = HASH_MODE_HMAC,
4007 		       },
4008 	 },
4009 	{
4010 	 .type = CRYPTO_ALG_TYPE_AHASH,
4011 	 .alg.hash = {
4012 		      .halg.digestsize = SHA3_512_DIGEST_SIZE,
4013 		      .halg.base = {
4014 				    .cra_name = "sha3-512",
4015 				    .cra_driver_name = "sha3-512-iproc",
4016 				    .cra_blocksize = SHA3_512_BLOCK_SIZE,
4017 				}
4018 		      },
4019 	 .cipher_info = {
4020 			 .alg = CIPHER_ALG_NONE,
4021 			 .mode = CIPHER_MODE_NONE,
4022 			 },
4023 	 .auth_info = {
4024 		       .alg = HASH_ALG_SHA3_512,
4025 		       .mode = HASH_MODE_HASH,
4026 		       },
4027 	 },
4028 	{
4029 	 .type = CRYPTO_ALG_TYPE_AHASH,
4030 	 .alg.hash = {
4031 		      .halg.digestsize = SHA3_512_DIGEST_SIZE,
4032 		      .halg.base = {
4033 				    .cra_name = "hmac(sha3-512)",
4034 				    .cra_driver_name = "hmac-sha3-512-iproc",
4035 				    .cra_blocksize = SHA3_512_BLOCK_SIZE,
4036 				}
4037 		      },
4038 	 .cipher_info = {
4039 			 .alg = CIPHER_ALG_NONE,
4040 			 .mode = CIPHER_MODE_NONE,
4041 			 },
4042 	 .auth_info = {
4043 		       .alg = HASH_ALG_SHA3_512,
4044 		       .mode = HASH_MODE_HMAC,
4045 		       },
4046 	 },
4047 	{
4048 	 .type = CRYPTO_ALG_TYPE_AHASH,
4049 	 .alg.hash = {
4050 		      .halg.digestsize = AES_BLOCK_SIZE,
4051 		      .halg.base = {
4052 				    .cra_name = "xcbc(aes)",
4053 				    .cra_driver_name = "xcbc-aes-iproc",
4054 				    .cra_blocksize = AES_BLOCK_SIZE,
4055 				}
4056 		      },
4057 	 .cipher_info = {
4058 			 .alg = CIPHER_ALG_NONE,
4059 			 .mode = CIPHER_MODE_NONE,
4060 			 },
4061 	 .auth_info = {
4062 		       .alg = HASH_ALG_AES,
4063 		       .mode = HASH_MODE_XCBC,
4064 		       },
4065 	 },
4066 	{
4067 	 .type = CRYPTO_ALG_TYPE_AHASH,
4068 	 .alg.hash = {
4069 		      .halg.digestsize = AES_BLOCK_SIZE,
4070 		      .halg.base = {
4071 				    .cra_name = "cmac(aes)",
4072 				    .cra_driver_name = "cmac-aes-iproc",
4073 				    .cra_blocksize = AES_BLOCK_SIZE,
4074 				}
4075 		      },
4076 	 .cipher_info = {
4077 			 .alg = CIPHER_ALG_NONE,
4078 			 .mode = CIPHER_MODE_NONE,
4079 			 },
4080 	 .auth_info = {
4081 		       .alg = HASH_ALG_AES,
4082 		       .mode = HASH_MODE_CMAC,
4083 		       },
4084 	 },
4085 };
4086 
generic_cra_init(struct crypto_tfm * tfm,struct iproc_alg_s * cipher_alg)4087 static int generic_cra_init(struct crypto_tfm *tfm,
4088 			    struct iproc_alg_s *cipher_alg)
4089 {
4090 	struct spu_hw *spu = &iproc_priv.spu;
4091 	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4092 	unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
4093 
4094 	flow_log("%s()\n", __func__);
4095 
4096 	ctx->alg = cipher_alg;
4097 	ctx->cipher = cipher_alg->cipher_info;
4098 	ctx->auth = cipher_alg->auth_info;
4099 	ctx->auth_first = cipher_alg->auth_first;
4100 	ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg,
4101 						    ctx->cipher.mode,
4102 						    blocksize);
4103 	ctx->fallback_cipher = NULL;
4104 
4105 	ctx->enckeylen = 0;
4106 	ctx->authkeylen = 0;
4107 
4108 	atomic_inc(&iproc_priv.stream_count);
4109 	atomic_inc(&iproc_priv.session_count);
4110 
4111 	return 0;
4112 }
4113 
skcipher_init_tfm(struct crypto_skcipher * skcipher)4114 static int skcipher_init_tfm(struct crypto_skcipher *skcipher)
4115 {
4116 	struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
4117 	struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
4118 	struct iproc_alg_s *cipher_alg;
4119 
4120 	flow_log("%s()\n", __func__);
4121 
4122 	crypto_skcipher_set_reqsize(skcipher, sizeof(struct iproc_reqctx_s));
4123 
4124 	cipher_alg = container_of(alg, struct iproc_alg_s, alg.skcipher);
4125 	return generic_cra_init(tfm, cipher_alg);
4126 }
4127 
ahash_cra_init(struct crypto_tfm * tfm)4128 static int ahash_cra_init(struct crypto_tfm *tfm)
4129 {
4130 	int err;
4131 	struct crypto_alg *alg = tfm->__crt_alg;
4132 	struct iproc_alg_s *cipher_alg;
4133 
4134 	cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s,
4135 				  alg.hash);
4136 
4137 	err = generic_cra_init(tfm, cipher_alg);
4138 	flow_log("%s()\n", __func__);
4139 
4140 	/*
4141 	 * export state size has to be < 512 bytes. So don't include msg bufs
4142 	 * in state size.
4143 	 */
4144 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4145 				 sizeof(struct iproc_reqctx_s));
4146 
4147 	return err;
4148 }
4149 
aead_cra_init(struct crypto_aead * aead)4150 static int aead_cra_init(struct crypto_aead *aead)
4151 {
4152 	unsigned int reqsize = sizeof(struct iproc_reqctx_s);
4153 	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4154 	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4155 	struct crypto_alg *alg = tfm->__crt_alg;
4156 	struct aead_alg *aalg = container_of(alg, struct aead_alg, base);
4157 	struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s,
4158 						      alg.aead);
4159 
4160 	int err = generic_cra_init(tfm, cipher_alg);
4161 
4162 	flow_log("%s()\n", __func__);
4163 
4164 	ctx->is_esp = false;
4165 	ctx->salt_len = 0;
4166 	ctx->salt_offset = 0;
4167 
4168 	/* random first IV */
4169 	get_random_bytes(ctx->iv, MAX_IV_SIZE);
4170 	flow_dump("  iv: ", ctx->iv, MAX_IV_SIZE);
4171 
4172 	if (err)
4173 		goto out;
4174 
4175 	if (!(alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK))
4176 		goto reqsize;
4177 
4178 	flow_log("%s() creating fallback cipher\n", __func__);
4179 
4180 	ctx->fallback_cipher = crypto_alloc_aead(alg->cra_name, 0,
4181 						 CRYPTO_ALG_ASYNC |
4182 						 CRYPTO_ALG_NEED_FALLBACK);
4183 	if (IS_ERR(ctx->fallback_cipher)) {
4184 		pr_err("%s() Error: failed to allocate fallback for %s\n",
4185 		       __func__, alg->cra_name);
4186 		return PTR_ERR(ctx->fallback_cipher);
4187 	}
4188 
4189 	reqsize += crypto_aead_reqsize(ctx->fallback_cipher);
4190 
4191 reqsize:
4192 	crypto_aead_set_reqsize(aead, reqsize);
4193 
4194 out:
4195 	return err;
4196 }
4197 
generic_cra_exit(struct crypto_tfm * tfm)4198 static void generic_cra_exit(struct crypto_tfm *tfm)
4199 {
4200 	atomic_dec(&iproc_priv.session_count);
4201 }
4202 
skcipher_exit_tfm(struct crypto_skcipher * tfm)4203 static void skcipher_exit_tfm(struct crypto_skcipher *tfm)
4204 {
4205 	generic_cra_exit(crypto_skcipher_tfm(tfm));
4206 }
4207 
aead_cra_exit(struct crypto_aead * aead)4208 static void aead_cra_exit(struct crypto_aead *aead)
4209 {
4210 	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4211 	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4212 
4213 	generic_cra_exit(tfm);
4214 
4215 	if (ctx->fallback_cipher) {
4216 		crypto_free_aead(ctx->fallback_cipher);
4217 		ctx->fallback_cipher = NULL;
4218 	}
4219 }
4220 
4221 /**
4222  * spu_functions_register() - Specify hardware-specific SPU functions based on
4223  * SPU type read from device tree.
4224  * @dev:	device structure
4225  * @spu_type:	SPU hardware generation
4226  * @spu_subtype: SPU hardware version
4227  */
spu_functions_register(struct device * dev,enum spu_spu_type spu_type,enum spu_spu_subtype spu_subtype)4228 static void spu_functions_register(struct device *dev,
4229 				   enum spu_spu_type spu_type,
4230 				   enum spu_spu_subtype spu_subtype)
4231 {
4232 	struct spu_hw *spu = &iproc_priv.spu;
4233 
4234 	if (spu_type == SPU_TYPE_SPUM) {
4235 		dev_dbg(dev, "Registering SPUM functions");
4236 		spu->spu_dump_msg_hdr = spum_dump_msg_hdr;
4237 		spu->spu_payload_length = spum_payload_length;
4238 		spu->spu_response_hdr_len = spum_response_hdr_len;
4239 		spu->spu_hash_pad_len = spum_hash_pad_len;
4240 		spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len;
4241 		spu->spu_assoc_resp_len = spum_assoc_resp_len;
4242 		spu->spu_aead_ivlen = spum_aead_ivlen;
4243 		spu->spu_hash_type = spum_hash_type;
4244 		spu->spu_digest_size = spum_digest_size;
4245 		spu->spu_create_request = spum_create_request;
4246 		spu->spu_cipher_req_init = spum_cipher_req_init;
4247 		spu->spu_cipher_req_finish = spum_cipher_req_finish;
4248 		spu->spu_request_pad = spum_request_pad;
4249 		spu->spu_tx_status_len = spum_tx_status_len;
4250 		spu->spu_rx_status_len = spum_rx_status_len;
4251 		spu->spu_status_process = spum_status_process;
4252 		spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload;
4253 		spu->spu_ccm_update_iv = spum_ccm_update_iv;
4254 		spu->spu_wordalign_padlen = spum_wordalign_padlen;
4255 		if (spu_subtype == SPU_SUBTYPE_SPUM_NS2)
4256 			spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload;
4257 		else
4258 			spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload;
4259 	} else {
4260 		dev_dbg(dev, "Registering SPU2 functions");
4261 		spu->spu_dump_msg_hdr = spu2_dump_msg_hdr;
4262 		spu->spu_ctx_max_payload = spu2_ctx_max_payload;
4263 		spu->spu_payload_length = spu2_payload_length;
4264 		spu->spu_response_hdr_len = spu2_response_hdr_len;
4265 		spu->spu_hash_pad_len = spu2_hash_pad_len;
4266 		spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len;
4267 		spu->spu_assoc_resp_len = spu2_assoc_resp_len;
4268 		spu->spu_aead_ivlen = spu2_aead_ivlen;
4269 		spu->spu_hash_type = spu2_hash_type;
4270 		spu->spu_digest_size = spu2_digest_size;
4271 		spu->spu_create_request = spu2_create_request;
4272 		spu->spu_cipher_req_init = spu2_cipher_req_init;
4273 		spu->spu_cipher_req_finish = spu2_cipher_req_finish;
4274 		spu->spu_request_pad = spu2_request_pad;
4275 		spu->spu_tx_status_len = spu2_tx_status_len;
4276 		spu->spu_rx_status_len = spu2_rx_status_len;
4277 		spu->spu_status_process = spu2_status_process;
4278 		spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload;
4279 		spu->spu_ccm_update_iv = spu2_ccm_update_iv;
4280 		spu->spu_wordalign_padlen = spu2_wordalign_padlen;
4281 	}
4282 }
4283 
4284 /**
4285  * spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox
4286  * channel for the SPU being probed.
4287  * @dev:  SPU driver device structure
4288  *
4289  * Return: 0 if successful
4290  *	   < 0 otherwise
4291  */
spu_mb_init(struct device * dev)4292 static int spu_mb_init(struct device *dev)
4293 {
4294 	struct mbox_client *mcl = &iproc_priv.mcl;
4295 	int err, i;
4296 
4297 	iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
4298 				  sizeof(struct mbox_chan *), GFP_KERNEL);
4299 	if (!iproc_priv.mbox)
4300 		return -ENOMEM;
4301 
4302 	mcl->dev = dev;
4303 	mcl->tx_block = false;
4304 	mcl->tx_tout = 0;
4305 	mcl->knows_txdone = true;
4306 	mcl->rx_callback = spu_rx_callback;
4307 	mcl->tx_done = NULL;
4308 
4309 	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4310 		iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
4311 		if (IS_ERR(iproc_priv.mbox[i])) {
4312 			err = PTR_ERR(iproc_priv.mbox[i]);
4313 			dev_err(dev,
4314 				"Mbox channel %d request failed with err %d",
4315 				i, err);
4316 			iproc_priv.mbox[i] = NULL;
4317 			goto free_channels;
4318 		}
4319 	}
4320 
4321 	return 0;
4322 free_channels:
4323 	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4324 		if (iproc_priv.mbox[i])
4325 			mbox_free_channel(iproc_priv.mbox[i]);
4326 	}
4327 
4328 	return err;
4329 }
4330 
spu_mb_release(struct platform_device * pdev)4331 static void spu_mb_release(struct platform_device *pdev)
4332 {
4333 	int i;
4334 
4335 	for (i = 0; i < iproc_priv.spu.num_chan; i++)
4336 		mbox_free_channel(iproc_priv.mbox[i]);
4337 }
4338 
spu_counters_init(void)4339 static void spu_counters_init(void)
4340 {
4341 	int i;
4342 	int j;
4343 
4344 	atomic_set(&iproc_priv.session_count, 0);
4345 	atomic_set(&iproc_priv.stream_count, 0);
4346 	atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
4347 	atomic64_set(&iproc_priv.bytes_in, 0);
4348 	atomic64_set(&iproc_priv.bytes_out, 0);
4349 	for (i = 0; i < SPU_OP_NUM; i++) {
4350 		atomic_set(&iproc_priv.op_counts[i], 0);
4351 		atomic_set(&iproc_priv.setkey_cnt[i], 0);
4352 	}
4353 	for (i = 0; i < CIPHER_ALG_LAST; i++)
4354 		for (j = 0; j < CIPHER_MODE_LAST; j++)
4355 			atomic_set(&iproc_priv.cipher_cnt[i][j], 0);
4356 
4357 	for (i = 0; i < HASH_ALG_LAST; i++) {
4358 		atomic_set(&iproc_priv.hash_cnt[i], 0);
4359 		atomic_set(&iproc_priv.hmac_cnt[i], 0);
4360 	}
4361 	for (i = 0; i < AEAD_TYPE_LAST; i++)
4362 		atomic_set(&iproc_priv.aead_cnt[i], 0);
4363 
4364 	atomic_set(&iproc_priv.mb_no_spc, 0);
4365 	atomic_set(&iproc_priv.mb_send_fail, 0);
4366 	atomic_set(&iproc_priv.bad_icv, 0);
4367 }
4368 
spu_register_skcipher(struct iproc_alg_s * driver_alg)4369 static int spu_register_skcipher(struct iproc_alg_s *driver_alg)
4370 {
4371 	struct skcipher_alg *crypto = &driver_alg->alg.skcipher;
4372 	int err;
4373 
4374 	crypto->base.cra_module = THIS_MODULE;
4375 	crypto->base.cra_priority = cipher_pri;
4376 	crypto->base.cra_alignmask = 0;
4377 	crypto->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4378 	crypto->base.cra_flags = CRYPTO_ALG_ASYNC |
4379 				 CRYPTO_ALG_ALLOCATES_MEMORY |
4380 				 CRYPTO_ALG_KERN_DRIVER_ONLY;
4381 
4382 	crypto->init = skcipher_init_tfm;
4383 	crypto->exit = skcipher_exit_tfm;
4384 	crypto->setkey = skcipher_setkey;
4385 	crypto->encrypt = skcipher_encrypt;
4386 	crypto->decrypt = skcipher_decrypt;
4387 
4388 	err = crypto_register_skcipher(crypto);
4389 	/* Mark alg as having been registered, if successful */
4390 	if (err == 0)
4391 		driver_alg->registered = true;
4392 	pr_debug("  registered skcipher %s\n", crypto->base.cra_driver_name);
4393 	return err;
4394 }
4395 
spu_register_ahash(struct iproc_alg_s * driver_alg)4396 static int spu_register_ahash(struct iproc_alg_s *driver_alg)
4397 {
4398 	struct spu_hw *spu = &iproc_priv.spu;
4399 	struct ahash_alg *hash = &driver_alg->alg.hash;
4400 	int err;
4401 
4402 	/* AES-XCBC is the only AES hash type currently supported on SPU-M */
4403 	if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4404 	    (driver_alg->auth_info.mode != HASH_MODE_XCBC) &&
4405 	    (spu->spu_type == SPU_TYPE_SPUM))
4406 		return 0;
4407 
4408 	/* SHA3 algorithm variants are not registered for SPU-M or SPU2. */
4409 	if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) &&
4410 	    (spu->spu_subtype != SPU_SUBTYPE_SPU2_V2))
4411 		return 0;
4412 
4413 	hash->halg.base.cra_module = THIS_MODULE;
4414 	hash->halg.base.cra_priority = hash_pri;
4415 	hash->halg.base.cra_alignmask = 0;
4416 	hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4417 	hash->halg.base.cra_init = ahash_cra_init;
4418 	hash->halg.base.cra_exit = generic_cra_exit;
4419 	hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC |
4420 				    CRYPTO_ALG_ALLOCATES_MEMORY;
4421 	hash->halg.statesize = sizeof(struct spu_hash_export_s);
4422 
4423 	if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
4424 		hash->init = ahash_init;
4425 		hash->update = ahash_update;
4426 		hash->final = ahash_final;
4427 		hash->finup = ahash_finup;
4428 		hash->digest = ahash_digest;
4429 		if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4430 		    ((driver_alg->auth_info.mode == HASH_MODE_XCBC) ||
4431 		    (driver_alg->auth_info.mode == HASH_MODE_CMAC))) {
4432 			hash->setkey = ahash_setkey;
4433 		}
4434 	} else {
4435 		hash->setkey = ahash_hmac_setkey;
4436 		hash->init = ahash_hmac_init;
4437 		hash->update = ahash_hmac_update;
4438 		hash->final = ahash_hmac_final;
4439 		hash->finup = ahash_hmac_finup;
4440 		hash->digest = ahash_hmac_digest;
4441 	}
4442 	hash->export = ahash_export;
4443 	hash->import = ahash_import;
4444 
4445 	err = crypto_register_ahash(hash);
4446 	/* Mark alg as having been registered, if successful */
4447 	if (err == 0)
4448 		driver_alg->registered = true;
4449 	pr_debug("  registered ahash %s\n",
4450 		 hash->halg.base.cra_driver_name);
4451 	return err;
4452 }
4453 
spu_register_aead(struct iproc_alg_s * driver_alg)4454 static int spu_register_aead(struct iproc_alg_s *driver_alg)
4455 {
4456 	struct aead_alg *aead = &driver_alg->alg.aead;
4457 	int err;
4458 
4459 	aead->base.cra_module = THIS_MODULE;
4460 	aead->base.cra_priority = aead_pri;
4461 	aead->base.cra_alignmask = 0;
4462 	aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4463 
4464 	aead->base.cra_flags |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4465 	/* setkey set in alg initialization */
4466 	aead->setauthsize = aead_setauthsize;
4467 	aead->encrypt = aead_encrypt;
4468 	aead->decrypt = aead_decrypt;
4469 	aead->init = aead_cra_init;
4470 	aead->exit = aead_cra_exit;
4471 
4472 	err = crypto_register_aead(aead);
4473 	/* Mark alg as having been registered, if successful */
4474 	if (err == 0)
4475 		driver_alg->registered = true;
4476 	pr_debug("  registered aead %s\n", aead->base.cra_driver_name);
4477 	return err;
4478 }
4479 
4480 /* register crypto algorithms the device supports */
spu_algs_register(struct device * dev)4481 static int spu_algs_register(struct device *dev)
4482 {
4483 	int i, j;
4484 	int err;
4485 
4486 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4487 		switch (driver_algs[i].type) {
4488 		case CRYPTO_ALG_TYPE_SKCIPHER:
4489 			err = spu_register_skcipher(&driver_algs[i]);
4490 			break;
4491 		case CRYPTO_ALG_TYPE_AHASH:
4492 			err = spu_register_ahash(&driver_algs[i]);
4493 			break;
4494 		case CRYPTO_ALG_TYPE_AEAD:
4495 			err = spu_register_aead(&driver_algs[i]);
4496 			break;
4497 		default:
4498 			dev_err(dev,
4499 				"iproc-crypto: unknown alg type: %d",
4500 				driver_algs[i].type);
4501 			err = -EINVAL;
4502 		}
4503 
4504 		if (err) {
4505 			dev_err(dev, "alg registration failed with error %d\n",
4506 				err);
4507 			goto err_algs;
4508 		}
4509 	}
4510 
4511 	return 0;
4512 
4513 err_algs:
4514 	for (j = 0; j < i; j++) {
4515 		/* Skip any algorithm not registered */
4516 		if (!driver_algs[j].registered)
4517 			continue;
4518 		switch (driver_algs[j].type) {
4519 		case CRYPTO_ALG_TYPE_SKCIPHER:
4520 			crypto_unregister_skcipher(&driver_algs[j].alg.skcipher);
4521 			driver_algs[j].registered = false;
4522 			break;
4523 		case CRYPTO_ALG_TYPE_AHASH:
4524 			crypto_unregister_ahash(&driver_algs[j].alg.hash);
4525 			driver_algs[j].registered = false;
4526 			break;
4527 		case CRYPTO_ALG_TYPE_AEAD:
4528 			crypto_unregister_aead(&driver_algs[j].alg.aead);
4529 			driver_algs[j].registered = false;
4530 			break;
4531 		}
4532 	}
4533 	return err;
4534 }
4535 
4536 /* ==================== Kernel Platform API ==================== */
4537 
4538 static struct spu_type_subtype spum_ns2_types = {
4539 	SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2
4540 };
4541 
4542 static struct spu_type_subtype spum_nsp_types = {
4543 	SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP
4544 };
4545 
4546 static struct spu_type_subtype spu2_types = {
4547 	SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1
4548 };
4549 
4550 static struct spu_type_subtype spu2_v2_types = {
4551 	SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2
4552 };
4553 
4554 static const struct of_device_id bcm_spu_dt_ids[] = {
4555 	{
4556 		.compatible = "brcm,spum-crypto",
4557 		.data = &spum_ns2_types,
4558 	},
4559 	{
4560 		.compatible = "brcm,spum-nsp-crypto",
4561 		.data = &spum_nsp_types,
4562 	},
4563 	{
4564 		.compatible = "brcm,spu2-crypto",
4565 		.data = &spu2_types,
4566 	},
4567 	{
4568 		.compatible = "brcm,spu2-v2-crypto",
4569 		.data = &spu2_v2_types,
4570 	},
4571 	{ /* sentinel */ }
4572 };
4573 
4574 MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids);
4575 
spu_dt_read(struct platform_device * pdev)4576 static int spu_dt_read(struct platform_device *pdev)
4577 {
4578 	struct device *dev = &pdev->dev;
4579 	struct spu_hw *spu = &iproc_priv.spu;
4580 	struct resource *spu_ctrl_regs;
4581 	const struct spu_type_subtype *matched_spu_type;
4582 	struct device_node *dn = pdev->dev.of_node;
4583 	int err, i;
4584 
4585 	/* Count number of mailbox channels */
4586 	spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
4587 
4588 	matched_spu_type = of_device_get_match_data(dev);
4589 	if (!matched_spu_type) {
4590 		dev_err(dev, "Failed to match device\n");
4591 		return -ENODEV;
4592 	}
4593 
4594 	spu->spu_type = matched_spu_type->type;
4595 	spu->spu_subtype = matched_spu_type->subtype;
4596 
4597 	for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
4598 		platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
4599 
4600 		spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
4601 		if (IS_ERR(spu->reg_vbase[i])) {
4602 			err = PTR_ERR(spu->reg_vbase[i]);
4603 			dev_err(dev, "Failed to map registers: %d\n",
4604 				err);
4605 			spu->reg_vbase[i] = NULL;
4606 			return err;
4607 		}
4608 	}
4609 	spu->num_spu = i;
4610 	dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
4611 
4612 	return 0;
4613 }
4614 
bcm_spu_probe(struct platform_device * pdev)4615 static int bcm_spu_probe(struct platform_device *pdev)
4616 {
4617 	struct device *dev = &pdev->dev;
4618 	struct spu_hw *spu = &iproc_priv.spu;
4619 	int err;
4620 
4621 	iproc_priv.pdev  = pdev;
4622 	platform_set_drvdata(iproc_priv.pdev,
4623 			     &iproc_priv);
4624 
4625 	err = spu_dt_read(pdev);
4626 	if (err < 0)
4627 		goto failure;
4628 
4629 	err = spu_mb_init(dev);
4630 	if (err < 0)
4631 		goto failure;
4632 
4633 	if (spu->spu_type == SPU_TYPE_SPUM)
4634 		iproc_priv.bcm_hdr_len = 8;
4635 	else if (spu->spu_type == SPU_TYPE_SPU2)
4636 		iproc_priv.bcm_hdr_len = 0;
4637 
4638 	spu_functions_register(dev, spu->spu_type, spu->spu_subtype);
4639 
4640 	spu_counters_init();
4641 
4642 	spu_setup_debugfs();
4643 
4644 	err = spu_algs_register(dev);
4645 	if (err < 0)
4646 		goto fail_reg;
4647 
4648 	return 0;
4649 
4650 fail_reg:
4651 	spu_free_debugfs();
4652 failure:
4653 	spu_mb_release(pdev);
4654 	dev_err(dev, "%s failed with error %d.\n", __func__, err);
4655 
4656 	return err;
4657 }
4658 
bcm_spu_remove(struct platform_device * pdev)4659 static void bcm_spu_remove(struct platform_device *pdev)
4660 {
4661 	int i;
4662 	struct device *dev = &pdev->dev;
4663 	char *cdn;
4664 
4665 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4666 		/*
4667 		 * Not all algorithms were registered, depending on whether
4668 		 * hardware is SPU or SPU2.  So here we make sure to skip
4669 		 * those algorithms that were not previously registered.
4670 		 */
4671 		if (!driver_algs[i].registered)
4672 			continue;
4673 
4674 		switch (driver_algs[i].type) {
4675 		case CRYPTO_ALG_TYPE_SKCIPHER:
4676 			crypto_unregister_skcipher(&driver_algs[i].alg.skcipher);
4677 			dev_dbg(dev, "  unregistered cipher %s\n",
4678 				driver_algs[i].alg.skcipher.base.cra_driver_name);
4679 			driver_algs[i].registered = false;
4680 			break;
4681 		case CRYPTO_ALG_TYPE_AHASH:
4682 			crypto_unregister_ahash(&driver_algs[i].alg.hash);
4683 			cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name;
4684 			dev_dbg(dev, "  unregistered hash %s\n", cdn);
4685 			driver_algs[i].registered = false;
4686 			break;
4687 		case CRYPTO_ALG_TYPE_AEAD:
4688 			crypto_unregister_aead(&driver_algs[i].alg.aead);
4689 			dev_dbg(dev, "  unregistered aead %s\n",
4690 				driver_algs[i].alg.aead.base.cra_driver_name);
4691 			driver_algs[i].registered = false;
4692 			break;
4693 		}
4694 	}
4695 	spu_free_debugfs();
4696 	spu_mb_release(pdev);
4697 }
4698 
4699 /* ===== Kernel Module API ===== */
4700 
4701 static struct platform_driver bcm_spu_pdriver = {
4702 	.driver = {
4703 		   .name = "brcm-spu-crypto",
4704 		   .of_match_table = of_match_ptr(bcm_spu_dt_ids),
4705 		   },
4706 	.probe = bcm_spu_probe,
4707 	.remove_new = bcm_spu_remove,
4708 };
4709 module_platform_driver(bcm_spu_pdriver);
4710 
4711 MODULE_AUTHOR("Rob Rice <rob.rice@broadcom.com>");
4712 MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver");
4713 MODULE_LICENSE("GPL v2");
4714